[WIP] New namespace for storing user schemas + analysis

.github/workflows/testing.yml

@@ -21,19 +21,19 @@ jobs:
     strategy:
       matrix:
         os: ["ubuntu-latest"]
-        python-version: ["3.10", "3.11"]
+        python-version: ["3.11", "3.12"]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v6
         with:
           fetch-depth: 0
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v6
         with:
           python-version: ${{ matrix.python-version }}
           cache: "pip"
       - name: Install dependencies
         run: |
-          python${{ matrix.python-version }} -m pip install --upgrade "pip<25.3" build pip-tools
+          python${{ matrix.python-version }} -m pip install --upgrade "pip<25.3" build pip-tools pre-commit
           python${{ matrix.python-version }} -m piptools compile --extra dev -o requirements.txt mpcontribs-client/setup.py mpcontribs-api/setup.py
           python${{ matrix.python-version }} -m pip install -r requirements.txt
           cd mpcontribs-api && python${{ matrix.python-version }} -m pip install --no-deps .
@@ -47,6 +47,13 @@ jobs:
         run: |
           $env:SSL_CERT_FILE=$(python${{ matrix.python-version }} -m certifi)
           echo "SSL_CERT_FILE=$env:SSL_CERT_FILE" >> $GITHUB_ENV
+
+      - name: Run pre-commit
+        run: |
+          python${{ matrix.python-version }} -m pip install pre-commit
+          pre-commit install
+          pre-commit run --all-files
+
       - name: Test Client and build
         env:
           MPCONTRIBS_API_KEY: ${{ secrets.MPCONTRIBS_API_KEY }}
@@ -59,6 +66,16 @@ jobs:
           python${{ matrix.python-version }} -m pytest -v -s --cov=mpcontribs/client --cov-report=term-missing --cov-report=xml --ignore=bravado
           python${{ matrix.python-version }} -m build --outdir ../dist
 
+      - name: Install lux and test with pytest
+        run: |
+          git lfs install
+          git lfs pull
+          cd mpcontribs-lux
+          python${{ matrix.python-version }} -m pip install -r requirements/requirements-${{ matrix.os }}_py${{ matrix.python-version }}_extras.txt
+          python${{ matrix.python-version }} -m pip install -e .
+          python${{ matrix.python-version }} -m pytest -n auto -x --cov=mpcontribs/lux --cov-report=xml
+
+
   auto-gen-release:
     needs:
       - test

.gitignore

@@ -31,3 +31,4 @@ mpcontribs-portal/mpcontribs/portal/templates/notebooks
 coverage.xml
 mpcontribs-api/supervisord.conf
 mpcontribs-portal/supervisord.conf
+**/.DS_Store

.pre-commit-config.yaml

@@ -2,8 +2,12 @@ exclude: |
   (?x)(
       ^mpcontribs-portal/mpcontribs/users/|
       ^binder/|
+      ^mpcontribs-api/|
       ^mpcontribs-ingester/|
-      ^mpcontribs-api/mpcontribs/api/redox_thermo_csp/
+      ^mpcontribs-io/|
+      ^mpcontribs-kernel-gateway/|
+      ^mpcontribs-portal/|
+      ^mpcontribs-serverless/
   )
 
 repos:
@@ -14,7 +18,6 @@ repos:
       - id: trailing-whitespace
       - id: check-added-large-files
       - id: check-merge-conflict
-      - id: fix-encoding-pragma
       - id: forbid-new-submodules
       - id: requirements-txt-fixer
 
@@ -35,3 +38,12 @@ repos:
     rev: 25.9.0
     hooks:
       - id: black
+
+  - repo: https://github.com/kynan/nbstripout
+    rev: 0.8.1
+    hooks:
+      - id: nbstripout
+        args:
+          - --drop-empty-cells
+          - --strip-init-cells
+          - --extra-keys=metadata.kernelspec

mpcontribs-client/README.md

@@ -33,7 +33,7 @@ Set the environment variable `SSL_CERT_FILE` to `$(python -m certifi)`.
 OverflowError: timeout value is too large
 ```
 
-Install the bravado fork ([PR](https://github.com/Yelp/bravado/pull/472)) manually via  
+Install the bravado fork ([PR](https://github.com/Yelp/bravado/pull/472)) manually via
 ```
 pip install "bravado[fido] @ git+https://github.com/tschaume/bravado@9ce06f2df7118e16af4a3d3fdc21ccfeedc5cd50#egg=bravado-11.0.3"
 ```

mpcontribs-client/requirements/deployment.txt

@@ -71,12 +71,12 @@ json2html==1.3.0
 jsonpointer==3.0.0
     # via jsonschema
 jsonref==1.1.0
-    # via bravado-core
-jsonschema[format-nongpl]==4.25.1
     # via
     #   bravado-core
     #   swagger-spec-validator
 jsonschema-specifications==2025.9.1
+    # via bravado-core
+jsonschema[format-nongpl]==4.25.1
     # via jsonschema
 kiwisolver==1.4.9
     # via matplotlib

mpcontribs-client/requirements/ubuntu-latest_py3.11.txt

@@ -71,12 +71,12 @@ json2html==1.3.0
 jsonpointer==3.0.0
     # via jsonschema
 jsonref==1.1.0
-    # via bravado-core
-jsonschema[format-nongpl]==4.25.1
     # via
     #   bravado-core
     #   swagger-spec-validator
 jsonschema-specifications==2025.9.1
+    # via bravado-core
+jsonschema[format-nongpl]==4.25.1
     # via jsonschema
 kiwisolver==1.4.9
     # via matplotlib

mpcontribs-client/requirements/ubuntu-latest_py3.11_extras.txt

@@ -79,12 +79,12 @@ json2html==1.3.0
 jsonpointer==3.0.0
     # via jsonschema
 jsonref==1.1.0
-    # via bravado-core
-jsonschema[format-nongpl]==4.25.1
     # via
     #   bravado-core
     #   swagger-spec-validator
 jsonschema-specifications==2025.9.1
+    # via bravado-core
+jsonschema[format-nongpl]==4.25.1
     # via jsonschema
 kiwisolver==1.4.9
     # via matplotlib

mpcontribs-client/requirements/ubuntu-latest_py3.12.txt

@@ -71,12 +71,12 @@ json2html==1.3.0
 jsonpointer==3.0.0
     # via jsonschema
 jsonref==1.1.0
-    # via bravado-core
-jsonschema[format-nongpl]==4.25.1
     # via
     #   bravado-core
     #   swagger-spec-validator
 jsonschema-specifications==2025.9.1
+    # via bravado-core
+jsonschema[format-nongpl]==4.25.1
     # via jsonschema
 kiwisolver==1.4.9
     # via matplotlib

mpcontribs-client/requirements/ubuntu-latest_py3.12_extras.txt

@@ -79,12 +79,12 @@ json2html==1.3.0
 jsonpointer==3.0.0
     # via jsonschema
 jsonref==1.1.0
-    # via bravado-core
-jsonschema[format-nongpl]==4.25.1
     # via
     #   bravado-core
     #   swagger-spec-validator
 jsonschema-specifications==2025.9.1
+    # via bravado-core
+jsonschema[format-nongpl]==4.25.1
     # via jsonschema
 kiwisolver==1.4.9
     # via matplotlib

mpcontribs-client/tests/conftest.py

@@ -6,7 +6,7 @@
 except ImportError:
     create_app = None
 
-os.environ['MPCONTRIBS_DB_NAME'] = 'mpcontribs-test'
+os.environ["MPCONTRIBS_DB_NAME"] = "mpcontribs-test"
 
 
 @pytest.fixture()

mpcontribs-lux/.python-version

@@ -0,0 +1 @@
+3.12

mpcontribs-lux/README.md

@@ -0,0 +1,44 @@
+## <span style="color:rgb(18, 180, 163)">MPContribs</span> <span style="color:goldenrod">LUX</span>
+
+<span style="color:goldenrod"><i><b>Ego sum lux datorum</b></i></span>.
+
+MPContribs-lux is a package which <it>sheds light</it> on data stored on the [Materials Project's AWS S3 MPContribs bucket](https://materialsproject-contribs.s3.amazonaws.com/index.html#) by providing annotated schemas and optionally analysis tools to better explore user-submitted data.
+
+Adding a schema to this database is a <span style="color:red"><b>pre-requisite</b></span> for obtaining permission/IAM credentials for uploading data to MP's MPContribs Bucket.
+Once a staff member from MP reviews and approves your data schema, your receive IAM role will be granted/updated (as appropriate).
+
+<span style="color:red"><b>What if I don't want my schemas / data made public yet?</b></span>
+
+To expedite the process of review, follow [these instructions](https://docs.github.com/en/repositories/creating-and-managing-repositories/duplicating-a-repository) to make a private copy (not a fork, which cannot be private) of the `MPContribs` repo.
+Suppose you name your new repository `PrivateMPContribs` and your username is `<username>`, you would run these commands from a terminal:
+
+```console
+git clone --bare https://github.com/materialsproject/MPContribs.git
+cd MPContribs
+git push --mirror https://github.com/<username>/PrivateMPContribs.git
+cd ..
+rm -rf MPContribs
+```
+
+Then add your schemas to the private repo `PrivateMPContribs` and invite the maintainers of `MPContribs` to view it (you don't need to give us edit access).
+We will then review your schemas.
+When you're ready to make your data public, you will also have to make a public PR with your new schemas.
+
+<span style="color:red"><b>But my CSV/JSON/YAML/etc. file isn't complicated. Why do I need to upload a schema?</b></span>
+
+Schemas are important for ensuring accessibility, interoperability, and reproducibility, and for ensuring that you are fully aware of possible errors in your dataset.
+
+If you are not comfortable mimicking the example `pydantic` schemas in `mpcontribs.lux.projects.examples`, you can either use the schema autogeneration features in `mpcontribs.lux.autogen`:
+
+```py
+from mpcontribs.lux.autogen import SchemaGenerator
+
+schema_gen = SchemaGenerator(file_name = "/path/to/some/csv/or/json/file")
+pydantic_model = schema_gen.pydantic_schema
+print(pydantic_model.schema())
+```
+
+...or reach out to the maintainers!
+
+The test suite for MPContribs-lux will automatically test your `pydantic` models for arrow/parquet compatibility using the [arrowize](https://github.com/materialsproject/emmet/blob/74194bbf8c7b32ce15141bb1c9ee0527a0fc6c45/emmet-core/emmet/core/arrow.py#L40) utility from `emmet-core` (MP's production data model and data pipeline repository).
+Schemas submitted to MPContribs-lux do not necessarily need to be parquet/arrow compatlible, the `@arrow_incompatible` decorator from `emmet-core`'s [utils](https://github.com/materialsproject/emmet/blob/74194bbf8c7b32ce15141bb1c9ee0527a0fc6c45/emmet-core/emmet/core/utils.py#L104) module can be used to mark a `pydantic` model to be skipped during arrow compatibility testing.