Merge pull request #61 from atomscale-ai/enhancement/update_cicd_tooling

Modernize CICD tooling with astral packages

Author: munrojm

.github/workflows/release.yml

@@ -139,17 +139,21 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - uses: actions/setup-python@v5
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          version: "latest"
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
         with:
           python-version: "3.11"
 
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements/requirements-ubuntu-latest_py3.11.txt
-          pip install -r requirements/requirements-ubuntu-latest_py3.11_extras.txt
-          pip install sphinx
-          pip install -e .
+          uv pip install --system -r requirements/requirements-ubuntu-latest_py3.11.txt
+          uv pip install --system -r requirements/requirements-ubuntu-latest_py3.11_extras.txt
+          uv pip install --system -e .
 
       - name: Build docs
         run: sphinx-build ./docs ./docs/_build

.github/workflows/testing.yml

@@ -17,30 +17,24 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          version: "latest"
+
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
           python-version: "3.11"
 
-      - name: Install Python dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements/requirements-ubuntu-latest_py3.11.txt
-          pip install -r requirements/requirements-ubuntu-latest_py3.11_extras.txt
-
       - name: Install dependencies
         run: |
-          pip install pre-commit
+          uv pip install --system -r requirements/requirements-ubuntu-latest_py3.11.txt
+          uv pip install --system -r requirements/requirements-ubuntu-latest_py3.11_extras.txt
+          uv pip install --system pre-commit ty
 
       - name: Run pre-commit
-        run: |
-          pre-commit run --all-files --show-diff-on-failure
-
-      - name: Run mypy
-        run: |
-          mypy --version
-          rm -rf .mypy_cache
-          mypy src/atomicds
+        run: pre-commit run --all-files --show-diff-on-failure
 
   test:
     needs: lint
@@ -55,16 +49,20 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          version: "latest"
+
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Install Python dependencies
+      - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements/requirements-${{ matrix.os }}_py${{ matrix.python-version }}.txt
-          pip install -r requirements/requirements-${{ matrix.os }}_py${{ matrix.python-version }}_extras.txt
+          uv pip install --system -r requirements/requirements-${{ matrix.os }}_py${{ matrix.python-version }}.txt
+          uv pip install --system -r requirements/requirements-${{ matrix.os }}_py${{ matrix.python-version }}_extras.txt
 
       - name: Set SSL_CERT_FILE (Linux/MacOS)
         if: matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest'
@@ -74,13 +72,14 @@ jobs:
         if: matrix.os == 'windows-latest'
         run: echo "SSL_CERT_FILE=$(python -m certifi)" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append
 
+      - name: Install package
+        run: uv pip install --system -e .
+
       - name: Test with pytest
         env:
           AS_API_KEY: ${{ secrets.AS_API_KEY }}
           AS_API_ENDPOINT: "https://api.atomscale.ai"
-        run: |
-          pip install -e .
-          pytest -x
+        run: pytest -x
 
   auto-gen-release:
     needs: test

.pre-commit-config.yaml

@@ -5,19 +5,8 @@ ci:
   autoupdate_commit_msg: "chore: update pre-commit hooks"
 
 repos:
-  - repo: https://github.com/psf/black
-    rev: 23.9.1
-    hooks:
-      - id: black
-
-  - repo: https://github.com/asottile/blacken-docs
-    rev: "1.16.0"
-    hooks:
-      - id: blacken-docs
-        additional_dependencies: [black>=23.7.0]
-
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v5.0.0
     hooks:
       - id: check-case-conflict
       - id: check-symlinks
@@ -28,15 +17,27 @@ repos:
       - id: trailing-whitespace
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.3.4
+    rev: v0.9.4
     hooks:
       - id: ruff
         args: [--fix, --show-fixes]
+        exclude: '\.ipynb$'
+      - id: ruff-format
+        exclude: '\.ipynb$'
 
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.2.6
+    rev: v2.4.1
     hooks:
       - id: codespell
         stages: [pre-commit, commit-msg]
         exclude_types: [json, bib, svg]
         args: [--ignore-words-list, "mater,fwe,te"]
+
+  - repo: local
+    hooks:
+      - id: ty
+        name: ty
+        entry: ty check src/atomicds
+        language: system
+        types: [python]
+        pass_filenames: false

pyproject.toml

@@ -49,10 +49,11 @@ dev = [
   "pytest-order",
   "pytest-dependency",
   "pytest-asyncio",
-  "ruff",
-  "mypy",
+  "ruff>=0.9.0",
+  "ty",
   "types-requests",
   "pandas-stubs",
+  "pre-commit>=4.0.0",
 ]
 docs = ["sphinx>=6.2", "sphinx-autodoc-typehints>=1.24", "alabaster>=0.7"]
 
@@ -73,29 +74,13 @@ path = "src/atomscale/streaming/Cargo.toml"
 binding = "PyO3"
 
 
-[tool.pyright]
-include = ["atomscale", "atomicds"]
-exclude = ["**/__pycache__"]
-
 [tool.pytest.ini_options]
 minversion = "6.0"
 addopts = ["-p no:warnings", "--import-mode=importlib"]
 xfail_strict = true
 log_cli_level = "warn"
 testpaths = ["tests"]
 
-[tool.black]
-exclude = '''
-/(
-    \.git
-  | \.tox
-)/
-'''
-
-[tool.isort]
-profile = 'black'
-skip_gitignore = true
-
 [tool.coverage.run]
 source = ["src"]
 
@@ -137,7 +122,6 @@ lint.select = [
 lint.extend-ignore = [
   "PLR",   # Design related pylint codes
   "E501",  # Line too long
-  "PT004", # Use underscore for non-returning fixture (use usefixture instead)
   "B028",  # No explicit stacklevel
   "EM101", # Exception must not use a string literal
   "EM102", # Exception must not use an f-string literal
@@ -151,20 +135,19 @@ lint.unfixable = [
 ]
 exclude = []
 lint.flake8-unused-arguments.ignore-variadic-names = true
-extend-exclude = ["tests"]
+extend-exclude = ["tests", "examples", "*.ipynb"]
 lint.pydocstyle.convention = "google"
 lint.isort.known-first-party = ["atomscale"]
 
 [tool.docformatter]
 pre-summary-newline = true
-black = true
-
-[tool.mypy]
-ignore_missing_imports = true
-namespace_packages = true
-explicit_package_bases = true
-no_implicit_optional = false
-disable_error_code = "annotation-unchecked"
+
+[tool.ty.environment]
+root = ["./src"]
+
+[tool.ty.src]
+include = ["src/atomicds"]
+exclude = ["tests", "examples", "docs"]
 
 [tool.codespell]
 ignore-words-list = "fpr"

src/atomscale/core/client.py

@@ -245,9 +245,9 @@ def _create_session(api_key: str):
         atomscale_info = "atomscale/" + __version__
         python_info = f"Python/{sys.version.split()[0]}"
         platform_info = f"{platform.system()}/{platform.release()}"
-        session.headers[
-            "user-agent"
-        ] = f"{atomscale_info} ({python_info} {platform_info})"
+        session.headers["user-agent"] = (
+            f"{atomscale_info} ({python_info} {platform_info})"
+        )
 
         # TODO: Add retry setting to configuration somewhere
         max_retry_num = 3

src/atomscale/results/__init__.py

@@ -18,7 +18,7 @@
     "RHEEDImageResult",
     "RHEEDVideoResult",
     "RamanResult",
-    "XPSResult",
     "UnknownResult",
+    "XPSResult",
     "_get_rheed_image_result",
 ]

src/atomscale/results/rheed_image.py

@@ -371,9 +371,7 @@ def merge_overlaps(node_df):
                 x,
                 new_df["mask_height"].iloc[0],  # type: ignore  # noqa: PGH003
                 new_df["mask_width"].iloc[0],  # type: ignore  # noqa: PGH003
-            )[
-                "counts"
-            ]
+            )["counts"]
 
             new_df = new_df.groupby("node_id").agg(agg_dict).reset_index(drop=True)
 

src/atomscale/timeseries/rheed.py

@@ -41,8 +41,13 @@ class RHEEDProvider(TimeseriesProvider[RHEEDVideoResult]):
         "first_order_fwhm_1": "First Order FWHM",
         "lattice_spacing": "Lattice Spacing",
         "tar_metric": "TAR Metric",
+        "composition_metric": "Composition Metric",
     }
-    DROP_IF_ALL_NA: Sequence[str] = ["reconstruction_intensity", "tar_metric"]
+    DROP_IF_ALL_NA: Sequence[str] = [
+        "reconstruction_intensity",
+        "tar_metric",
+        "composition_metric",
+    ]
     INDEX_COLS: Sequence[str] = ["Angle", "Frame Number"]
 
     def fetch_raw(self, client: BaseClient, data_id: str, **kwargs) -> Any:

tests/test_rheed_video.py

@@ -22,6 +22,7 @@ def result(client: Client):
 
 
 def test_get_dataframe(result: RHEEDVideoResult):
+    # Core columns that should always be present
     column_names = set(
         [
             "Strain",
@@ -38,9 +39,12 @@ def test_get_dataframe(result: RHEEDVideoResult):
             "Half Order Intensity R",
             "Specular FWHM",
             "First Order FWHM",
-            "Time",
             "UNIX Timestamp",
             "Relative Time",
+            # Optional columns (included if data exists)
+            "Time",
+            "TAR Metric",
+            "Composition Metric",
         ]
     )