refactor: make data a pyproject (#125)

With this refactoring, we can move data-only dependencies inside the
proper pyproject rather than artificially inflating the library
dependencies for no good reason. While there, add quality checks for
data like we do for library.

Author: bassosimone

.github/workflows/ci.yml

@@ -52,6 +52,33 @@ jobs:
             exit 1
           }
 
+      - name: Check code formatting with ruff (data)
+        working-directory: data
+        run: |
+          uv run ruff format --check || {
+            echo "❌ Code formatting check failed!"
+            echo "To fix locally, run: cd data && uv run ruff format"
+            exit 1
+          }
+
+      - name: Lint code with ruff (data)
+        working-directory: data
+        run: |
+          uv run ruff check || {
+            echo "❌ Linting check failed!"
+            echo "To fix locally, run: cd data && uv run ruff check --fix"
+            exit 1
+          }
+
+      - name: Type check with pyright (data)
+        working-directory: data
+        run: |
+          uv run pyright || {
+            echo "❌ Type checking failed!"
+            echo "To see errors locally, run: cd data && uv run pyright"
+            exit 1
+          }
+
   test-all:
     name: Test All Components
     runs-on: ubuntu-latest

data/generate_data.py

@@ -10,11 +10,9 @@
 import os
 import sys
 from dataclasses import dataclass
+from datetime import date, datetime
 from pathlib import Path
 
-# Add library to path so we can import iqb modules
-sys.path.insert(0, str(Path(__file__).parent.parent / "library" / "src"))
-
 import click
 import dacite
 import yaml
@@ -48,6 +46,11 @@ class PipelineConfig:
 def load_pipeline_config(config_path):
     """Load pipeline configuration matrix from YAML script."""
 
+    def coerce_str(value):
+        if isinstance(value, (date, datetime)):
+            return value.isoformat()
+        return value
+
     try:
         content = config_path.read_text()
     except FileNotFoundError as exc:
@@ -62,26 +65,24 @@ def load_pipeline_config(config_path):
         raise click.ClickException("Pipeline config must be a mapping.")
 
     try:
-        config = dacite.from_dict(PipelineConfig, data)
+        config = dacite.from_dict(
+            PipelineConfig,
+            data,
+            config=dacite.Config(type_hooks={str: coerce_str}),
+        )
     except dacite.DaciteError as exc:
         raise click.ClickException(f"Invalid pipeline config: {exc}") from exc
 
     if config.version != "v0":
-        raise click.ClickException(
-            f"Unsupported pipeline config version: {config.version}"
-        )
+        raise click.ClickException(f"Unsupported pipeline config version: {config.version}")
 
     time_periods = [(entry.start, entry.end) for entry in config.matrix.dates]
     if not time_periods:
-        raise click.ClickException(
-            "Pipeline config matrix must include non-empty dates."
-        )
+        raise click.ClickException("Pipeline config matrix must include non-empty dates.")
 
     granularities = tuple(grain.strip() for grain in config.matrix.granularities)
     if not granularities or any(not grain for grain in granularities):
-        raise click.ClickException(
-            "Pipeline config matrix must include non-empty granularities."
-        )
+        raise click.ClickException("Pipeline config matrix must include non-empty granularities.")
 
     return time_periods, granularities
 

data/ghcache.py

@@ -41,7 +41,6 @@
 import sys
 from pathlib import Path
 
-
 MANIFEST_PATH = Path("state") / "ghremote" / "manifest.json"
 CACHE_DIR = Path("cache/v1")
 SHA256_PREFIX_LENGTH = 12
@@ -96,10 +95,7 @@ def validate_cache_path(path: str) -> bool:
         return False
 
     # Component 6: data.parquet or stats.json
-    if parts[5] not in ("data.parquet", "stats.json"):
-        return False
-
-    return True
+    return parts[5] in ("data.parquet", "stats.json")
 
 
 def mangle_path(local_path: str, sha256: str) -> str:
@@ -121,7 +117,7 @@ def load_manifest() -> dict:
     if not MANIFEST_PATH.exists():
         return {"v": 0, "files": {}}
 
-    with open(MANIFEST_PATH, "r") as f:
+    with open(MANIFEST_PATH) as f:
         return json.load(f)
 
 
@@ -219,9 +215,7 @@ def cmd_scan(args) -> int:
 
         # Prepare manifest entry (URL will need to be filled in manually or via script)
         # For now, use placeholder URL
-        url_placeholder = (
-            f"https://github.com/m-lab/iqb/releases/download/v0.2.0/{mangled_name}"
-        )
+        url_placeholder = f"https://github.com/m-lab/iqb/releases/download/v0.2.0/{mangled_name}"
 
         files_dict[rel_path] = {"sha256": sha256, "url": url_placeholder}
         files_to_upload.append(mangled_name)

data/pyproject.toml

@@ -0,0 +1,48 @@
+[project]
+name = "iqb-data"
+version = "0.1.0"
+requires-python = ">=3.13"
+dependencies = [
+    "click>=8.3.0",
+    "mlab-iqb",
+    "pyyaml>=6.0.0",
+]
+
+[tool.uv.sources]
+mlab-iqb = { workspace = true }
+
+[tool.ruff]
+line-length = 100
+target-version = "py313"
+
+[tool.ruff.format]
+indent-style = "space"
+quote-style = "double"
+line-ending = "lf"
+
+[tool.ruff.lint]
+select = [
+    "E",   # pycodestyle errors
+    "W",   # pycodestyle warnings
+    "F",   # pyflakes
+    "I",   # isort
+    "N",   # pep8-naming
+    "UP",  # pyupgrade
+    "B",   # flake8-bugbear
+    "C4",  # flake8-comprehensions
+    "SIM", # flake8-simplify
+]
+ignore = [
+    "E501",   # line too long (handled by formatter)
+    "SIM114", # combine if branches - prefer explicit branches for domain clarity
+]
+
+[tool.ruff.lint.isort]
+known-first-party = ["iqb"]
+
+[tool.pyright]
+include = ["generate_data.py", "tests"]
+exclude = ["**/__pycache__", ".venv"]
+pythonVersion = "3.13"
+typeCheckingMode = "basic"
+reportMissingTypeStubs = false

data/run_query.py

@@ -10,10 +10,10 @@
 
 from iqb.scripting import iqb_exception, iqb_logging, iqb_pipeline
 
+DEFAULT_PROJECT_ID = "measurement-lab"
 
-def main():
-    DEFAULT_PROJECT_ID = "measurement-lab"
 
+def main():
     parser = argparse.ArgumentParser(
         description="Execute BigQuery query template and save results to v1 Parquet cache"
     )
@@ -47,6 +47,7 @@ def main():
         pipeline = iqb_pipeline.create(data_dir=data_dir, project=args.project_id)
         pipeline.sync_mlab(
             args.granularity,
+            enable_bigquery=True,
             start_date=args.start_date,
             end_date=args.end_date,
         )

data/tests/generate_data_test.py

@@ -0,0 +1,115 @@
+from __future__ import annotations
+
+import importlib.util
+from pathlib import Path
+
+import pytest
+
+
+def load_generate_data_module():
+    module_path = Path(__file__).parents[1] / "generate_data.py"
+    spec = importlib.util.spec_from_file_location("generate_data", module_path)
+    if spec is None or spec.loader is None:
+        raise RuntimeError(f"Unable to load module from {module_path}")
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def test_load_pipeline_config_valid(tmp_path: Path):
+    module = load_generate_data_module()
+    config_path = tmp_path / "pipeline.yaml"
+    config_path.write_text(
+        "\n".join(
+            [
+                'version: "v0"',
+                "matrix:",
+                "  dates:",
+                "    - start: 2024-01-01",
+                "      end: 2024-02-01",
+                "  granularities:",
+                "    - day",
+                "    - week",
+                "",
+            ]
+        )
+    )
+
+    time_periods, granularities = module.load_pipeline_config(config_path)
+
+    assert time_periods == [("2024-01-01", "2024-02-01")]
+    assert granularities == ("day", "week")
+
+
+def test_load_pipeline_config_rejects_wrong_version(tmp_path: Path):
+    module = load_generate_data_module()
+    config_path = tmp_path / "pipeline.yaml"
+    config_path.write_text(
+        "\n".join(
+            [
+                'version: "v1"',
+                "matrix:",
+                "  dates:",
+                "    - start: 2024-01-01",
+                "      end: 2024-02-01",
+                "  granularities:",
+                "    - day",
+                "",
+            ]
+        )
+    )
+
+    with pytest.raises(module.click.ClickException, match="Unsupported pipeline config"):
+        module.load_pipeline_config(config_path)
+
+
+def test_load_pipeline_config_rejects_empty_dates(tmp_path: Path):
+    module = load_generate_data_module()
+    config_path = tmp_path / "pipeline.yaml"
+    config_path.write_text(
+        "\n".join(
+            [
+                'version: "v0"',
+                "matrix:",
+                "  dates: []",
+                "  granularities:",
+                "    - day",
+                "",
+            ]
+        )
+    )
+
+    with pytest.raises(module.click.ClickException, match="matrix must include non-empty"):
+        module.load_pipeline_config(config_path)
+
+
+def test_load_pipeline_config_rejects_blank_granularity(tmp_path: Path):
+    module = load_generate_data_module()
+    config_path = tmp_path / "pipeline.yaml"
+    config_path.write_text(
+        "\n".join(
+            [
+                'version: "v0"',
+                "matrix:",
+                "  dates:",
+                "    - start: 2024-01-01",
+                "      end: 2024-02-01",
+                "  granularities:",
+                "    - day",
+                '    - ""',
+                "",
+            ]
+        )
+    )
+
+    with pytest.raises(module.click.ClickException, match="matrix must include non-empty"):
+        module.load_pipeline_config(config_path)
+
+
+def test_load_pipeline_config_rejects_non_mapping(tmp_path: Path):
+    module = load_generate_data_module()
+    config_path = tmp_path / "pipeline.yaml"
+    config_path.write_text("- not-a-mapping")
+
+    with pytest.raises(module.click.ClickException, match="must be a mapping"):
+        module.load_pipeline_config(config_path)

library/pyproject.toml

@@ -16,12 +16,10 @@ dependencies = [
     "pyarrow>=14.0.0",
     "pandas>=2.0.0",
     "db-dtypes>=1.0.0",
-    "pyyaml>=6.0.0",
     "python-dateutil>=2.9.0.post0",
     "dacite>=1.9.2",
     "filelock>=3.20.1",
     "rich>=14.2.0",
-    "click>=8.3.0",
 ]
 
 [project.urls]

pyproject.toml

@@ -1,5 +1,5 @@
 [tool.uv.workspace]
-members = ["library", "prototype", "analysis"]
+members = ["library", "prototype", "analysis", "data"]
 
 # Pytest configuration for workspace-level test execution
 # This allows running "uv run pytest" from the root to test all workspace members
@@ -8,6 +8,7 @@ testpaths = [
     "library/tests",
     "prototype/tests",
     "analysis/tests",
+    "data/tests",
 ]
 python_files = ["*_test.py", "test_*.py"]
 python_classes = ["Test*"]