Improved CI/CD pipeline, added dependencies, and other miscellaneous improvements

.github/workflows/pr_code_changes.yaml

@@ -9,6 +9,7 @@ on:
     paths:
       - policyengine_us_data/**
       - tests/**
+      - .github/workflows/**
 
 jobs:
   Lint:
@@ -19,8 +20,47 @@ jobs:
           uses: "lgeiger/black-action@master"
           with:
               args: ". -l 79 --check"
-  Test:
+
+  SmokeTestForMultipleVersions:
+    # Updated name to include OS
+    name: Test Minimal Install & Import (${{ matrix.os }}, py${{ matrix.python-version }})
+    # Runs on the OS defined in the matrix strategy
+    runs-on: ${{ matrix.os }}
+    needs: Lint
+    strategy:
+      # Optional: Prevent one failure from cancelling all other matrix jobs
+      fail-fast: false
+      matrix:
+        # Add os to the matrix
+        os: [ubuntu-latest, windows-latest]
+        python-version: ['3.10', '3.11', '3.12']
+        # You could add include/exclude rules here if specific combinations
+        # are not needed or expected to fail.
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install package ONLY (no dev deps)
+        # Using python -m pip is slightly more robust
+        run: python -m pip install .
+
+      - name: Test basic import
+        run: python -c "import policyengine_us_data; print('Minimal import OK')"
+        # Default shell works fine for this on both Ubuntu (bash) and Windows (pwsh)
+
+      - name: Test specific core import
+        # Corrected indentation from your example
+        run: python -c "from policyengine_core.data import Dataset; print('Core import OK')"
+        # Default shell also works fine here
+
+  Test-Ubuntu:
       runs-on: ubuntu-latest
+      needs: Lint
       steps:
           - name: Checkout repo
             uses: actions/checkout@v2
@@ -30,7 +70,7 @@ jobs:
           - name: Set up Python
             uses: actions/setup-python@v2
             with:
-                python-version: '3.10'
+                python-version: '3.12'
 
           - name: Install package
             run: uv pip install -e .[dev] --system
@@ -46,4 +86,50 @@ jobs:
           - name: Run tests
             run: pytest
           - name: Test documentation builds
-            run: make documentation
+            run: make documentation
+
+  Test-Windows: # New parallel job for Windows
+    runs-on: windows-latest
+    needs: Lint
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v1
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install package and dev dependencies
+        # Use default shell (PowerShell) for Windows runner
+        run: uv pip install -e .[dev] --system
+        shell: pwsh # Explicitly state shell if needed, though usually default
+
+      - name: Download data inputs
+        # Replaced 'make download' using Python scripts
+        run: |
+          python policyengine_us_data/storage/download_public_prerequisites.py
+          python policyengine_us_data/storage/download_private_prerequisites.py
+        env:
+          HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}
+          POLICYENGINE_US_DATA_GITHUB_TOKEN: ${{ secrets.POLICYENGINE_US_DATA_GITHUB_TOKEN }}
+        shell: pwsh # Or cmd if python paths work better there, test needed
+
+      - name: Build datasets
+        run: |
+          python policyengine_us_data/datasets/acs/acs.py
+          python policyengine_us_data/datasets/cps/cps.py
+          python policyengine_us_data/datasets/puf/irs_puf.py
+          python policyengine_us_data/datasets/puf/puf.py
+          python policyengine_us_data/datasets/cps/extended_cps.py
+          python policyengine_us_data/datasets/cps/enhanced_cps.py
+        env:
+          LITE_MODE: "true"
+        shell: pwsh
+
+      - name: Run tests
+        run: pytest
+        shell: pwsh

README.md

@@ -2,9 +2,17 @@
 
 ## Installation
 
+While it is possible to install via PyPi:
 ```bash
 pip install policyengine-us-data
 ```
+the recommended installion is 
+```
+pip install -e .[dev]
+```
+which installs the development dependencies in a reference-only manner (so that changes
+to the package code will be reflected immediately); `policyengine-us-data` is a dev package
+and not intended for direct access.
 
 ## Building the Paper
 

policyengine_us_data/datasets/cps/cps.py

@@ -1,3 +1,4 @@
+from importlib.resources import files
 from policyengine_core.data import Dataset
 from policyengine_us_data.storage import STORAGE_FOLDER
 import h5py
@@ -340,10 +341,13 @@ def add_personal_income_variables(
         year (int): The CPS year
     """
     # Get income imputation parameters.
-    yamlfilename = os.path.join(
-        os.path.abspath(os.path.dirname(__file__)),
-        "imputation_parameters.yaml",
+    yamlfilename = (
+        files("policyengine_us_data")
+        / "datasets"
+        / "cps"
+        / "imputation_parameters.yaml"
     )
+
     with open(yamlfilename, "r", encoding="utf-8") as yamlfile:
         p = yaml.safe_load(yamlfile)
     assert isinstance(p, dict)

pyproject.toml

@@ -22,6 +22,7 @@ dependencies = [
     "requests",
     "tqdm",
     "microdf_python>=0.4.3",
+    "setuptools>=60"
 ]
 
 [project.optional-dependencies]
@@ -69,4 +70,4 @@ extend-exclude = '''
   | build
   | dist
 )/
-'''
+'''