Merge pull request #38 from e-lo/pedro/pandas_compatibility

This makes GMNSPy compatible with Pandas 3.0, fixes #39.
Adds testing across multiple versions of Python and Pandas.
Fixes typos

Author: e-lo

.github/workflows/documentation.yml

@@ -13,13 +13,13 @@ jobs:
   build-deploy-docs:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v6
       with:
         fetch-depth: 0
     - name: Document which branch
       run: echo $DOC_REF_NAME
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v6
       with:
         python-version: '3.x'
         cache: 'pip'

.github/workflows/publish.yml

@@ -21,9 +21,9 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v6
       with:
         python-version: "3.x"
     - name: Install pypa/build
@@ -35,7 +35,7 @@ jobs:
     - name: Build a binary wheel and a source tarball
       run: python3 -m build
     - name: Store the distribution packages
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: python-package-distributions
         path: dist/
@@ -55,7 +55,7 @@ jobs:
 
     steps:
     - name: Download all the dists
-      uses: actions/download-artifact@v3
+      uses: actions/download-artifact@v4
       with:
         name: python-package-distributions
         path: dist/
@@ -78,7 +78,7 @@ jobs:
 
     steps:
     - name: Download all the dists
-      uses: actions/download-artifact@v3
+      uses: actions/download-artifact@v4
       with:
         name: python-package-distributions
         path: dist/

.github/workflows/tests.yml

@@ -5,16 +5,12 @@ on: [push, pull_request]
 jobs:
   linting:
     runs-on: ubuntu-latest
-    strategy:
-      max-parallel: 4
-      matrix:
-        python-version: ['3.10']
     steps:
-    - uses: actions/checkout@v3
-    - uses: actions/setup-python@v4
+    - uses: actions/checkout@v6
+    - uses: actions/setup-python@v6
       with:
-        python-version: 3.x
-    - uses: pre-commit/action@v3.0.0
+        python-version: '3.13'
+    - uses: pre-commit/action@v3.0.1
       with:
         extra_args: --all-files --verbose
   tests:
@@ -23,11 +19,11 @@ jobs:
     strategy:
       max-parallel: 4
       matrix:
-        python-version: ['3.10']
+        python-version: ['3.10', '3.11', '3.12', '3.13']
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v6
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v6
       with:
         python-version: ${{ matrix.python-version }}
         cache: 'pip'

.pre-commit-config.yaml

@@ -38,7 +38,7 @@ repos:
     - id: isort
       args: ["--profile", "black"]
   - repo: https://github.com/python-jsonschema/check-jsonschema
-    rev: 0.14.3
+    rev: 0.36.2
     hooks:
     - id: check-github-workflows
   - repo: http://github.com/pycqa/flake8

CONTRIBUTING.md

@@ -40,8 +40,8 @@ Get assigned. If you are working on issue, please tag yourself as the assignee (
 
 Generally:
 
-- Try to be backwards compatable to Python 3.10.
-- Be compatable with Numpy 2+, Pandas 2+, OSMNX 1+, PyProj 3.3+
+- Try to be backwards compatible to Python 3.10.
+- Be compatible with Numpy 2+, Pandas 2.x and 3.x, OSMNX 1+, PyProj 3.3+
 - Use PEP8 and autoformat with `black`
 - Use Google-style docstrings for all classes and methods
 - Test formatting and autoformat using `pre-commit`

gmnspy/in_out.py

@@ -16,7 +16,7 @@
 from gmnspy.validation import (
     apply_schema_to_df,
     check_required_files,
-    update_resources_based_on_existance,
+    update_resources_based_on_existence,
     validate_foreign_keys,
 )
 
@@ -91,7 +91,7 @@ def read_gmns_network(data_directory: str, config: str = None, raise_error=False
     check_required_files(resource_df, raise_error)
 
     # update resource dictionary based on what files are in the directory
-    resource_df = update_resources_based_on_existance(resource_df)
+    resource_df = update_resources_based_on_existence(resource_df)
 
     # read each csv to a df and validate format
     # todo add paired schema

gmnspy/schema.py

@@ -22,11 +22,11 @@
 from .utils import list_to_md_table, logger
 
 SCHEMA_TO_PANDAS_TYPES = {
-    "integer": "int64",
-    "number": "float",
+    "integer": "Int64",
+    "number": "Float64",
     "string": "string",
     "any": "object",
-    "boolean": "bool",
+    "boolean": "boolean",
 }
 
 FORMAT_TO_REGEX = {
@@ -37,6 +37,27 @@
 }
 
 
+def read_schema_for_resource(resource_df: pd.DataFrame, table_name: str, raise_error: bool) -> dict:
+    """
+    Read in schema from schema json file and returns as dictionary.
+
+    ##TODO validate schema itself
+
+    Args:
+        schema_file: File location of the schema json file.
+
+    Returns: The schema as a dictionary
+    """
+    matching_schema_paths = resource_df.loc[resource_df["name"] == table_name, "fullpath_schema"]
+    if matching_schema_paths.empty:
+        msg = f"FAIL. Could not find schema path for table {table_name}"
+        logger.error(msg)
+        if raise_error:
+            raise Exception(msg)
+        return dict()
+    return read_schema(schema_file=matching_schema_paths.iloc[0])
+
+
 def read_schema(schema_file: str) -> dict:
     """
     Read in schema from schema json file and returns as dictionary.
@@ -90,7 +111,7 @@ def read_config(config_file: str, data_dir: str = "", schema_dir: str = "") -> p
     ## todo validate config
 
     resource_df = pd.DataFrame(config["resources"])
-    resource_df["required"].fillna(False, inplace=True)
+    resource_df["required"] = resource_df["required"].fillna(False)
 
     logger.info(str(resource_df))
 
@@ -105,15 +126,15 @@ def read_config(config_file: str, data_dir: str = "", schema_dir: str = "") -> p
     resource_df["fullpath_schema"] = resource_df["schema"].apply(lambda x: join(schema_dir, x))
     logger.info(str(resource_df))
 
-    resource_df.set_index("name", drop=False, inplace=True)
+    resource_df = resource_df.set_index("name", drop=False)
     return resource_df
 
 
 def document_schemas_to_md(schema_path: str = None, out_path: str = None) -> str:
     """Create markdown for each **.schema.json file in schema_path.
 
     Args:
-        schema_path (str, optional): Path fo tlook for schema files.
+        schema_path (str, optional): Path to look for schema files.
             Defaults to join(dirname(realpath(__file__)), "spec")
         out_path (str, optional): If specified, will write out resulting markdown to this file.
             Defaults to None.

gmnspy/validation/__init__.py

@@ -2,5 +2,5 @@
 
 from .foreign_keys import validate_foreign_keys
 from .required_files import check_required_files
-from .resources_existance import update_resources_based_on_existance
+from .resources_existance import update_resources_based_on_existence
 from .schema_to_df import apply_schema_to_df

gmnspy/validation/foreign_keys.py

@@ -4,7 +4,7 @@
 
 import pandas as pd
 
-from gmnspy.schema import read_schema
+from gmnspy.schema import read_schema_for_resource
 from gmnspy.utils import logger
 
 
@@ -28,7 +28,9 @@ def validate_foreign_keys(gmns_net_d: Dict[str, pd.DataFrame], resource_df: pd.D
 
     fkey_errors = []
     for table_name, df in gmns_net_d.items():
-        schema = read_schema(schema_file=resource_df[resource_df["name"] == table_name]["fullpath_schema"][0])
+        schema = read_schema_for_resource(resource_df, table_name, raise_error)
+        if len(schema) == 0:
+            continue
 
         foreign_keys = [
             (f["name"], f["foreign_key"])

gmnspy/validation/required_files.py

@@ -16,7 +16,7 @@ def check_required_files(resource_df: pd.DataFrame, raise_error=False) -> None:
             the the columns "fullpath", "name", and "required" (boolean).
         raise_error: Raises error if missing folder
     """
-    req_files = resource_df[resource_df["required"]]
+    req_files = resource_df.query("required == True")
 
     missing_required_files = req_files[req_files["fullpath"].apply(lambda x: not exists(x))][["name", "fullpath"]]