Merge pull request #1252 from CodeForPhilly/staging

Weekly PR from Staging to Main

Author: nlebovits

.pre-commit-config.yaml

@@ -1,61 +1,54 @@
 repos:
   # Python hooks
-  - repo: local
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    # Ruff version.
+    rev: v0.12.0
     hooks:
-      # Ruff for Python linting
+      # Run the linter.
       - id: ruff
-        name: ruff (Python linting)
-        entry: ruff check --fix
-        language: python
-        types_or: [python, pyi, jupyter]
-        files: \.py$
-
-      # Ruff for Python formatting
+        args: [--fix]
+      # Run the formatter.
       - id: ruff-format
-        name: ruff-format (Python formatting)
-        entry: ruff format
-        language: python
-        types_or: [python, pyi, jupyter]
-        files: \.py$
-
+  - repo: local
+    hooks:
       # Radon MI and Halstead checks
       - id: radon-check
         name: Radon Maintainability and Halstead Checks
         entry: |
-          bash -c 'radon mi -s {} | grep -E " - [CDEF] \(" && echo "Radon MI grade below B detected!" && exit 1 || echo "All MI grades are B or above."
-          radon hal {} -s | awk "/effort/ { if (\$3 > 2000) print FILENAME \": High Halstead effort:\" \$3 }"'
+          bash -c 'for file in "$@"; do 
+            echo "Checking $file"; 
+            radon mi -s "$file" | grep -E " - [CDEF] \(" && echo "Radon MI grade below B detected in $file!" && exit 1; 
+            radon hal "$file" -s | awk -v filename="$file" "/effort/ { if (\$3 > 2000) print filename \": High Halstead effort:\" \$3 }"; 
+          done; 
+          echo "All MI grades are B or above."'
         language: system
         files: \.py$
-
+        pass_filenames: true
   - repo: https://github.com/jendrikseipp/vulture
-    rev: 'v2.3' # or any later Vulture version
+    rev: 'v2.3'
     hooks:
       - id: vulture
         name: vulture (Dead code detection)
-        entry: vulture data/
-        language: python
-        types: [python]
-        files: ^data/
-
+        args: [--min-confidence, '80', data/src]
   # JavaScript hooks
   - repo: local
     hooks:
-      # Formatting with Prettier
+      # Formatting with Prettier - run directly on files
       - id: prettier
         name: Prettier (Code formatting)
-        entry: npm run format
+        entry: npx prettier --write
         language: node
-        files: \.(js|jsx|ts|tsx)$
-
-      # Linting with ESLint
+        files: \.(js|jsx|ts|tsx|json|css|scss|md|yml|yaml)$
+      # Linting with ESLint - run directly on files
       - id: eslint
         name: ESLint (JavaScript linting)
-        entry: npm run lint
+        entry: npx eslint --fix
         language: node
         files: \.(js|jsx|ts|tsx)$
-
       # Dependency checks for Node.js
       - id: npm-audit
         name: Check Node.js dependencies
         entry: npm audit
         language: node
+        files: ^package(-lock)?\.json$
+        pass_filenames: false

data/src/classes/data_diff.py

@@ -35,6 +35,8 @@ def generate_diff(self):
             print(
                 f"Table {self.table_name} has less than two separate files with different timestamps. Cannot perform comparison"
             )
+            self.summary_text = f"Table {self.table_name} has less than two separate files with different timestamps. Cannot perform comparison"
+            return self
 
         def extract_date(str) -> datetime:
             pattern = "\b\d{4}_\d{1,2}_\d{1,2}\b"

data/src/classes/file_manager.py

@@ -1,4 +1,6 @@
+import glob
 import os
+import time
 import zipfile
 from datetime import datetime
 from enum import Enum
@@ -118,12 +120,30 @@ def check_source_cache_file_exists(
             table_name (str): The name of the table of source data.
             load_type (LoadType): The destination type of the file (either SOURCE_CACHE or PIPELINE_CACHE).
         """
+        start_time = time.time()
+        print(
+            f"    FileManager.check_source_cache_file_exists: Checking for {table_name}"
+        )
+
         directory = (
             self.source_cache_directory
             if load_type == LoadType.SOURCE_CACHE
             else self.pipeline_cache_directory
         )
-        return len([file for file in os.listdir(directory) if table_name in file]) > 0
+        # Use glob pattern matching for more efficient file searching
+        pattern = os.path.join(directory, f"*{table_name}*.parquet")
+
+        glob_start = time.time()
+        files = glob.glob(pattern)
+        glob_time = time.time() - glob_start
+
+        result = len(files) > 0
+        total_time = time.time() - start_time
+
+        print(
+            f"    FileManager.check_source_cache_file_exists: Found {len(files)} files in {glob_time:.2f}s (total: {total_time:.2f}s)"
+        )
+        return result
 
     def get_most_recent_cache(self, table_name: str) -> gpd.GeoDataFrame | None:
         """
@@ -134,25 +154,45 @@ def get_most_recent_cache(self, table_name: str) -> gpd.GeoDataFrame | None:
             GeoDataFrame: The dataframe loaded from the most recent cached file.
             None: If no files exist for the given table name.
         """
-        cached_files = [
-            file
-            for file in os.listdir(self.source_cache_directory)
-            if table_name in file
-        ]
+        start_time = time.time()
+        print(
+            f"    FileManager.get_most_recent_cache: Loading most recent cache for {table_name}"
+        )
+
+        # Use glob pattern matching for more efficient file searching
+        pattern = os.path.join(self.source_cache_directory, f"*{table_name}*.parquet")
+
+        glob_start = time.time()
+        cached_files = glob.glob(pattern)
+        glob_time = time.time() - glob_start
 
         if not cached_files:
+            print("    FileManager.get_most_recent_cache: No cached files found")
             return None
 
-        cached_files.sort(
-            key=lambda x: os.path.getmtime(
-                os.path.join(self.source_cache_directory, x)
-            ),
-            reverse=True,
+        # Get the most recent file by modification time
+        mtime_start = time.time()
+        most_recent_file = max(cached_files, key=os.path.getmtime)
+        mtime_time = time.time() - mtime_start
+
+        print(
+            f"    FileManager.get_most_recent_cache: Found {len(cached_files)} files, most recent: {os.path.basename(most_recent_file)}"
+        )
+        print(
+            f"    FileManager.get_most_recent_cache: Glob took {glob_time:.2f}s, mtime check took {mtime_time:.2f}s"
+        )
+
+        # Load the parquet file
+        load_start = time.time()
+        gdf = gpd.read_parquet(most_recent_file)
+        load_time = time.time() - load_start
+
+        total_time = time.time() - start_time
+        print(
+            f"    FileManager.get_most_recent_cache: Parquet load took {load_time:.2f}s (total: {total_time:.2f}s)"
         )
-        most_recent_file = cached_files[0]
-        file_path = self.get_file_path(most_recent_file, LoadType.SOURCE_CACHE)
 
-        return gpd.read_parquet(file_path)
+        return gdf
 
     def load_gdf(
         self, file_name: str, load_type: LoadType, file_type: FileType | None = None
@@ -194,16 +234,38 @@ def save_gdf(
             file_type (FileType): The type of the file (GEOJSON or PARQUET).
             load_type (LoadType): The destination type of the file (TEMP or CACHE).
         """
+        start_time = time.time()
+        print(f"    FileManager.save_gdf: Starting save for {file_name}")
+
         file_path = self.get_file_path(file_name, load_type, file_type)
+        print(f"    FileManager.save_gdf: Target path: {file_path}")
+
         if file_type == FileType.PARQUET:
+            print(
+                f"    FileManager.save_gdf: Writing parquet file ({len(gdf)} rows, {len(gdf.columns)} columns)"
+            )
+            parquet_start = time.time()
             gdf.to_parquet(file_path, index=False)
+            parquet_time = time.time() - parquet_start
+            print(f"    FileManager.save_gdf: Parquet write took {parquet_time:.2f}s")
         elif file_type == FileType.GEOJSON:
+            print("    FileManager.save_gdf: Writing GeoJSON file")
+            geojson_start = time.time()
             gdf.to_file(file_path, driver="GeoJSON")
+            geojson_time = time.time() - geojson_start
+            print(f"    FileManager.save_gdf: GeoJSON write took {geojson_time:.2f}s")
         elif file_type == FileType.CSV:
+            print("    FileManager.save_gdf: Writing CSV file")
+            csv_start = time.time()
             gdf.to_csv(file_path)
+            csv_time = time.time() - csv_start
+            print(f"    FileManager.save_gdf: CSV write took {csv_time:.2f}s")
         else:
             raise ValueError(f"Unsupported file type: {file_type}")
 
+        total_time = time.time() - start_time
+        print(f"    FileManager.save_gdf: Total save operation took {total_time:.2f}s")
+
     def save_fractional_gdf(
         self,
         gdf: gpd.GeoDataFrame,