Modify Python Structure so that we treat src as a package

.github/workflows/pr_checks_backend.yml

@@ -70,7 +70,7 @@ jobs:
 
       - name: Run Pytest
         working-directory: data/src
-        run: PYTHONPATH=$PYTHONPATH:. pipenv run pytest
+        run: pipenv run pytest
 
   run-formatter:
     runs-on: ubuntu-latest

data/Dockerfile

@@ -30,7 +30,7 @@ ENV GDAL_CONFIG=/usr/bin/gdal-config
 RUN pip install pipenv
 
 # Copy the Pipfile and Pipfile.lock from the src directory
-COPY src/Pipfile src/Pipfile.lock ./
+COPY Pipfile Pipfile.lock ./
 
 # update pipfile
 RUN pipenv lock

data/src/classes/backup_archive_database.py

@@ -6,15 +6,15 @@
 import sqlalchemy as sa
 from sqlalchemy import inspect
 
-from classes.featurelayer import google_cloud_bucket
-from config.config import (
+from src.classes.featurelayer import google_cloud_bucket
+from src.config.config import (
     log_level,
     max_backup_schema_days,
     tile_file_backup_directory,
     tiles_file_id_prefix,
 )
-from config.psql import conn, local_engine, url
-from data_utils.utils import mask_password
+from src.config.psql import conn, local_engine, url
+from src.data_utils.utils import mask_password
 
 log.basicConfig(level=log_level)
 

data/src/classes/bucket_manager.py

@@ -3,7 +3,7 @@
 
 from google.cloud import storage
 
-from config.config import log_level
+from src.config.config import log_level
 
 log.basicConfig(level=log_level)
 

data/src/classes/diff_report.py

@@ -7,17 +7,17 @@
 
 from slack_sdk import WebClient
 
-from classes.backup_archive_database import backup_schema_name
-from classes.featurelayer import google_cloud_bucket
-from config.config import (
+from src.classes.backup_archive_database import backup_schema_name
+from src.classes.featurelayer import google_cloud_bucket
+from src.config.config import (
     from_email,
     log_level,
     report_to_email,
     report_to_slack_channel,
     smtp_server,
 )
-from config.psql import conn, url
-from data_utils.utils import mask_password
+from src.config.psql import conn, url
+from src.data_utils.utils import mask_password
 
 log.basicConfig(level=log_level)
 

data/src/classes/featurelayer.py

@@ -11,15 +11,15 @@
 from google.cloud import storage
 from shapely import Point, wkb
 
-from classes.bucket_manager import GCSBucketManager
-from config.config import (
+from src.classes.bucket_manager import GCSBucketManager
+from src.config.config import (
     FORCE_RELOAD,
     USE_CRS,
     log_level,
     min_tiles_file_size_in_bytes,
     write_production_tiles_file,
 )
-from config.psql import conn, local_engine
+from src.config.psql import conn, local_engine
 
 log.basicConfig(level=log_level)
 
@@ -283,7 +283,7 @@ def build_and_publish(self, tiles_file_id_prefix: str) -> None:
         Builds PMTiles and a Parquet file from a GeoDataFrame and publishes them to Google Cloud Storage.
 
         Args:
-            tiles_file_id_prefix (str): The ID prefix used for naming the PMTiles and Parquet files, coming from config.
+            tiles_file_id_prefix (str): The ID prefix used for naming the PMTiles and Parquet files, coming from src.config.
 
         Raises:
             ValueError: Raised if the generated PMTiles file is smaller than the minimum allowed size, indicating a potential corruption or incomplete file.

data/src/config/psql.py

@@ -2,7 +2,7 @@
 
 from sqlalchemy import create_engine
 
-from config.config import is_docker
+from src.config.config import is_docker
 
 
 def get_db_url():

data/src/data_utils/city_owned_properties.py

@@ -1,5 +1,5 @@
-from classes.featurelayer import FeatureLayer
-from constants.services import CITY_OWNED_PROPERTIES_TO_LOAD
+from src.classes.featurelayer import FeatureLayer
+from src.constants.services import CITY_OWNED_PROPERTIES_TO_LOAD
 
 
 def city_owned_properties(primary_featurelayer: FeatureLayer) -> FeatureLayer:

data/src/data_utils/community_gardens.py

@@ -1,5 +1,5 @@
-from classes.featurelayer import FeatureLayer
-from constants.services import COMMUNITY_GARDENS_TO_LOAD
+from src.classes.featurelayer import FeatureLayer
+from src.constants.services import COMMUNITY_GARDENS_TO_LOAD
 
 
 def community_gardens(primary_featurelayer):

data/src/data_utils/deliquencies.py

@@ -1,5 +1,5 @@
-from classes.featurelayer import FeatureLayer
-from constants.services import DELINQUENCIES_QUERY
+from src.classes.featurelayer import FeatureLayer
+from src.constants.services import DELINQUENCIES_QUERY
 
 
 def deliquencies(primary_featurelayer):

data/src/data_utils/dev_probability.py

@@ -2,10 +2,10 @@
 import jenkspy
 import pandas as pd
 import requests
-from classes.featurelayer import FeatureLayer
-from constants.services import CENSUS_BGS_URL, PERMITS_QUERY
 
-from config.config import USE_CRS
+from src.classes.featurelayer import FeatureLayer
+from src.config.config import USE_CRS
+from src.constants.services import CENSUS_BGS_URL, PERMITS_QUERY
 
 
 def dev_probability(primary_featurelayer):

data/src/data_utils/drug_crimes.py

@@ -1,7 +1,5 @@
-from constants.services import DRUGCRIME_SQL_QUERY
-
-
-from data_utils.kde import apply_kde_to_primary
+from src.constants.services import DRUGCRIME_SQL_QUERY
+from src.data_utils.kde import apply_kde_to_primary
 
 
 def drug_crimes(primary_featurelayer):

data/src/data_utils/gun_crimes.py

@@ -1,7 +1,5 @@
-from constants.services import GUNCRIME_SQL_QUERY
-
-
-from data_utils.kde import apply_kde_to_primary
+from src.constants.services import GUNCRIME_SQL_QUERY
+from src.data_utils.kde import apply_kde_to_primary
 
 
 def gun_crimes(primary_featurelayer):

data/src/data_utils/imm_dang_buildings.py

@@ -1,5 +1,5 @@
-from classes.featurelayer import FeatureLayer
-from constants.services import IMMINENT_DANGER_BUILDINGS_QUERY
+from src.classes.featurelayer import FeatureLayer
+from src.constants.services import IMMINENT_DANGER_BUILDINGS_QUERY
 
 
 def imm_dang_buildings(primary_featurelayer):

data/src/data_utils/kde.py

@@ -1,13 +1,14 @@
+from concurrent.futures import ProcessPoolExecutor, as_completed
+
+import mapclassify
 import numpy as np
 import rasterio
 from awkde.awkde import GaussianKDE
-from classes.featurelayer import FeatureLayer
-from config.config import USE_CRS
 from rasterio.transform import Affine
 from tqdm import tqdm
-from concurrent.futures import ProcessPoolExecutor, as_completed
 
-import mapclassify
+from src.classes.featurelayer import FeatureLayer
+from src.config.config import USE_CRS
 
 resolution = 1320  # 0.25 miles (in feet, bc the CRS is 2272)
 batch_size = 100000

data/src/data_utils/l_and_i.py

@@ -1,8 +1,10 @@
-import pandas as pd
-import geopandas as gpd
 from typing import List
-from classes.featurelayer import FeatureLayer
-from constants.services import COMPLAINTS_SQL_QUERY, VIOLATIONS_SQL_QUERY
+
+import geopandas as gpd
+import pandas as pd
+
+from src.classes.featurelayer import FeatureLayer
+from src.constants.services import COMPLAINTS_SQL_QUERY, VIOLATIONS_SQL_QUERY
 
 
 def l_and_i(primary_featurelayer: FeatureLayer) -> FeatureLayer:

data/src/data_utils/nbhoods.py

@@ -1,8 +1,8 @@
 import geopandas as gpd
-from classes.featurelayer import FeatureLayer
-from constants.services import NBHOODS_URL
 
-from config.config import USE_CRS
+from src.classes.featurelayer import FeatureLayer
+from src.config.config import USE_CRS
+from src.constants.services import NBHOODS_URL
 
 
 def nbhoods(primary_featurelayer):

data/src/data_utils/opa_properties.py

@@ -1,5 +1,5 @@
-from classes.featurelayer import FeatureLayer
-from constants.services import OPA_PROPERTIES_QUERY
+from src.classes.featurelayer import FeatureLayer
+from src.constants.services import OPA_PROPERTIES_QUERY
 
 
 def opa_properties(primary_featurelayer):

data/src/data_utils/owner_type.py

@@ -1,5 +1,6 @@
 import pandas as pd
-from classes.featurelayer import FeatureLayer
+
+from src.classes.featurelayer import FeatureLayer
 
 
 def owner_type(primary_featurelayer: FeatureLayer) -> FeatureLayer:

data/src/data_utils/park_priority.py

@@ -4,12 +4,13 @@
 from typing import List, Union
 
 import geopandas as gpd
+import pyogrio
 import requests
 from bs4 import BeautifulSoup
-from classes.featurelayer import FeatureLayer
-from config.config import USE_CRS
 from tqdm import tqdm
-import pyogrio
+
+from src.classes.featurelayer import FeatureLayer
+from src.config.config import USE_CRS
 
 
 def get_latest_shapefile_url() -> str:

data/src/data_utils/phs_properties.py

@@ -1,5 +1,5 @@
-from classes.featurelayer import FeatureLayer
-from constants.services import PHS_LAYERS_TO_LOAD
+from src.classes.featurelayer import FeatureLayer
+from src.constants.services import PHS_LAYERS_TO_LOAD
 
 
 def phs_properties(primary_featurelayer: FeatureLayer) -> FeatureLayer:

data/src/data_utils/ppr_properties.py

@@ -2,10 +2,10 @@
 
 import geopandas as gpd
 import requests
-from classes.featurelayer import FeatureLayer
-from constants.services import PPR_PROPERTIES_TO_LOAD
 
-from config.config import USE_CRS
+from src.classes.featurelayer import FeatureLayer
+from src.config.config import USE_CRS
+from src.constants.services import PPR_PROPERTIES_TO_LOAD
 
 
 def ppr_properties(primary_featurelayer):

data/src/data_utils/rco_geoms.py

@@ -1,7 +1,8 @@
-from classes.featurelayer import FeatureLayer
-from constants.services import RCOS_LAYERS_TO_LOAD
 import pandas as pd
 
+from src.classes.featurelayer import FeatureLayer
+from src.constants.services import RCOS_LAYERS_TO_LOAD
+
 pd.set_option("future.no_silent_downcasting", True)
 
 

data/src/data_utils/tree_canopy.py

@@ -1,9 +1,11 @@
-import requests
 import io
 import zipfile
+
 import geopandas as gpd
-from classes.featurelayer import FeatureLayer
-from config.config import USE_CRS
+import requests
+
+from src.classes.featurelayer import FeatureLayer
+from src.config.config import USE_CRS
 
 
 def tree_canopy(primary_featurelayer):

data/src/data_utils/unsafe_buildings.py

@@ -1,5 +1,5 @@
-from classes.featurelayer import FeatureLayer
-from constants.services import UNSAFE_BUILDINGS_QUERY
+from src.classes.featurelayer import FeatureLayer
+from src.constants.services import UNSAFE_BUILDINGS_QUERY
 
 
 def unsafe_buildings(primary_featurelayer):

data/src/data_utils/vacant_properties.py

@@ -1,11 +1,12 @@
-from classes.featurelayer import FeatureLayer, google_cloud_bucket
-from constants.services import VACANT_PROPS_LAYERS_TO_LOAD
-import geopandas as gpd
-from config.config import USE_CRS
 from io import BytesIO
 
+import geopandas as gpd
 import pandas as pd
 
+from src.classes.featurelayer import FeatureLayer, google_cloud_bucket
+from src.config.config import USE_CRS
+from src.constants.services import VACANT_PROPS_LAYERS_TO_LOAD
+
 
 def load_backup_data_from_gcs(file_name: str) -> gpd.GeoDataFrame:
     bucket = google_cloud_bucket()

data/src/main.py

@@ -2,15 +2,15 @@
 
 import pandas as pd
 
-from config.config import tiles_file_id_prefix
-from config.psql import conn
-from new_etl.classes.data_diff import DiffReport
-from new_etl.classes.slack_reporters import (
+from src.config.config import tiles_file_id_prefix
+from src.config.psql import conn
+from src.new_etl.classes.data_diff import DiffReport
+from src.new_etl.classes.slack_reporters import (
     send_dataframe_profile_to_slack,
     send_error_to_slack,
     send_pg_stats_to_slack,
 )
-from new_etl.data_utils import (
+from src.new_etl.data_utils import (
     access_process,
     city_owned_properties,
     community_gardens,
@@ -39,22 +39,22 @@
     unsafe_buildings,
     vacant_properties,
 )
-from new_etl.database import to_postgis_with_schema
-from new_etl.validation import (
+from src.new_etl.database import to_postgis_with_schema
+from src.new_etl.validation import (
     CommunityGardensValidator,
     KDEValidator,
     LIViolationsValidator,
     OwnerTypeValidator,
     TreeCanopyValidator,
     VacantValidator,
 )
-from new_etl.validation.access_process import AccessProcessValidator
-from new_etl.validation.city_owned_properties import CityOwnedPropertiesValidator
-from new_etl.validation.council_dists import CouncilDistrictsValidator
-from new_etl.validation.nbhoods import NeighborhoodsValidator
-from new_etl.validation.phs_properties import PHSPropertiesValidator
-from new_etl.validation.ppr_properties import PPRPropertiesValidator
-from new_etl.validation.rco_geoms import RCOGeomsValidator
+from src.new_etl.validation.access_process import AccessProcessValidator
+from src.new_etl.validation.city_owned_properties import CityOwnedPropertiesValidator
+from src.new_etl.validation.council_dists import CouncilDistrictsValidator
+from src.new_etl.validation.nbhoods import NeighborhoodsValidator
+from src.new_etl.validation.phs_properties import PHSPropertiesValidator
+from src.new_etl.validation.ppr_properties import PPRPropertiesValidator
+from src.new_etl.validation.rco_geoms import RCOGeomsValidator
 
 # Map services to their validators
 SERVICE_VALIDATORS = {

data/src/new_etl/classes/bucket_manager.py

@@ -3,7 +3,7 @@
 
 from google.cloud import storage
 
-from config.config import log_level
+from src.config.config import log_level
 
 log.basicConfig(level=log_level)