Weekly PR from Staging to Main (#1186)

* feat: Unvendor awkde

* chore: Remove unused dev dependencies and pin versions

* chore: Alphabatize backend dependencies

* chore: Pin backend dependencies to minor versions

* Basic technologies documentation (#1184)

Add in additional documentation about different technologies used in the project for reference for newcomers. Can continue to flush it out as our pipeline gets more settled.

---------

Co-authored-by: Graham Beckley <[email protected]>
Co-authored-by: Nissim Lebovits <[email protected]>
Co-authored-by: Collum Freedman <[email protected]>
Co-authored-by: Adam Levin <[email protected]>

Author: 5 people

.github/workflows/pr_checks_backend.yml

@@ -68,10 +68,6 @@ jobs:
           echo "Using Python: $(which python)"
           pipenv --python $(which python) install --dev
 
-      - name: Install awkde
-        working-directory: data/src/awkde
-        run: pipenv run pip install .
-
       - name: Run Pytest
         working-directory: data/src
         run: PYTHONPATH=$PYTHONPATH:. pipenv run pytest

.gitignore

@@ -60,6 +60,7 @@ data/src/app/service-account-key.json
 # compiled python files
 *.pyc
 
-# awkde build files
-data/src/awkde/build/
-tmp/
+tmp/
+
+# Local python development files
+.python-version

.pre-commit-config.yaml

@@ -9,7 +9,6 @@ repos:
         language: python
         types_or: [python, pyi, jupyter]
         files: \.py$
-        exclude: ^data/src/awkde/
 
       # Ruff for Python formatting
       - id: ruff-format
@@ -18,7 +17,6 @@ repos:
         language: python
         types_or: [python, pyi, jupyter]
         files: \.py$
-        exclude: ^data/src/awkde/
 
       # Dependency checks for Python
       - id: check-python-dependencies
@@ -35,14 +33,13 @@ repos:
           radon hal {} -s | awk "/effort/ { if (\$3 > 2000) print FILENAME \": High Halstead effort:\" \$3 }"'
         language: system
         files: \.py$
-        exclude: ^data/src/awkde/
 
   - repo: https://github.com/jendrikseipp/vulture
     rev: 'v2.3' # or any later Vulture version
     hooks:
       - id: vulture
         name: vulture (Dead code detection)
-        entry: vulture data/ --exclude=data/src/awkde
+        entry: vulture data/
         language: python
         types: [python]
         files: ^data/

data/Dockerfile

@@ -29,12 +29,12 @@ ENV GDAL_CONFIG=/usr/bin/gdal-config
 # Install Pipenv
 RUN pip install pipenv
 
-# update pipfile
-RUN pipenv lock
-
 # Copy the Pipfile and Pipfile.lock from the src directory
 COPY src/Pipfile src/Pipfile.lock ./
 
+# update pipfile
+RUN pipenv lock
+
 # Install the dependencies from Pipfile
 RUN pipenv install --deploy --ignore-pipfile
 
@@ -44,17 +44,9 @@ RUN git clone https://github.com/felt/tippecanoe.git \
     && make \
     && make install
 
-# Copy the src directory (including awkde)
+# Copy the src directory
 COPY src/ .
 
-# Install awkde within the Pipenv environment
-# Assuming awkde is located in /usr/src/app/awkde
-WORKDIR /usr/src/app/awkde
-RUN pipenv run pip install .
-
-# Change back to the original working directory
-WORKDIR /usr/src/app
-
 # Use Pipenv to run the script
 # Adjust the path to your main Python script if needed
 CMD ["pipenv", "run", "python", "./script.py"]

data/docker-compose.yml

@@ -26,15 +26,15 @@ services:
       context: .
     volumes:
       - ./src:/usr/src/app
-    command: sh -c "pip install ruff && ruff format --exclude '/usr/src/app/awkde/'"
+    command: sh -c "pip install ruff && ruff format"
     network_mode: 'host'
 
   linter:
     build:
       context: .
     volumes:
       - ./src:/usr/src/app
-    command: sh -c "pip install ruff && ruff check --fix --exclude '/usr/src/app/awkde/'"
+    command: sh -c "pip install ruff && ruff check --fix"
     network_mode: 'host'
 
   streetview:

data/pyproject.toml

@@ -1,25 +1,14 @@
-[tool.pylint]
-ignore = ["awkde/"]
-output-format = "colorized,parseable"
-
-[tool.pylint.'MESSAGES CONTROL']
-disable = ["all"]
-enable = ["duplicate-code"]
-
 [tool.vulture]
-exclude = ["awkde/"]
 make_whitelist = true
 # min_confidence = 80
 sort_by_size = true
 
 [tool.mypy]
 ignore_missing_imports = true
-exclude = 'awkde/'
 
 [tool.ruff]
 # Exclude a variety of commonly ignored directories, plus our own
 exclude = [
-    "awkde/",
     ".bzr",
     ".direnv",
     ".eggs",

data/src/Pipfile

@@ -4,38 +4,38 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-jupyter = "*"
-requests = "*"
-pandas = "==2.2.2"
+awkde = { git = 'https://github.com/mennthor/awkde.git', ref = '5b601fe4d92229d5deb8737fd121dce193bac552' }
+esridump = "~=1.13.0"
+fiona = "~=1.10.1"
+future = "~=1.0.0"
+geoalchemy2 = "~=0.16.0"
 geopandas = "==1.0.1"
-shapely = "*"
-matplotlib = "*"
-rasterio = "*"
-scikit-learn = "*"
-mapclassify = "*"
-fiona = "*"
-esridump = "*"
-sqlalchemy = "*"
-psycopg2-binary = "*"
-mapbox = "*"
-google-cloud-storage = "*"
+google-cloud-storage = "~=2.19.0"
+jenkspy = "~=0.4.1"
+jupyter = "~=1.1.1"
+libpysal = "~=4.12.1"
+mapbox = "~=0.18.1"
+mapclassify = "~=2.8.1"
+matplotlib = "~=3.9.3"
+networkx = "~=3.4.2"
+pandas = "==2.2.2"
+psycopg2-binary = "~=2.9.10"
+pyarrow = "~=18.1.0"
 pydantic = "==2.8.2"
-future = "*"
-slack-sdk = "*"
-networkx = "*"
-libpysal = "*"
-jenkspy = "*"
-pyarrow = "*"
-tqdm = "*"
-geoalchemy2 ="*"
+rasterio = "~=1.4.3"
+requests = "~=2.32.3"
+scikit-learn = "~=1.6.0"
+shapely = "~=2.0.6"
+slack-sdk = "~=3.33.5"
+sqlalchemy = "~=2.0.36"
+tqdm = "~=4.67.1"
+
 
 [dev-packages]
-black = "*"
-pytest = "*"
-vulture = "*"
-pylint = "*"
-radon = "*"
-ruff = "*"
+pytest = "~=8.3.4"
+vulture = "~=2.14"
+radon = "~=6.0.1"
+ruff = "~=0.8.2"
 
 [requires]
 python_version = "3.11"