feat: update all dependencies and project configs

Changes:

  1. Update github actions base image to 22.04 since github is retiring 20.04.
  2. Use pip-tools to compile all Python requirements.
  3. Update all Python dependeny versions
  4. Regenerated all pb2 files.

Author: kunlin596

.github/workflows/build-docker.yml

@@ -3,17 +3,17 @@ name: build-docker
 on:
   push:
     branches:
-      - 'master'
+      - "master"
     tags:
-      - '*'
+      - "*"
 
 env:
   REGISTRY: ghcr.io
   IMAGE_NAME: ${{ github.repository }}
 
 jobs:
   build-and-push-image:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     permissions:
       contents: read
       packages: write

.github/workflows/coverage.yml

@@ -2,24 +2,24 @@ name: coverage
 on:
   pull_request:
     branches:
-      - 'master'
+      - "master"
   push:
     branches:
-      - 'master'
+      - "master"
 jobs:
   code-coverage:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     env:
       ENVIRONMENT: TEST_RUNNER
-      OS: ubuntu-20.04
-      PYTHON: '3.9'
+      OS: ubuntu-22.04
+      PYTHON: "3.9"
       COVERAGE_TOTAL: 49 # Coverage threshold percentage
     steps:
       - name: Checkout (admin token)
         if: ${{github.event_name != 'pull_request'}} # We don't want to use the admin token for PR flows
         uses: actions/checkout@master
         with:
-          token: '${{ secrets.GIT_ADMIN_WORKFLOW_TOKEN }}'
+          token: "${{ secrets.GIT_ADMIN_WORKFLOW_TOKEN }}"
           fetch-depth: "2" # Original commit + code cov badge commit
       - name: Checkout (normal flow)
         if: ${{github.event_name == 'pull_request'}}
@@ -34,10 +34,8 @@ jobs:
         id: coverage-installer
         run: |
           python -m pip install --upgrade pip
-          pip install cython==0.29.21 numpy==1.23.2
           sudo apt-get install jq
-          pip install -r requirements.txt
-          pip install -r requirements-dev.txt
+          pip install -r requirements-dev.lock
           pip install coverage-badge
       - name: Run tests and calculate coverage
         id: test-runner

.github/workflows/doc-gen.yml

@@ -2,17 +2,17 @@ name: doc-gen
 on:
   push:
     branches:
-      - 'master'
+      - "master"
   pull_request:
     branches:
-      - 'master'
+      - "master"
 
 jobs:
   run:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     env:
-      OS: ubuntu-20.04
-      PYTHON: '3.9'
+      OS: ubuntu-22.04
+      PYTHON: "3.9"
     steps:
       - uses: actions/checkout@master
         with:
@@ -26,9 +26,8 @@ jobs:
       - name: Setup requirements and run sphinx
         run: |
           python -m pip install --upgrade pip
-          pip install cython==0.29.21 numpy==1.23.2
-          pip install -r requirements.txt
-          pip install -r requirements-dev.txt
+          pip install -r requirements.lock
+          pip install -r requirements-dev.lock
           pip install -r docs/requirements-doc.txt
           cd docs
           make html
@@ -39,4 +38,4 @@ jobs:
         with:
           branch: gh-pages
           folder: ./docs/build/html
-          commit-message: 'docs: update build documentation'
+          commit-message: "docs: update build documentation"

.github/workflows/pre-merge.yml

@@ -9,7 +9,7 @@ on:
 
 jobs:
   pre-merge-tests:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     env:
       ENVIRONMENT: TEST_RUNNER
     steps:
@@ -28,9 +28,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install cython==0.29.21 numpy==1.23.2
-          pip install -r requirements.txt
-          pip install -r requirements-dev.txt
+          pip install -r requirements-dev.lock
       - name: Lint all files with pre-commit
         run: |
           pre-commit install

Dockerfile

@@ -45,10 +45,8 @@ RUN pip install --no-cache-dir \
 # Install python dependencies
 ARG WORKSPACE=/home/dgp
 WORKDIR ${WORKSPACE}
-COPY requirements.txt requirements-dev.txt /tmp/
-RUN pip install --no-cache-dir cython==0.29.30 numpy==1.20.3
-RUN pip install --no-cache-dir -r /tmp/requirements.txt
-RUN pip install --no-cache-dir -r /tmp/requirements-dev.txt
+COPY requirements-dev.lock /tmp/
+RUN pip install --no-cache-dir -r /tmp/requirements-dev.lock
 
 # Settings for S3
 RUN aws configure set default.s3.max_concurrent_requests 100 && \

dgp/__init__.py

@@ -2,51 +2,51 @@
 import os
 from collections import OrderedDict
 
-__version__ = '1.0'
+__version__ = "1.6.0"
 
-DGP_PATH = os.getenv('DGP_PATH', default=os.getenv('HOME', os.getcwd()))
-DGP_DATA_DIR = os.path.join(DGP_PATH, '.dgp')
-DGP_CACHE_DIR = os.path.join(DGP_DATA_DIR, 'cache')
-DGP_DATASETS_CACHE_DIR = os.path.join(DGP_DATA_DIR, 'datasets')
+DGP_PATH = os.getenv("DGP_PATH", default=os.getenv("HOME", os.getcwd()))
+DGP_DATA_DIR = os.path.join(DGP_PATH, ".dgp")
+DGP_CACHE_DIR = os.path.join(DGP_DATA_DIR, "cache")
+DGP_DATASETS_CACHE_DIR = os.path.join(DGP_DATA_DIR, "datasets")
 
 TRI_DGP_FOLDER_PREFIX = "dgp/"
 TRI_RAW_FOLDER_PREFIX = "raw/"
 TRI_DGP_JSON_PREFIX = "dataset_v"
 
 # DGP Directory structure constants
-RGB_FOLDER = 'rgb'
-POINT_CLOUD_FOLDER = 'point_cloud'
+RGB_FOLDER = "rgb"
+POINT_CLOUD_FOLDER = "point_cloud"
 RADAR_POINT_CLOUD_FOLDER = "radar_point_cloud"
-BOUNDING_BOX_2D_FOLDER = 'bounding_box_2d'
-BOUNDING_BOX_3D_FOLDER = 'bounding_box_3d'
-SEMANTIC_SEGMENTATION_2D_FOLDER = 'semantic_segmentation_2d'
-SEMANTIC_SEGMENTATION_3D_FOLDER = 'semantic_segmentation_3d'
-INSTANCE_SEGMENTATION_2D_FOLDER = 'instance_segmentation_2d'
-INSTANCE_SEGMENTATION_3D_FOLDER = 'instance_segmentation_3d'
-DEPTH_FOLDER = 'depth'
+BOUNDING_BOX_2D_FOLDER = "bounding_box_2d"
+BOUNDING_BOX_3D_FOLDER = "bounding_box_3d"
+SEMANTIC_SEGMENTATION_2D_FOLDER = "semantic_segmentation_2d"
+SEMANTIC_SEGMENTATION_3D_FOLDER = "semantic_segmentation_3d"
+INSTANCE_SEGMENTATION_2D_FOLDER = "instance_segmentation_2d"
+INSTANCE_SEGMENTATION_3D_FOLDER = "instance_segmentation_3d"
+DEPTH_FOLDER = "depth"
 EXTRA_DATA_FOLDER = "extra_data"
 FEATURE_ONTOLOGY_FOLDER = "feature_ontology"
 AGENT_FOLDER = "agent"
 CLASSIFICATION_FOLDER = "classification"
 
 # Scene Directory structure constants
-CALIBRATION_FOLDER = 'calibration'
-ONTOLOGY_FOLDER = 'ontology'
-SCENE_JSON_FILENAME = 'scene.json'
+CALIBRATION_FOLDER = "calibration"
+ONTOLOGY_FOLDER = "ontology"
+SCENE_JSON_FILENAME = "scene.json"
 
 # DGP file naming conventions
 TRI_DGP_SCENE_DATASET_JSON_NAME = "scene_dataset_v{version}.json"
 TRI_DGP_SCENE_JSON_NAME = "scene_{scene_hash}.json"
-ANNOTATION_FILE_NAME = '{image_content_hash}_{annotation_content_hash}.json'
+ANNOTATION_FILE_NAME = "{image_content_hash}_{annotation_content_hash}.json"
 
 # DGP file naming conventions
 TRI_DGP_SCENE_DATASET_JSON_NAME = "scene_dataset_v{version}.json"
 TRI_DGP_AGENT_TRACKS_JSON_NAME = "agent_tracks_{track_hash}.json"
 TRI_DGP_SCENE_JSON_NAME = "scene_{scene_hash}.json"
-ANNOTATION_FILE_NAME = '{image_content_hash}_{annotation_content_hash}.json'
+ANNOTATION_FILE_NAME = "{image_content_hash}_{annotation_content_hash}.json"
 TRI_DGP_AGENTS_JSON_NAME = "agents_{agent_hash}.json"
 TRI_DGP_AGENTS_SLICES_JSON_NAME = "agents_slices_{slice_hash}.json"
 
 # Autolabel constants
-AUTOLABEL_FOLDER = 'autolabels'
-AUTOLABEL_SCENE_JSON_NAME = 'scene.json'
+AUTOLABEL_FOLDER = "autolabels"
+AUTOLABEL_SCENE_JSON_NAME = "scene.json"

dgp/contribs/dgp2wicker/requirements.in

@@ -0,0 +1,5 @@
+grpcio>=1.51.1
+grpcio-tools>=1.51.1
+protobuf>=3.20.3,<5.0.0
+wicker[spark]
+retry

dgp/contribs/dgp2wicker/requirements.lock

@@ -0,0 +1,50 @@
+# This file was autogenerated by uv via the following command:
+#    uv pip compile requirements.in -o requirements.lock --resolver=backtracking
+boto3==1.37.36
+    # via wicker
+botocore==1.37.36
+    # via
+    #   boto3
+    #   s3transfer
+decorator==5.2.1
+    # via retry
+grpcio==1.70.0
+    # via
+    #   -r requirements.in
+    #   grpcio-tools
+grpcio-tools==1.62.3
+    # via -r requirements.in
+jmespath==1.0.1
+    # via
+    #   boto3
+    #   botocore
+numpy==1.24.4
+    # via
+    #   pyarrow
+    #   wicker
+protobuf==4.25.6
+    # via
+    #   -r requirements.in
+    #   grpcio-tools
+py==1.11.0
+    # via retry
+py4j==0.10.9.7
+    # via pyspark
+pyarrow==17.0.0
+    # via wicker
+pyspark==3.5.5
+    # via wicker
+python-dateutil==2.9.0.post0
+    # via botocore
+retry==0.9.2
+    # via -r requirements.in
+s3transfer==0.11.5
+    # via boto3
+setuptools==75.3.2
+    # via grpcio-tools
+six==1.17.0
+    # via python-dateutil
+urllib3==1.26.20
+    # via botocore
+wicker==0.0.16
+    # via -r requirements.in

dgp/contribs/dgp2wicker/requirements.txt

@@ -19,35 +19,34 @@ def run(self):
         develop.run(self)
 
 
-__version__ = importlib.import_module('dgp2wicker').__version__
+__version__ = importlib.import_module("dgp2wicker").__version__
 
-with open('requirements.txt', 'r', encoding='utf-8') as f:
+with open("requirements.lock", "r", encoding="utf-8") as f:
     requirements = f.read().splitlines()
 
-with open('README.md', 'r', encoding='utf-8') as f:
+with open("README.md", "r", encoding="utf-8") as f:
     readme = f.read()
 
-packages = find_packages(exclude=['tests'])
+packages = find_packages(exclude=["tests"])
 setup(
     name="dgp2wicker",
     version=__version__,
     description="Tools to convert TRI's DGP to L5's Wicker format.",
     long_description=readme,
-    long_description_content_type='text/markdown',
+    long_description_content_type="text/markdown",
     author="Chris Ochoa, Kuan Lee",
-    author_email='charles.ochoa@woven-planet.global, kuan-hui.lee@woven-planet.global',
+    author_email="charles.ochoa@woven-planet.global, kuan-hui.lee@woven-planet.global",
     url="https://github.com/TRI-ML/dgp/tree/master/dgp/contribs/dgp2wicker",
     packages=packages,
-    entry_points={'console_scripts': [
-        'dgp2wicker=dgp2wicker.cli:cli',
+    entry_points={"console_scripts": [
+        "dgp2wicker=dgp2wicker.cli:cli",
     ]},
     include_package_data=True,
-    setup_requires=['cython==0.29.21', 'grpcio==1.41.0', 'grpcio-tools==1.41.0'],
     install_requires=requirements,
     zip_safe=False,
-    python_requires='>=3.7',
+    python_requires=">=3.8",
     cmdclass={
-        'install': CustomInstallCommand,
-        'develop': CustomDevelopCommand,
-    }
+        "install": CustomInstallCommand,
+        "develop": CustomDevelopCommand,
+    },
 )