chore: Functions python refactoring (#850)

Refactored functions-python build to get packages from api and more

Author: jcpitre

.github/workflows/api-deployer.yml

@@ -201,10 +201,6 @@ jobs:
           name: database_gen
           path: api/src/database_gen/
 
-      - name: Copy to db models to functions directory
-        run: |
-          cp -R api/src/database_gen/ functions-python/database_gen
-
       # api schema was generated and uploaded in api-build-test job above.
       - uses: actions/download-artifact@v4
         with:
@@ -249,10 +245,6 @@ jobs:
           name: database_gen
           path: api/src/database_gen/
 
-      - name: Copy to db models to functions directory
-        run: |
-          cp -R api/src/database_gen/ functions-python/database_gen
-
       # api schema was generated and uploaded in api-build-test job above.
       - uses: actions/download-artifact@v4
         with:

.github/workflows/datasets-batch-deployer.yml

@@ -119,7 +119,7 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: database_gen
-          path: functions-python/database_gen/
+          path: api/src/database_gen/
 
       - name: Build python functions
         run: |

functions-python/.gitignore

@@ -1 +1,3 @@
-.dist
+.dist
+shared
+test_shared

functions-python/README.md

@@ -22,7 +22,8 @@ The function configuration file contains the following properties:
 - `timeout`: The timeout of the function in seconds. The default value is 60 seconds.
 - `memory`: The memory of the function in MB. The default value is 128 MB.
 - `trigger_http`: A boolean value that indicates if the function is triggered by an HTTP request. The default value is `false`.
-- `include_folders`: A list of folders to be included in the function zip. By default, the function zip will include all the files in the function folder.
+- `include_folders`: A list of folders from functions-python to be included in the function zip. By default, the function zip will include all the files in the function folder.
+- `include_api_folders`: A list of folders from the api folder to be included in the function zip. By default, the function zip will include all the files in the function folder.
 - `secret_environment_variables`: A list of objects, each representing a secret environment variable. These are securely used within the function. Each object should include:
   - `key`: The name of the environment variable as used in the function, acting as the secret's identifier.
   - `secret` [Optional]: The specific GCP secret to be used. If omitted, a default secret name is generated using the environment prefix (`DEV`, `QA`, or `PROD`) followed by an underscore and the `key` value. For example, if `key` is `api_key` in the `DEV` environment, the default secret name is `DEV_api_key`.
@@ -33,6 +34,13 @@ The function configuration file contains the following properties:
 - `available_cpu_count`: The number of CPU cores that are available to the function.
 - `available_memory`: The amount of memory available to the function.
 
+# Test configuration(test_config.json)
+Some folders in functions-python are not destined to be deployed functions but are in support for other functions (e.g. helpers)m
+Some of these folders contain tests and the `test_config.json` file is used to configure the tests.
+The test configuration file contains the following properties:
+- `include_folders`: A list of folders from functions-python used in the tests.
+- `include_api_folders`: A list of folders from the api folder to be included in the tests.
+
 # Local Setup
 
 ## Requirements
@@ -53,19 +61,76 @@ gcloud components install pubsub-emulator
 ```
 
 # Useful scripts
-- To locally execute a function use the following command:
+
+The following sections uses `batch_datasets` as an example function. Replace `batch_datasets` with the function name you want to work with.
+
+## Setting up an environment for editing and testing a function
+- To setup an self-contained environment specific to a function, you can use the following command:
+```
+./scripts/function-python-setup.sh --function_name batch_datasets
+```
+This will create a `shared` folder in the function's folder (e.g. batch_datasets/src/shared) 
+with symbolic links to the necessary packages to run the function locally.
+It will also create a `test_shared` folder in the function's test folder (e.g. batch_datasets/tests/test_shared) 
+e.g.:
+   - functions-python/batch_datasets/src/shared
+      - database_gen -> symlink to api/database_gen
+      - dataset_service	-> symlink to functions-python/dataset_service
+      - helpers -> symlink to functions-python/helpers
+   - functions-python/batch_datasets/tests/test_shared
+     - test_utils -> symlink to functions-python/test_utils
+
+The python code should refer to these shared folders to import the necessary modules.
+e.g. in `batch_datasets/src/main.py` we use the import:
+```
+from shared.database_gen.sqlacodegen_models import Gtfsfeed, Gtfsdataset
+```
+and in `batch_datasets/tests/testbatch_datasets_main.py` we use the import:
+```
+from test_shared.test_utils.database_utils import get_testing_session, default_db_url
+```
+Notice the `shared` and `test_shared` prefixes in the import path.
+
+- Also you can do a setup for all functions by running:
+```
+./scripts/function-python-setup.sh --all
+```
+- And remove the shared and test_shared folders by running:
+```
+./scripts/function-python-setup.sh --function_name batch_datasets --clean
+```
+or
 ```
-./scripts/function-python-run.sh --function_name tokens
+./scripts/function-python-setup.sh --all --clean
 ```
+
+## Creating a distribution zip
+
 - To locally create a distribution zip use the following command:
 ```
-./scripts/function-python-build.sh --function_name tokens
+./scripts/function-python-build.sh --function_name batch_datasets
 ```
 or 
 ```
 ./scripts/function-python-build.sh --all
 ```
-- Start local and test database
+This script will create a .dist folder in the function's folder with the distribution zip and a build folder with the necessary files.
+e.g. 
+```
+functions-python/batch_datasets/.dist/batch_datasets.zip
+functions-python/batch_datasets/build/
+```
+
+## Executing a function
+- To locally execute a function use the following command:
+```
+./scripts/function-python-run.sh --function_name batch_datasets
+```
+This will create a virtual environment specific to the function (i.e with the specific requirements.txt installed) 
+and run the function locally using function-framework
+
+## Start local and test database
+
 ```
 docker compose --env-file ./config/.env.local up -d liquibase-test
 ```
@@ -82,11 +147,26 @@ Make sure the testing database is running before executing the tests.
 ```
 docker compose --env-file ./config/.env.local up -d liquibase-test
 ```
-Execute all tests within the functions-python folder
+
+To run the tests:
 ```
-./scripts/api-tests.sh --folder functions-python 
+scripts/api-tests.sh --folder functions-python/batch_datasets
 ```
-Execute test from a specific function
+or
 ```
-./scripts/api-tests.sh --folder functions-python/batch_datasets
-```
+scripts/api-tests.sh --folder functions-python
+```
+This will 
+- run the `function-python-setup.sh` script for the function (ie create the `shared` and `test_shared` folders with symlinks) 
+- Create a python virtual environment in the function folder, e.g.: `functions-python/batch_datasets/venv`
+- Install the requirements.txt and requirements_dev.txt specific to the function
+- Run the tests with coverage using installed virtual environment
+- If there is any requirements missing you should be able to catch it at this point.
+
+# Development using Pycharm
+
+- You can open the function directly in Pycharm, e.g. open `functions-python/batch_datasets` directly.
+- You need to set `batch_datasets/src` as the source root in Pycharm.
+- You need to set the python interpreter to the one in the virtual environment created by 
+the `function-python-build.sh` or `api-tests.sh` script, i.e. `functions-python/batch_datasets/venv`
+- This will provide an environment the same or similar to the one used in deployment and allow you to catch issues early.

functions-python/batch_datasets/function_config.json

@@ -5,7 +5,8 @@
   "timeout": 20,
   "memory": "256Mi",
   "trigger_http": true,
-  "include_folders": ["database_gen", "helpers", "dataset_service"],
+  "include_folders": ["helpers", "dataset_service"],
+  "include_api_folders": ["database_gen"],
   "secret_environment_variables": [
     {
       "key": "FEEDS_DATABASE_URL"

functions-python/batch_datasets/main_local_debug.py

@@ -12,7 +12,7 @@
 
 # Usage:
 # - python batch_datasets/main_local_debug.py
-from src.main import batch_datasets
+from main import batch_datasets
 from dotenv import load_dotenv
 
 # Load environment variables from .env.local

functions-python/batch_datasets/src/main.py

@@ -25,9 +25,9 @@
 from google.cloud.pubsub_v1.futures import Future
 from sqlalchemy import or_
 from sqlalchemy.orm import Session
-from database_gen.sqlacodegen_models import Gtfsfeed, Gtfsdataset
-from dataset_service.main import BatchExecutionService, BatchExecution
-from helpers.database import Database
+from shared.database_gen.sqlacodegen_models import Gtfsfeed, Gtfsdataset
+from shared.dataset_service.main import BatchExecutionService, BatchExecution
+from shared.helpers.database import Database
 
 pubsub_topic_name = os.getenv("PUBSUB_TOPIC_NAME")
 project_id = os.getenv("PROJECT_ID")

functions-python/batch_datasets/tests/conftest.py

@@ -17,8 +17,12 @@
 from faker import Faker
 from faker.generator import random
 from datetime import datetime
-from database_gen.sqlacodegen_models import Gtfsfeed, Gtfsrealtimefeed, Gtfsdataset
-from test_utils.database_utils import clean_testing_db, get_testing_session
+from shared.database_gen.sqlacodegen_models import (
+    Gtfsfeed,
+    Gtfsrealtimefeed,
+    Gtfsdataset,
+)
+from test_shared.test_utils.database_utils import clean_testing_db, get_testing_session
 
 
 def populate_database():

functions-python/batch_datasets/tests/test_batch_datasets_main.py

@@ -18,8 +18,8 @@
 from unittest import mock
 import pytest
 from unittest.mock import Mock, patch, MagicMock
-from batch_datasets.src.main import get_non_deprecated_feeds, batch_datasets
-from test_utils.database_utils import get_testing_session, default_db_url
+from main import get_non_deprecated_feeds, batch_datasets
+from test_shared.test_utils.database_utils import get_testing_session, default_db_url
 
 
 def test_get_non_deprecated_feeds():
@@ -39,17 +39,19 @@ def test_get_non_deprecated_feeds():
         "FEEDS_LIMIT": "5",
     },
 )
-@patch("batch_datasets.src.main.publish")
-@patch("batch_datasets.src.main.get_pubsub_client")
+@patch("main.publish")
+@patch("main.get_pubsub_client")
 def test_batch_datasets(mock_client, mock_publish):
     mock_client.return_value = MagicMock()
     with get_testing_session() as session:
         feeds = get_non_deprecated_feeds(session)
         with patch(
-            "dataset_service.main.BatchExecutionService.__init__", return_value=None
+            "shared.dataset_service.main.BatchExecutionService.__init__",
+            return_value=None,
         ):
             with patch(
-                "dataset_service.main.BatchExecutionService.save", return_value=None
+                "shared.dataset_service.main.BatchExecutionService.save",
+                return_value=None,
             ):
                 batch_datasets(Mock())
                 assert mock_publish.call_count == 5
@@ -64,7 +66,7 @@ def test_batch_datasets(mock_client, mock_publish):
                     ]
 
 
-@patch("batch_datasets.src.main.Database")
+@patch("main.Database")
 def test_batch_datasets_exception(database_mock):
     exception_message = "Failure occurred"
     mock_session = MagicMock()

functions-python/batch_process_dataset/function_config.json

@@ -5,7 +5,8 @@
   "timeout": 540,
   "memory": "2Gi",
   "trigger_http": true,
-  "include_folders": ["database_gen", "helpers", "dataset_service"],
+  "include_folders": ["helpers", "dataset_service"],
+  "include_api_folders": ["database_gen"],
   "secret_environment_variables": [
     {
       "key": "FEEDS_DATABASE_URL"