Hea 573/wealth group income expenditure kcals

.github/workflows/01-build-then-test.yml

@@ -271,9 +271,6 @@ jobs:
           docker cp ci-${APP}-${CI_PIPELINE_ID}-${{ github.job }}:/usr/src/app/log ./ || true
           docker cp ci-${APP}-${CI_PIPELINE_ID}-${{ github.job }}:/usr/src/app/coverage.txt ./ || true
 
-          # Save the database schema as an artifact
-          docker compose run --no-deps --rm --entrypoint dbtoyaml app --no-owner --no-privileges test_${PGDATABASE} > schema.yml
-          diff pyrseas/schema.yaml schema.yml > schema.diff || true
       - name: "Upload test artifacts"
         if: success() || failure()
         uses: actions/upload-artifact@v4
@@ -400,9 +397,6 @@ jobs:
           # Copy the artifacts out of the Docker container to project directory
           docker cp ci-${APP}-${CI_PIPELINE_ID}-${{ github.job }}:/usr/src/app/log ./ || true
           docker cp ci-${APP}-${CI_PIPELINE_ID}-${{ github.job }}:/usr/src/app/coverage.txt ./ || true
-          # Save the database schema as an artifact
-          docker compose run --no-deps --rm --entrypoint dbtoyaml app --no-owner --no-privileges test_${PGDATABASE} > schema.yml
-          diff pyrseas/schema.yaml schema.yml > schema.diff || true
       - name: "Upload test artifacts"
         if: success() || failure()
         uses: actions/upload-artifact@v4
@@ -530,9 +524,6 @@ jobs:
           # Copy the artifacts out of the Docker container to project directory
           docker cp ci-${APP}-${CI_PIPELINE_ID}-${{ github.job }}:/usr/src/app/log ./ || true
           docker cp ci-${APP}-${CI_PIPELINE_ID}-${{ github.job }}:/usr/src/app/coverage.txt ./ || true
-          # Save the database schema as an artifact
-          docker compose run --no-deps --rm --entrypoint dbtoyaml app --no-owner --no-privileges test_${PGDATABASE} > schema.yml
-          diff pyrseas/schema.yaml schema.yml > schema.diff || true
       - name: "Upload test artifacts"
         if: success() || failure()
         uses: actions/upload-artifact@v4
@@ -661,11 +652,6 @@ jobs:
           # Copy the artifacts out of the Docker container to project directory
           docker cp ci-${APP}-${CI_PIPELINE_ID}-${{ github.job }}:/usr/src/app/log ./ || true
           docker cp ci-${APP}-${CI_PIPELINE_ID}-${{ github.job }}:/usr/src/app/coverage.txt ./ || true
-          # The prod image does not include pyrseas/dbtoyaml. Building a test image to include that
-          docker compose build app
-          # Save the database schema as an artifact
-          docker compose run --no-deps --rm --entrypoint dbtoyaml app --no-owner --no-privileges test_${PGDATABASE} > schema.yml
-          diff pyrseas/schema.yaml schema.yml > schema.diff || true
       - name: "Upload test artifacts"
         if: success() || failure()
         uses: actions/upload-artifact@v4

README.md

@@ -23,3 +23,53 @@ baseline
 
 This produces a .puml file that can be rendered using a PlantUML
 server, either within your IDE or using a service like http://www.plantuml.com/.
+
+## Debugging inside Docker Containers
+
+The `LAUNCHER` environment sets a wrapper program around the Python process
+(`gunicorn`, `dagster-daemon`, `dagster-webserver`). This can be used to
+enable a debugger inside Docker Containers:
+
+1. Set `LAUNCHER="python3 -m debugpy --listen 0.0.0.0:5678"` in `.env`
+2. Create Launch Configurations in Visual Studio Code like:
+
+```json
+			{
+				"name": "Python: Attach to app (Docker Container)",
+				"type": "debugpy",
+				"request": "attach",
+				"connect": {
+					"host": "localhost",
+					"port": 5678
+				},
+				"pathMappings": [
+					{
+						"localRoot": "${workspaceFolder:hea}",
+						"remoteRoot": "/usr/src/app"
+					}
+				],
+				"django": true,
+				"justMyCode": false
+			},
+			{
+				"name": "Python: Attach to dagster-daemon (Docker Container)",
+				"type": "debugpy",
+				"request": "attach",
+				"connect": {
+					"host": "localhost",
+					"port": 5680
+				},
+				"pathMappings": [
+					{
+						"localRoot": "${workspaceFolder:hea}",
+						"remoteRoot": "/usr/src/app"
+					}
+				],
+				"django": true,
+				"justMyCode": false
+			}
+```
+
+3. Start the Docker containers as normal, and then use the Run and Debug
+pane in Visual Studio code to launch the configuration that attaches to
+the desired server.

apps/metadata/migrations/0012_alter_activitylabel_activity_type_and_more.py

@@ -0,0 +1,71 @@
+# Generated by Django 5.2.6 on 2025-10-08 22:47
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("metadata", "0011_alter_activitylabel_additional_identifier"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="activitylabel",
+            name="activity_type",
+            field=models.CharField(
+                choices=[
+                    ("LivelihoodActivity", "Livelihood Activity"),
+                    ("OtherCashIncome", "Other Cash Income"),
+                    ("WildFoods", "Wild Foods, Fishing or Hunting"),
+                    ("LivelihoodSummary", "Livelihood Summary"),
+                ],
+                default="LivelihoodActivity",
+                help_text="The type of Livelihood Activity the label is for: either a general Livelihood Activity, or an Other Cash Income activity from the 'Data2' worksheet, or a Wild Foods, Fishing or Hunting activity from the 'Data3' worksheet, or a label from the 'Summary' section of the 'Data' worksheet.",
+                max_length=20,
+                verbose_name="Activity Type",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="activitylabel",
+            name="status",
+            field=models.CharField(
+                blank=True,
+                choices=[
+                    ("Regular Expression", "Processed by Regular Expression"),
+                    ("Override", "Override automatically recognized metadata"),
+                    ("Discussion", "Under Discussion"),
+                    ("Correct BSS", "Correct the BSS"),
+                    ("Ignore", "Ignore this label and associated data in the row"),
+                ],
+                max_length=20,
+                verbose_name="Status",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="activitylabel",
+            name="strategy_type",
+            field=models.CharField(
+                blank=True,
+                choices=[
+                    ("MilkProduction", "Milk Production"),
+                    ("ButterProduction", "Butter Production"),
+                    ("MeatProduction", "Meat Production"),
+                    ("LivestockSale", "Livestock Sale"),
+                    ("CropProduction", "Crop Production"),
+                    ("FoodPurchase", "Food Purchase"),
+                    ("PaymentInKind", "Payment in Kind"),
+                    ("ReliefGiftOther", "Relief, Gift or Other Food"),
+                    ("Hunting", "Hunting"),
+                    ("Fishing", "Fishing"),
+                    ("WildFoodGathering", "Wild Food Gathering"),
+                    ("OtherCashIncome", "Other Cash Income"),
+                    ("OtherPurchase", "Other Purchase"),
+                    ("LivestockProduction", "Livestock Production"),
+                ],
+                help_text="The type of livelihood strategy, such as crop production, or wild food gathering.",
+                max_length=30,
+                verbose_name="Strategy Type",
+            ),
+        ),
+    ]

apps/metadata/models.py

@@ -393,11 +393,15 @@ class LabelStatus(models.TextChoices):
         OVERRIDE = "Override", _("Override automatically recognized metadata")
         DISCUSSION = "Discussion", _("Under Discussion")
         CORRECT_BSS = "Correct BSS", _("Correct the BSS")
+        IGNORE = "Ignore", _("Ignore this label and associated data in the row")
 
     class LivelihoodActivityType(models.TextChoices):
         LIVELIHOOD_ACTIVITY = "LivelihoodActivity", _("Livelihood Activity")  # Labels from the 'Data' worksheet
         OTHER_CASH_INCOME = "OtherCashIncome", _("Other Cash Income")  # Labels from the 'Data2' worksheet
-        WILD_FOODS = "WildFoods", _("Wild Foods")  # Labels from the 'Data3' worksheet
+        WILD_FOODS = "WildFoods", _("Wild Foods, Fishing or Hunting")  # Labels from the 'Data3' worksheet
+        LIVELIHOOD_SUMMARY = "LivelihoodSummary", _(
+            "Livelihood Summary"
+        )  # Labels from the 'Summary' section of the 'Data' worksheet
 
     activity_label = common_models.NameField(max_length=200, verbose_name=_("Activity Label"))
     activity_type = models.CharField(
@@ -406,9 +410,9 @@ class LivelihoodActivityType(models.TextChoices):
         choices=LivelihoodActivityType.choices,
         default=LivelihoodActivityType.LIVELIHOOD_ACTIVITY,
         help_text=_(
-            "The type of Livelihood Activity, either a general Livelihood Activity, or an Other Cash Income "
-            "activity from the 'Data2' worksheet, or a Wild Foods, Fishing or Hunting activity from the "
-            "'Data3' worksheet."
+            "The type of Livelihood Activity the label is for: either a general Livelihood Activity, or an Other Cash "
+            "Income activity from the 'Data2' worksheet, or a Wild Foods, Fishing or Hunting activity from the "
+            "'Data3' worksheet, or a label from the 'Summary' section of the 'Data' worksheet."
         ),
     )
     status = models.CharField(blank=True, max_length=20, choices=LabelStatus.choices, verbose_name=_("Status"))
@@ -420,7 +424,10 @@ class LivelihoodActivityType(models.TextChoices):
     strategy_type = models.CharField(
         max_length=30,
         blank=True,
-        choices=LivelihoodStrategyType.choices,
+        # We add an additional choice for LivestockProduction here, which is only valid when
+        # activity_type is LivelihoodSummary. LivestockProduction is the total of MeatProduction,
+        # MilkProduction and ButterProduction, and is used in the Summary section of the Data worksheet only
+        choices=LivelihoodStrategyType.choices + [("LivestockProduction", _("Livestock Production"))],  # type: ignore
         verbose_name=_("Strategy Type"),
         help_text=_("The type of livelihood strategy, such as crop production, or wild food gathering."),
     )

docker-compose.override.yml

@@ -15,6 +15,7 @@ services:
     build:
       target: dev
     ports:
+      - "5678:5678"
       - "8000:8000"
       - "8888:8888"
     volumes:
@@ -30,6 +31,9 @@ services:
       - ./env.example:/usr/src/app/.env
     environment:
       DJANGO_SETTINGS_MODULE: hea.settings.local
+      LAUNCHER: ${LAUNCHER}  # e.g. "debugpy" or "ddtrace"
+      # Disable frozen modules warning
+      PYDEVD_DISABLE_FILE_VALIDATION: 1
       # Put .coverage in a writable directory
       COVERAGE_FILE: log/.coverage
     command:
@@ -41,6 +45,7 @@ services:
     restart: no
     ports:
       - "3000:3000"
+      - "5679:5678"
     volumes:
       - ./:/usr/src/app
       # Separate volumes for writable directories inside the container
@@ -54,9 +59,14 @@ services:
       - ./env.example:/usr/src/app/.env
     environment:
       DJANGO_SETTINGS_MODULE: hea.settings.local
+      LAUNCHER: ${LAUNCHER}  # e.g. "debugpy" or "ddtrace"
+      # Disable frozen modules warning
+      PYDEVD_DISABLE_FILE_VALIDATION: 1
 
   dagster_daemon:
     restart: no
+    ports:
+      - "5680:5678"
     volumes:
       - ./:/usr/src/app
       # Separate volumes for writable directories inside the container
@@ -70,4 +80,7 @@ services:
       - ./env.example:/usr/src/app/.env
     environment:
       DJANGO_SETTINGS_MODULE: hea.settings.local
+      LAUNCHER: ${LAUNCHER}  # e.g. "debugpy" or "ddtrace"
+      # Disable frozen modules warning
+      PYDEVD_DISABLE_FILE_VALIDATION: 1
 

docker-compose.yml

@@ -110,14 +110,7 @@ services:
       MINIO_ENDPOINT_URL: http://minio:9000
       SUPPORT_EMAIL_ADDRESS: ${SUPPORT_EMAIL_ADDRESS}
       DJANGO_MIGRATE: 1
-      KILUIGI_INTERMEDIATETARGET_BACKEND_CLASS: ${KILUIGI_INTERMEDIATETARGET_BACKEND_CLASS}
-      KILUIGI_INTERMEDIATETARGET_ROOT_PATH: ${KILUIGI_INTERMEDIATETARGET_ROOT_PATH}
-      KILUIGI_FINALTARGET_BACKEND_CLASS: ${KILUIGI_FINALTARGET_BACKEND_CLASS}
-      KILUIGI_FINALTARGET_ROOT_PATH: ${KILUIGI_FINALTARGET_ROOT_PATH}
-      KILUIGI_REPORTTARGET_BACKEND_CLASS: ${KILUIGI_REPORTTARGET_BACKEND_CLASS}
-      KILUIGI_REPORTTARGET_ROOT_PATH: ${KILUIGI_REPORTTARGET_ROOT_PATH}      
       GOOGLE_APPLICATION_CREDENTIALS: ${GOOGLE_APPLICATION_CREDENTIALS}
-      GOOGLE_ADMIN_EMAIL: ${GOOGLE_ADMIN_EMAIL}
     command:
       - --timeout=3600
       - --workers=12

docker/app/Dockerfile

@@ -1,4 +1,4 @@
-FROM --platform=linux/amd64 python:3.12-bookworm as base
+FROM python:3.12-bookworm as base
 
 # set up apt repositories for postgres installation
 RUN curl -s https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | tee /usr/share/keyrings/pgdg.gpg >/dev/null && \

docker/app/run_dagster_daemon.sh

@@ -13,5 +13,8 @@ echo Setting up logs
 touch log/django.log
 chown -R django:django log/*
 
-echo Starting Dagster
-gosu django dagster-daemon run $*
+echo Starting Dagster Daemon
+if [ x"$LAUNCHER" != x"" ]; then
+    echo using ${LAUNCHER}
+fi
+gosu django ${LAUNCHER} /usr/local/bin/dagster-daemon run $*

docker/app/run_dagster_webserver.sh

@@ -13,5 +13,8 @@ echo Setting up logs
 touch log/django.log
 chown -R django:django log/*
 
-echo Starting Dagster
-gosu django dagster-webserver -h 0.0.0.0 -p 3000 -m pipelines --path-prefix /${DAGSTER_WEBSERVER_PREFIX} $*
+echo Starting Dagster Webserver
+if [ x"$LAUNCHER" != x"" ]; then
+    echo using ${LAUNCHER}
+fi
+gosu django ${LAUNCHER} /usr/local/bin/dagster-webserver -h 0.0.0.0 -p 3000 -m pipelines --path-prefix /${DAGSTER_WEBSERVER_PREFIX} $*

docker/app/run_django.sh

@@ -40,7 +40,10 @@ touch log/django_sql.log
 chown -R django:django log/*
 
 echo Starting Gunicorn with DJANGO_SETTINGS_MODULE=${DJANGO_SETTINGS_MODULE}
-gosu django  gunicorn ${APP}.wsgi:application \
+if [ x"$LAUNCHER" != x"" ]; then
+    echo using ${LAUNCHER}
+fi
+gosu django ${LAUNCHER} /usr/local/bin/gunicorn ${APP}.wsgi:application \
     --name ${APP}${ENV} \
     --config $(dirname $(readlink -f "$0"))/gunicorn_config.py \
     $* 2>&1

docker/db/Dockerfile

@@ -1,3 +1,5 @@
-FROM --platform=linux/amd64 postgis/postgis:17-3.5
+# Use a third party multicarch base image for compatibility with both ARM and AMD architectures
+# until PostGIS fix https://github.com/postgis/docker-postgis/issues/216
+FROM ghcr.io/baosystems/postgis:17-3.5
 
 COPY create_db.sh /docker-entrypoint-initdb.d/create_db.sh

docker/db/create_db.sh

@@ -1,5 +1,5 @@
 #!/bin/sh
-psql --set=CLIENT=$CLIENT --set=APP=$APP --set=ENV=$ENV --set=POSTGRES_PASSWORD=$POSTGRES_PASSWORD --set=DAGSTER_PASSWORD=$DAGSTER_PASSWORD --set=CREATE_TEMPLATE=${CREATE_TEMPLATE:-false} -d postgres --echo-all << EOF
+psql --set=CLIENT=$CLIENT --set=APP=$APP --set=ENV=$ENV --set=POSTGRES_PASSWORD=$POSTGRES_PASSWORD --set=CREATE_TEMPLATE=${CREATE_TEMPLATE:-false} -d postgres --echo-all << EOF
 
 \set DATABASE :CLIENT :APP :ENV
 \set OWNER :CLIENT :APP :ENV
@@ -74,7 +74,7 @@ ALTER DEFAULT PRIVILEGES IN SCHEMA :SCHEMA
   GRANT SELECT ON TABLES TO :OWNER;
 
 \set DAGSTER :CLIENT :APP :ENV
-CREATE ROLE :DAGSTER PASSWORD :'DAGSTER_PASSWORD' NOLOGIN CREATEDB NOCREATEROLE NOSUPERUSER;
+CREATE ROLE :DAGSTER PASSWORD :'POSTGRES_PASSWORD' NOLOGIN CREATEDB NOCREATEROLE NOSUPERUSER;
 COMMENT ON ROLE :DAGSTER IS 'Main Dagster pipeline user for :CLIENT :PRJ :ENV';
 GRANT :DAGSTER TO :OWNER;
 GRANT CONNECT, TEMPORARY, CREATE ON DATABASE :DATABASE TO :DAGSTER;

env.example

@@ -48,3 +48,8 @@ BSS_METADATA_WORKBOOK='gdrive://Database Design/BSS Metadata'  # 15XVXFjbom1sScV
 BSS_METADATA_STORAGE_OPTIONS='{"token": "service_account", "access": "read_only", "creds": ${GOOGLE_APPLICATION_CREDENTIALS}, "root_file_id": "0AOJ0gJ8sjnO7Uk9PVA"}'
 BSS_FILES_FOLDER='gdrive://Discovery Folder/Baseline Storage Sheets (BSS)'
 BSS_FILES_STORAGE_OPTIONS='{"token": "service_account", "access": "read_only", "creds": ${GOOGLE_APPLICATION_CREDENTIALS}, "root_file_id": "0AOJ0gJ8sjnO7Uk9PVA"}'
+
+# LAUNCHER can be used to configure a wrapper program around the Python process
+# For example, to add ddtrace or debugpy
+# Use the VSCode debugger as a launcher
+# LAUNCHER = "python3 -m debugpy --listen 0.0.0.0:5679"

pipelines/__init__.py

@@ -17,13 +17,17 @@
 )
 from .assets.livelihood_activity import (
     all_livelihood_activity_labels_dataframe,
+    all_livelihood_summary_labels_dataframe,
     imported_livelihood_activities,
     livelihood_activity_dataframe,
     livelihood_activity_fixture,
     livelihood_activity_instances,
     livelihood_activity_label_dataframe,
     livelihood_activity_valid_instances,
+    livelihood_summary_dataframe,
+    livelihood_summary_label_dataframe,
     summary_livelihood_activity_labels_dataframe,
+    summary_livelihood_summary_labels_dataframe,
 )
 from .assets.other_cash_income import (
     all_other_cash_income_labels_dataframe,
@@ -85,6 +89,10 @@
         livelihood_activity_label_dataframe,
         all_livelihood_activity_labels_dataframe,
         summary_livelihood_activity_labels_dataframe,
+        livelihood_summary_dataframe,
+        livelihood_summary_label_dataframe,
+        all_livelihood_summary_labels_dataframe,
+        summary_livelihood_summary_labels_dataframe,
         livelihood_activity_instances,
         livelihood_activity_valid_instances,
         livelihood_activity_fixture,