Skip to content

Commit e2c246c

Browse files
girumbgirumb
committed
Merge branch 'main' into HEA-752/Dagster-GraphQL-API-is-intermittently-failing-with-a-ProtocolError-when-accessed-via-the-revproxy-Django-view
2 parents f7658cc + 2c6ce2f commit e2c246c

File tree

11 files changed

+528
-77
lines changed

11 files changed

+528
-77
lines changed

‎apps/metadata/migrations/0012_alter_activitylabel_activity_type_and_more.py‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
# Generated by Django 5.2.6 on 2025-10-08 22:47
2+
3+
from django.db import migrations, models
4+
5+
6+
class Migration(migrations.Migration):
7+
8+
dependencies = [
9+
("metadata", "0011_alter_activitylabel_additional_identifier"),
10+
]
11+
12+
operations = [
13+
migrations.AlterField(
14+
model_name="activitylabel",
15+
name="activity_type",
16+
field=models.CharField(
17+
choices=[
18+
("LivelihoodActivity", "Livelihood Activity"),
19+
("OtherCashIncome", "Other Cash Income"),
20+
("WildFoods", "Wild Foods, Fishing or Hunting"),
21+
("LivelihoodSummary", "Livelihood Summary"),
22+
],
23+
default="LivelihoodActivity",
24+
help_text="The type of Livelihood Activity the label is for: either a general Livelihood Activity, or an Other Cash Income activity from the 'Data2' worksheet, or a Wild Foods, Fishing or Hunting activity from the 'Data3' worksheet, or a label from the 'Summary' section of the 'Data' worksheet.",
25+
max_length=20,
26+
verbose_name="Activity Type",
27+
),
28+
),
29+
migrations.AlterField(
30+
model_name="activitylabel",
31+
name="status",
32+
field=models.CharField(
33+
blank=True,
34+
choices=[
35+
("Regular Expression", "Processed by Regular Expression"),
36+
("Override", "Override automatically recognized metadata"),
37+
("Discussion", "Under Discussion"),
38+
("Correct BSS", "Correct the BSS"),
39+
("Ignore", "Ignore this label and associated data in the row"),
40+
],
41+
max_length=20,
42+
verbose_name="Status",
43+
),
44+
),
45+
migrations.AlterField(
46+
model_name="activitylabel",
47+
name="strategy_type",
48+
field=models.CharField(
49+
blank=True,
50+
choices=[
51+
("MilkProduction", "Milk Production"),
52+
("ButterProduction", "Butter Production"),
53+
("MeatProduction", "Meat Production"),
54+
("LivestockSale", "Livestock Sale"),
55+
("CropProduction", "Crop Production"),
56+
("FoodPurchase", "Food Purchase"),
57+
("PaymentInKind", "Payment in Kind"),
58+
("ReliefGiftOther", "Relief, Gift or Other Food"),
59+
("Hunting", "Hunting"),
60+
("Fishing", "Fishing"),
61+
("WildFoodGathering", "Wild Food Gathering"),
62+
("OtherCashIncome", "Other Cash Income"),
63+
("OtherPurchase", "Other Purchase"),
64+
("LivestockProduction", "Livestock Production"),
65+
],
66+
help_text="The type of livelihood strategy, such as crop production, or wild food gathering.",
67+
max_length=30,
68+
verbose_name="Strategy Type",
69+
),
70+
),
71+
]

‎apps/metadata/models.py‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -393,11 +393,15 @@ class LabelStatus(models.TextChoices):
393393
OVERRIDE = "Override", _("Override automatically recognized metadata")
394394
DISCUSSION = "Discussion", _("Under Discussion")
395395
CORRECT_BSS = "Correct BSS", _("Correct the BSS")
396+
IGNORE = "Ignore", _("Ignore this label and associated data in the row")
396397

397398
class LivelihoodActivityType(models.TextChoices):
398399
LIVELIHOOD_ACTIVITY = "LivelihoodActivity", _("Livelihood Activity") # Labels from the 'Data' worksheet
399400
OTHER_CASH_INCOME = "OtherCashIncome", _("Other Cash Income") # Labels from the 'Data2' worksheet
400-
WILD_FOODS = "WildFoods", _("Wild Foods") # Labels from the 'Data3' worksheet
401+
WILD_FOODS = "WildFoods", _("Wild Foods, Fishing or Hunting") # Labels from the 'Data3' worksheet
402+
LIVELIHOOD_SUMMARY = "LivelihoodSummary", _(
403+
"Livelihood Summary"
404+
) # Labels from the 'Summary' section of the 'Data' worksheet
401405

402406
activity_label = common_models.NameField(max_length=200, verbose_name=_("Activity Label"))
403407
activity_type = models.CharField(
@@ -406,9 +410,9 @@ class LivelihoodActivityType(models.TextChoices):
406410
choices=LivelihoodActivityType.choices,
407411
default=LivelihoodActivityType.LIVELIHOOD_ACTIVITY,
408412
help_text=_(
409-
"The type of Livelihood Activity, either a general Livelihood Activity, or an Other Cash Income "
410-
"activity from the 'Data2' worksheet, or a Wild Foods, Fishing or Hunting activity from the "
411-
"'Data3' worksheet."
413+
"The type of Livelihood Activity the label is for: either a general Livelihood Activity, or an Other Cash "
414+
"Income activity from the 'Data2' worksheet, or a Wild Foods, Fishing or Hunting activity from the "
415+
"'Data3' worksheet, or a label from the 'Summary' section of the 'Data' worksheet."
412416
),
413417
)
414418
status = models.CharField(blank=True, max_length=20, choices=LabelStatus.choices, verbose_name=_("Status"))
@@ -420,7 +424,10 @@ class LivelihoodActivityType(models.TextChoices):
420424
strategy_type = models.CharField(
421425
max_length=30,
422426
blank=True,
423-
choices=LivelihoodStrategyType.choices,
427+
# We add an additional choice for LivestockProduction here, which is only valid when
428+
# activity_type is LivelihoodSummary. LivestockProduction is the total of MeatProduction,
429+
# MilkProduction and ButterProduction, and is used in the Summary section of the Data worksheet only
430+
choices=LivelihoodStrategyType.choices + [("LivestockProduction", _("Livestock Production"))], # type: ignore
424431
verbose_name=_("Strategy Type"),
425432
help_text=_("The type of livelihood strategy, such as crop production, or wild food gathering."),
426433
)

‎pipelines/__init__.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,17 @@
1717
)
1818
from .assets.livelihood_activity import (
1919
all_livelihood_activity_labels_dataframe,
20+
all_livelihood_summary_labels_dataframe,
2021
imported_livelihood_activities,
2122
livelihood_activity_dataframe,
2223
livelihood_activity_fixture,
2324
livelihood_activity_instances,
2425
livelihood_activity_label_dataframe,
2526
livelihood_activity_valid_instances,
27+
livelihood_summary_dataframe,
28+
livelihood_summary_label_dataframe,
2629
summary_livelihood_activity_labels_dataframe,
30+
summary_livelihood_summary_labels_dataframe,
2731
)
2832
from .assets.other_cash_income import (
2933
all_other_cash_income_labels_dataframe,
@@ -85,6 +89,10 @@
8589
livelihood_activity_label_dataframe,
8690
all_livelihood_activity_labels_dataframe,
8791
summary_livelihood_activity_labels_dataframe,
92+
livelihood_summary_dataframe,
93+
livelihood_summary_label_dataframe,
94+
all_livelihood_summary_labels_dataframe,
95+
summary_livelihood_summary_labels_dataframe,
8896
livelihood_activity_instances,
8997
livelihood_activity_valid_instances,
9098
livelihood_activity_fixture,

‎pipelines/assets/base.py‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -417,7 +417,7 @@ def get_bss_dataframe(
417417
df.loc[:, "B":].apply(lambda row: sum((row != 0) & (row != "")), axis="columns").sum()
418418
),
419419
"preview": MetadataValue.md(df.head(config.preview_rows).to_markdown()),
420-
"sample": MetadataValue.md(sample_df.sample(sample_rows).to_markdown()),
420+
"sample": MetadataValue.md(sample_df.sample(sample_rows).sort_index().to_markdown()),
421421
},
422422
)
423423

@@ -477,7 +477,7 @@ def get_bss_label_dataframe(
477477
"num_summaries": int(label_df["in_summary"].sum()),
478478
# Escape the ~ in the partition_key, otherwise it is rendered as strikethrough
479479
"preview": MetadataValue.md(label_df.head(config.preview_rows).to_markdown().replace("~", "\\~")),
480-
"sample": MetadataValue.md(sample_df.sample(sample_rows).to_markdown().replace("~", "\\~")),
480+
"sample": MetadataValue.md(sample_df.sample(sample_rows).sort_index().to_markdown().replace("~", "\\~")),
481481
},
482482
)
483483

@@ -498,7 +498,7 @@ def get_all_bss_labels_dataframe(
498498
# Escape the ~ in the partition_key, otherwise it is rendered as strikethrough
499499
"preview": MetadataValue.md(df.head(config.preview_rows).to_markdown().replace("~", "\\~")),
500500
"sample": MetadataValue.md(
501-
df[df["in_summary"]].sample(config.preview_rows).to_markdown().replace("~", "\\~")
501+
df[df["in_summary"]].sample(config.preview_rows).sort_index().to_markdown().replace("~", "\\~")
502502
),
503503
},
504504
)
@@ -587,7 +587,8 @@ def translate_label(label, langs):
587587
label_metadata_df = pd.DataFrame.from_records(queryset)
588588

589589
# Merge the label metadata into the dataframe
590-
df = df.merge(label_metadata_df, left_on="label", right_on="label", how="left")
590+
if not label_metadata_df.empty:
591+
df = df.merge(label_metadata_df, left_on="label", right_on="label", how="left")
591592

592593
# Rename the columns to match what we need in the GSheet when we run jobs.metadata.load_all_metadata
593594
df = df.rename(

0 commit comments

Comments
 (0)