📊 agriculture: Update FAOSTAT data

dag/faostat.yml

@@ -159,5 +159,163 @@ steps:
   # Global food explorer.
   #
   export://explorers/faostat/latest/global_food:
-    - data://grapher/faostat/2025-03-17/faostat_qcl
-    - data://grapher/faostat/2025-03-17/faostat_fbsc
+    - data://grapher/faostat/2026-02-25/faostat_qcl
+    - data://grapher/faostat/2026-02-25/faostat_fbsc
+  #
+  # FAOSTAT meadow steps for version 2026-02-25
+  #
+  data://meadow/faostat/2026-02-25/faostat_cisp:
+    - snapshot://faostat/2026-02-25/faostat_cisp.zip
+  ##################################################################################################################
+  # NOTE: The latest version of FBS is missing data for various countries that are currently under review.
+  # For now, import the data for those missing countries from the previous version:
+  ##################################################################################################################
+  data://meadow/faostat/2026-02-25/faostat_fbs:
+    - snapshot://faostat/2026-02-25/faostat_fbs.zip
+    - snapshot://faostat/2025-03-17/faostat_fbs.zip
+  data://meadow/faostat/2026-02-25/faostat_fbsh:
+    - snapshot://faostat/2026-02-25/faostat_fbsh.zip
+  data://meadow/faostat/2026-02-25/faostat_fs:
+    - snapshot://faostat/2026-02-25/faostat_fs.zip
+  data://meadow/faostat/2026-02-25/faostat_lc:
+    - snapshot://faostat/2026-02-25/faostat_lc.zip
+  data://meadow/faostat/2026-02-25/faostat_metadata:
+    - snapshot://faostat/2026-02-25/faostat_metadata.json
+  data://meadow/faostat/2026-02-25/faostat_qcl:
+    - snapshot://faostat/2026-02-25/faostat_qcl.zip
+  data://meadow/faostat/2026-02-25/faostat_qi:
+    - snapshot://faostat/2026-02-25/faostat_qi.zip
+  data://meadow/faostat/2026-02-25/faostat_qv:
+    - snapshot://faostat/2026-02-25/faostat_qv.zip
+  data://meadow/faostat/2026-02-25/faostat_rfn:
+    - snapshot://faostat/2026-02-25/faostat_rfn.zip
+  data://meadow/faostat/2026-02-25/faostat_rl:
+    - snapshot://faostat/2026-02-25/faostat_rl.zip
+  data://meadow/faostat/2026-02-25/faostat_rp:
+    - snapshot://faostat/2026-02-25/faostat_rp.zip
+  data://meadow/faostat/2026-02-25/faostat_sdgb:
+    - snapshot://faostat/2026-02-25/faostat_sdgb.zip
+  #
+  # FAOSTAT garden steps for version 2026-02-25
+  #
+  data://garden/faostat/2026-02-25/faostat_cisp:
+    - data://meadow/faostat/2026-02-25/faostat_cisp
+    - data://garden/demography/2024-07-15/population
+    - data://garden/faostat/2026-02-25/faostat_metadata
+    - data://garden/wb/2025-07-01/income_groups
+    - data://garden/regions/2023-01-01/regions
+  data://garden/faostat/2026-02-25/faostat_fbsc:
+    - data://meadow/faostat/2026-02-25/faostat_fbsh
+    - data://garden/demography/2024-07-15/population
+    - data://garden/faostat/2026-02-25/faostat_metadata
+    - data://garden/wb/2025-07-01/income_groups
+    - data://meadow/faostat/2026-02-25/faostat_fbs
+    - data://garden/regions/2023-01-01/regions
+  data://garden/faostat/2026-02-25/faostat_fs:
+    - data://meadow/faostat/2026-02-25/faostat_fs
+    - data://garden/demography/2024-07-15/population
+    - data://garden/faostat/2026-02-25/faostat_metadata
+    - data://garden/wb/2025-07-01/income_groups
+    - data://garden/regions/2023-01-01/regions
+  data://garden/faostat/2026-02-25/faostat_lc:
+    - data://garden/demography/2024-07-15/population
+    - data://meadow/faostat/2026-02-25/faostat_lc
+    - data://garden/faostat/2026-02-25/faostat_metadata
+    - data://garden/wb/2025-07-01/income_groups
+    - data://garden/regions/2023-01-01/regions
+  data://garden/faostat/2026-02-25/faostat_metadata:
+    - data://meadow/faostat/2026-02-25/faostat_rp
+    - data://meadow/faostat/2026-02-25/faostat_fbsh
+    - data://meadow/faostat/2026-02-25/faostat_sdgb
+    - data://meadow/faostat/2026-02-25/faostat_cisp
+    - data://meadow/faostat/2026-02-25/faostat_metadata
+    - data://meadow/faostat/2026-02-25/faostat_qi
+    - data://meadow/faostat/2026-02-25/faostat_fs
+    - data://meadow/faostat/2026-02-25/faostat_rfn
+    - data://meadow/faostat/2026-02-25/faostat_rl
+    - data://meadow/faostat/2026-02-25/faostat_lc
+    - data://meadow/faostat/2026-02-25/faostat_qcl
+    - data://meadow/faostat/2026-02-25/faostat_fbs
+    - data://meadow/faostat/2026-02-25/faostat_qv
+  data://garden/faostat/2026-02-25/faostat_qcl:
+    - data://garden/demography/2024-07-15/population
+    - data://meadow/faostat/2026-02-25/faostat_qcl
+    - data://garden/faostat/2026-02-25/faostat_metadata
+    - data://garden/wb/2025-07-01/income_groups
+    - data://garden/regions/2023-01-01/regions
+  data://garden/faostat/2026-02-25/faostat_qi:
+    - data://meadow/faostat/2026-02-25/faostat_qi
+    - data://garden/demography/2024-07-15/population
+    - data://garden/faostat/2026-02-25/faostat_metadata
+    - data://garden/wb/2025-07-01/income_groups
+    - data://garden/regions/2023-01-01/regions
+  data://garden/faostat/2026-02-25/faostat_qv:
+    - data://garden/regions/2023-01-01/regions
+    - data://garden/demography/2024-07-15/population
+    - data://garden/faostat/2026-02-25/faostat_metadata
+    - data://garden/wb/2025-07-01/income_groups
+    - data://meadow/faostat/2026-02-25/faostat_qv
+  data://garden/faostat/2026-02-25/faostat_rfn:
+    - data://garden/demography/2024-07-15/population
+    - data://meadow/faostat/2026-02-25/faostat_rfn
+    - data://garden/faostat/2026-02-25/faostat_metadata
+    - data://garden/wb/2025-07-01/income_groups
+    - data://garden/regions/2023-01-01/regions
+  data://garden/faostat/2026-02-25/faostat_rl:
+    - data://garden/demography/2024-07-15/population
+    - data://meadow/faostat/2026-02-25/faostat_rl
+    - data://garden/faostat/2026-02-25/faostat_metadata
+    - data://garden/wb/2025-07-01/income_groups
+    - data://garden/regions/2023-01-01/regions
+  data://garden/faostat/2026-02-25/faostat_rp:
+    - data://meadow/faostat/2026-02-25/faostat_rp
+    - data://garden/demography/2024-07-15/population
+    - data://garden/faostat/2026-02-25/faostat_metadata
+    - data://garden/wb/2025-07-01/income_groups
+    - data://garden/regions/2023-01-01/regions
+  data://garden/faostat/2026-02-25/faostat_sdgb:
+    - data://meadow/faostat/2026-02-25/faostat_sdgb
+    - data://garden/demography/2024-07-15/population
+    - data://garden/faostat/2026-02-25/faostat_metadata
+    - data://garden/wb/2025-07-01/income_groups
+    - data://garden/regions/2023-01-01/regions
+  #
+  # FAOSTAT grapher steps for version 2026-02-25
+  #
+  data://grapher/faostat/2026-02-25/faostat_cisp:
+    - data://garden/faostat/2026-02-25/faostat_cisp
+  data://grapher/faostat/2026-02-25/faostat_fbsc:
+    - data://garden/faostat/2026-02-25/faostat_fbsc
+  data://grapher/faostat/2026-02-25/faostat_fs:
+    - data://garden/faostat/2026-02-25/faostat_fs
+  data://grapher/faostat/2026-02-25/faostat_lc:
+    - data://garden/faostat/2026-02-25/faostat_lc
+  data://grapher/faostat/2026-02-25/faostat_qcl:
+    - data://garden/faostat/2026-02-25/faostat_qcl
+  data://grapher/faostat/2026-02-25/faostat_qi:
+    - data://garden/faostat/2026-02-25/faostat_qi
+  data://grapher/faostat/2026-02-25/faostat_qv:
+    - data://garden/faostat/2026-02-25/faostat_qv
+  data://grapher/faostat/2026-02-25/faostat_rfn:
+    - data://garden/faostat/2026-02-25/faostat_rfn
+  data://grapher/faostat/2026-02-25/faostat_rl:
+    - data://garden/faostat/2026-02-25/faostat_rl
+  data://grapher/faostat/2026-02-25/faostat_rp:
+    - data://garden/faostat/2026-02-25/faostat_rp
+  data://grapher/faostat/2026-02-25/faostat_sdgb:
+    - data://garden/faostat/2026-02-25/faostat_sdgb
+  #
+  # FAOSTAT garden step for additional variables for version 2026-02-25
+  #
+  data://garden/faostat/2026-02-25/additional_variables:
+    - data://garden/faostat/2026-02-25/faostat_rl
+    - data://garden/faostat/2026-02-25/faostat_qi
+    - data://garden/faostat/2026-02-25/faostat_qcl
+    - data://garden/faostat/2026-02-25/faostat_sdgb
+    - data://garden/faostat/2026-02-25/faostat_fbsc
+    - data://garden/faostat/2026-02-25/faostat_rfn
+  #
+  # FAOSTAT grapher step for additional variables for version 2026-02-25
+  #
+  data://grapher/faostat/2026-02-25/additional_variables:
+    - data://garden/faostat/2026-02-25/additional_variables

docs/data/faostat.md

@@ -206,12 +206,6 @@ Using data from garden, we create an additional dataset in the `explorers` chann
 These are the steps OWID follows to ensure that FAOSTAT data is up-to-date, or to update one or more datasets for
 which there is new data (let us call the new dataset version to be created `YYYY-MM-DD`):
 
-0. Activate the etl virtual environment (from the root folder of the etl repository):
-
-```bash
-  . .venv/bin/activate
-```
-
 1.  Execute the ingestion script, to fetch data for any dataset that may have been updated in FAOSTAT.
     If no dataset requires an update, the workflow stops here.
 
@@ -220,7 +214,7 @@ which there is new data (let us call the new dataset version to be created `YYYY
             This can be executed with the `-r` flag to simply check for updates without writing anything.
 
         ```bash
-        python etl/scripts/faostat/create_new_snapshots.py
+        python etl/scripts/faostat/create_new_snapshots.py -a
         ```
 
         !!! note
@@ -233,17 +227,7 @@ which there is new data (let us call the new dataset version to be created `YYYY
             downloading this domain, add it to the list `INCLUDED_DATASETS_CODES`. Then replace variables used in those
             charts with the new ones.
 
-2.  Manually inspect the snapshot metadata files, and fix common issues in dataset descriptions:
-
-- Insert line break after first sentence (which usually is the general description of the dataset).
-- Remove spurious symbols.
-- Insert spaces where missing (e.g. "end of sentence.Start of next sentence").
-- Remove double spaces (e.g. "end of sentence. Start of next sentence").
-- Insert line breaks to create paragraphs (by context).
-- Remove incomplete sentences (sometimes there are half sentences that may have been added by mistake).
-- Remove mentions to links in FAOSTAT page (since they will not be seen from grapher).
-
-3.  Create new meadow steps.
+2.  Create new meadow steps.
 
     !!! note
 
@@ -253,13 +237,13 @@ which there is new data (let us call the new dataset version to be created `YYYY
     python etl/scripts/faostat/create_new_steps.py -c meadow -a
     ```
 
-4.  Run the new etl meadow steps, to generate the meadow datasets.
+3.  Run the new etl meadow steps, to generate the meadow datasets.
 
     ```bash
     etl run meadow/faostat/YYYY-MM-DD
     ```
 
-5.  Create new garden steps.
+4.  Create new garden steps.
 
     ```bash
     python etl/scripts/faostat/create_new_steps.py -c garden
@@ -272,7 +256,7 @@ which there is new data (let us call the new dataset version to be created `YYYY
         This way we can be aware of any unexpected FAO changes in units.
         If any changes are made to `custom_*.csv` files, you may need to force-run the garden `faostat_metadata` step to implement those changes.
 
-6.  Run the new etl garden steps, to generate the garden datasets.
+5.  Run the new etl garden steps, to generate the garden datasets.
 
     ```bash
     etl run garden/faostat/YYYY-MM-DD
@@ -296,7 +280,7 @@ which there is new data (let us call the new dataset version to be created `YYYY
 
         TODO: The descriptions of anomalies used to appear in `description`, but now they are not included in any indicator metadata. Ideally they should appear in `description_processing`. Consider doing this in the next update.
 
-7.  Inspect and update any possible changes of dataset/item/element/unit names and descriptions.
+6.  Inspect and update any possible changes of dataset/item/element/unit names and descriptions.
 
     ```bash
     python etl/scripts/faostat/update_custom_metadata.py
@@ -308,38 +292,31 @@ which there is new data (let us call the new dataset version to be created `YYYY
     etl run garden/faostat/YYYY-MM-DD
     ```
 
-8.  Create new grapher steps.
+7.  Create new grapher steps.
 
     ```bash
     python etl/scripts/faostat/create_new_steps.py -c grapher
     ```
 
-9.  Run the new etl grapher steps, to generate the grapher charts.
+8.  Run the new etl grapher steps, to generate the grapher charts.
 
     ```bash
     etl run faostat/YYYY-MM-DD --grapher
     ```
 
-10. From the ETL Wizard, use Indicator Upgrader for each of the grapher datasets to replace variables in charts to their latest versions.
+9. Replace variables in charts to their latest versions.
 
-11. Update the versions of the dependencies of the explorers step `export://explorers/faostat/latest/global_food` in the dag (for the moment, this has to be done manually).
+  ```bash
+  etl indicator-upgrade auto
+  ```
 
-12. Run the explorers step, to update the global food explorer.
+10. Run the explorers step, to update the global food explorer.
 
     ```bash
     etl run explorers/faostat/latest/global_food --export
     ```
 
-13. From the ETL Wizard, use Chart Diff to visually inspect changes between the old and new versions of updated charts, and
-    accept or reject changes. Inspect also changes in the global food explorer using Explorer Diff.
-
-14. Manually create a new garden dataset of additional variables `additional_variables` for the new version, and update its metadata. Then create a new grapher dataset too. Manually update all other datasets that use any faostat dataset as a dependency.
-
-    !!! note
-
-        In the future this could be handled automatically by one of the existing scripts.
-
-15. Update titles and descriptions of snapshot origins (to use the custom dataset titles and descriptions defined in garden). Also, attributions will be added to origins.
+11. Update titles and descriptions of snapshot origins (to use the custom dataset titles and descriptions defined in garden). Also, attributions will be added to origins.
 
     ```bash
     python etl/scripts/faostat/update_snapshots_metadata.py
@@ -349,11 +326,22 @@ which there is new data (let us call the new dataset version to be created `YYYY
 
         The current workflow is a bit convoluted: we fetch snapshots, create meadow and garden steps, and the edit snapshots again. But for now, this workflow is the safest working solution.
 
-16. Manually update the version of any `faostat` used as dependency in unrelated datasets (`faostat_rl` is used in `weekly_wildfires` and `population`).
+12. From the ETL Wizard, use Anomalist to visually inspect potential data issues.
+
+13. From the ETL Wizard, use Anomalist and Chart Diff to visually inspect changes between the old and new versions of updated charts, and
+    accept or reject changes. Inspect also changes in the global food explorer using Explorer Diff.
+
+14. Manually update the version of any `faostat` used as dependency in unrelated datasets (`faostat_rl` is used in `weekly_wildfires`).
 
-17. From the ETL dashboard, select archivable, namespace `faostat`, and archive all old steps.
+15. Update other steps in the `agriculture` namespace that rely on any `faostat_*` step.
+
+16. Archive old steps.
+
+    ```bash
+    etl archive faostat/YYYY-MM-DD --include-usages
+    ```
 
-18. After merging all code and once production is up-to-date, archive unnecessary grapher datasets.
+17. After merging all code and once production is up-to-date, archive unnecessary grapher datasets.
 
 ## Workflow to make changes to a dataset
 

etl/scripts/faostat/create_new_snapshots.py

@@ -20,6 +20,10 @@
 ```
 uv run python -m create_new_snapshots
 ```
+* To create new snapshots for all datasets, even if the source data was not updated:
+```
+uv run python -m create_new_snapshots -a
+```
 
 """
 
@@ -280,7 +284,7 @@ def to_snapshot(self) -> None:
             snap.create_snapshot(filename=f.name, upload=True)
 
 
-def main(read_only: bool = False) -> None:
+def main(read_only: bool = False, include_all_datasets: bool = False) -> None:
     # Load list of existing snapshots related to current NAMESPACE.
     existing_snapshots = [
         snapshot for snapshot in list(snapshot_catalog(match=NAMESPACE)) if "backport/" not in snapshot.uri
@@ -303,7 +307,7 @@ def main(read_only: bool = False) -> None:
         dataset_code = description["DatasetCode"].lower()
         if dataset_code in INCLUDED_DATASETS_CODES:
             faostat_dataset = FAODataset(description)
-            if is_dataset_already_up_to_date(
+            if not include_all_datasets and is_dataset_already_up_to_date(
                 existing_snapshots=existing_snapshots,
                 source_data_url=faostat_dataset.source_data_url,
                 source_modification_date=faostat_dataset.modification_date,
@@ -337,5 +341,12 @@ def main(read_only: bool = False) -> None:
         action="store_true",
         help="If given, simply check for updates without creating snapshots.",
     )
+    argument_parser.add_argument(
+        "-a",
+        "--include_all_datasets",
+        default=False,
+        action="store_true",
+        help="If given, create snapshots for all datasets, even if the source data was not updated.",
+    )
     args = argument_parser.parse_args()
-    main(read_only=args.read_only)
+    main(read_only=args.read_only, include_all_datasets=args.include_all_datasets)