Update datasets

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 
+- Support HCAT mapping CSV files without crop_code
+- Split Germany BB and NDS in block dataset and crop fields
+- Add HCAT to datasets where possible
+- Updated years & variants for at_crop, be_vlg, es_an, es_cl, es_pv, ie, pt, se
 - Extend create_stac, include include fiboa data
 - Publish command; skip hidden files, generate better texts
 - Fix to vecorel: converter.license and provider should be string

fiboa_cli/datasets/ai4sf.py

@@ -111,7 +111,7 @@ def migrate(self, gdf):
         gdf["fiboa_id"] = (
             gdf["id"].astype(str).str.zfill(2) + "_" + gdf.index.astype(str).str.zfill(5)
         )
-        return gdf
+        return super().migrate(gdf)
 
     missing_schemas = {
         "properties": {

fiboa_cli/datasets/at.py

@@ -1,44 +1,47 @@
 from vecorel_cli.conversion.admin import AdminConverterMixin
 
-from fiboa_cli.conversion.fiboa_converter import FiboaBaseConverter
+from ..conversion.fiboa_converter import FiboaBaseConverter
+from .commons.ec import AddHCATMixin
 
 
-class Converter(AdminConverterMixin, FiboaBaseConverter):
-    sources = {
-        "https://inspire.lfrz.gv.at/009501/ds/inspire_referenzen_2021_polygon.gpkg.zip": [
-            "INSPIRE_REFERENZEN_2021_POLYGON.gpkg"
-        ]
+class Converter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter):
+    variants = {
+        "2025": "https://inspire.lfrz.gv.at/009501/ds/inspire_schlaege_2025-1_polygon.gpkg.zip",
+        "2024": "https://inspire.lfrz.gv.at/009501/ds/inspire_schlaege_2024-2_polygon.gpkg.zip",
+        "2023": "https://inspire.lfrz.gv.at/009501/ds/inspire_schlaege_2023-2_polygon.gpkg.zip",
+        "2022": "https://inspire.lfrz.gv.at/009501/ds/inspire_schlaege_2022_polygon.gpkg.zip",
+        "2021": "https://inspire.lfrz.gv.at/009501/ds/inspire_schlaege_2021_polygon.gpkg.zip",
+        "2020": "https://inspire.lfrz.gv.at/009501/ds/inspire_schlaege_2020_polygon.gpkg.zip",
+        "2019": "https://inspire.lfrz.gv.at/009501/ds/inspire_schlaege_2019_polygon.gpkg.zip",
+        "2018": "https://inspire.lfrz.gv.at/009501/ds/inspire_schlaege_2018_polygon.gpkg.zip",
     }
+
     id = "at"
     country = "AT"
     short_name = "Austria"
     title = "Field boundaries for Austria"
     description = """
-**Field boundaries for Austria - INVEKOS Referenzen Österreich 2021.**
-
-The layer includes all reference parcels ("Referenzparzellen") defined by the paying agency Agrarmarkt Austria and recorded landscape elements (landscape element layers) within the meaning of Art. 5 of Regulation (EU) No. 640/2014 and Regulation of the competent federal ministry with horizontal rules for the area of the Common Agricultural Policy (Horizontal CAP Regulation) StF: Federal Law Gazette II No. 100/2015.
+**Crop Field boundaries for Austria - INVEKOS Schläge Österreich 2025.**
 
-Reference parcel: is the physical block that can be clearly delimited from the outside (e.g. forest, roads, water bodies) and is formed by contiguous agricultural areas that are recognizable in nature.
+This layer includes all field uses recorded by the applicants, which serve as the basis for the funding process. A field
+is a contiguous area of a piece of land that is cultivated for a growing season with only one crop (field use type) and
+uniform management requirements or as a landscape element type in accordance with Annex 1 of the regulation of the responsible
+Federal Ministry with horizontal rules for the area of the Common Agricultural Policy (Horizontal CAP Regulation)
+StF: BGBl. II No. 100/2015 or is simply maintained in good agricultural and ecological condition in accordance with
+Art. 94 of Regulation (EU) No. 1306/2013 and is digitized in the GIS as a polygon or as a point.
     """
     provider = "Agrarmarkt Austria <https://geometadatensuche.inspire.gv.at/metadatensuche/inspire/api/records/9db8a0c3-e92a-4df4-9d55-8210e326a7ed>"
     license = "CC-BY-4.0"
     columns = {
-        "geometry": "geometry",
-        "RFL_ID": "id",
-        "REF_ART": "ref_art",
-        "BRUTTOFLAECHE_HA": "metrics:area",
+        "GEO_ID": "id",
         "INSPIRE_ID": "inspire:id",
-        "REF_ART_BEZEICHNUNG": "ref_art_bezeichnung",
-        "REFERENZ_KENNUNG": "referenz_kennung",
-        "FART_ID": "fart_id",
-        "GEO_DATERF": "determination:datetime",
+        "geometry": "geometry",
+        "SNAR_CODE": "crop:code",
+        "SNAR_BEZEICHNUNG": "crop:name",
+        "SL_FLAECHE_BRUTTO_HA": "metrics:area",
+        "GEOM_DATE_CREATED": "determination:datetime",
     }
-    extensions = {"https://fiboa.org/inspire-extension/v0.3.0/schema.yaml"}
-    missing_schemas = {
-        "properties": {
-            "ref_art": {"type": "string"},
-            "ref_art_bezeichnung": {"type": "string"},
-            "referenz_kennung": {"type": "uint64"},
-            "fart_id": {"type": "uint32"},
-        }
+    ec_mapping_csv = "at_2021.csv"
+    extensions = {
+        "https://fiboa.org/inspire-extension/v0.3.0/schema.yaml",
     }

fiboa_cli/datasets/at_block.py

@@ -0,0 +1,44 @@
+from vecorel_cli.conversion.admin import AdminConverterMixin
+
+from fiboa_cli.conversion.fiboa_converter import FiboaBaseConverter
+
+
+class Converter(AdminConverterMixin, FiboaBaseConverter):
+    sources = {
+        "https://inspire.lfrz.gv.at/009501/ds/inspire_referenzen_2021_polygon.gpkg.zip": [
+            "INSPIRE_REFERENZEN_2021_POLYGON.gpkg"
+        ]
+    }
+    id = "at_block"
+    country = "AT"
+    short_name = "Austria"
+    title = "Field boundaries for Austria"
+    description = """
+**Field boundaries for Austria - INVEKOS Referenzen Österreich 2021.**
+
+The layer includes all reference parcels ("Referenzparzellen") defined by the paying agency Agrarmarkt Austria and recorded landscape elements (landscape element layers) within the meaning of Art. 5 of Regulation (EU) No. 640/2014 and Regulation of the competent federal ministry with horizontal rules for the area of the Common Agricultural Policy (Horizontal CAP Regulation) StF: Federal Law Gazette II No. 100/2015.
+
+Reference parcel: is the physical block that can be clearly delimited from the outside (e.g. forest, roads, water bodies) and is formed by contiguous agricultural areas that are recognizable in nature.
+    """
+    provider = "Agrarmarkt Austria <https://geometadatensuche.inspire.gv.at/metadatensuche/inspire/api/records/9db8a0c3-e92a-4df4-9d55-8210e326a7ed>"
+    license = "CC-BY-4.0"
+    columns = {
+        "geometry": "geometry",
+        "RFL_ID": "id",
+        "REF_ART": "ref_art",
+        "BRUTTOFLAECHE_HA": "metrics:area",
+        "INSPIRE_ID": "inspire:id",
+        "REF_ART_BEZEICHNUNG": "ref_art_bezeichnung",
+        "REFERENZ_KENNUNG": "referenz_kennung",
+        "FART_ID": "fart_id",
+        "GEO_DATERF": "determination:datetime",
+    }
+    extensions = {"https://fiboa.org/inspire-extension/v0.3.0/schema.yaml"}
+    missing_schemas = {
+        "properties": {
+            "ref_art": {"type": "string"},
+            "ref_art_bezeichnung": {"type": "string"},
+            "referenz_kennung": {"type": "uint64"},
+            "fart_id": {"type": "uint32"},
+        }
+    }

fiboa_cli/datasets/be_vlg.py

@@ -10,7 +10,8 @@ class Converter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter):
     variants = {
         str(k): {PREFIX + v: [v.replace("_GPKG.zip", ".gpkg")]}
         for k, v in (
-            (2024, "Landbouwgebruikspercelen_2024_-_Voorlopig_(extractie_15-10-2024)_GPKG.zip"),
+            (2025, "Landbouwgebruikspercelen_2025_-_Voorlopig_(extractie_02-06-2025)_GPKG.zip"),
+            (2024, "Landbouwgebruikspercelen_2024_-_Definitief_(extractie_27-03-2025)_GPKG.zip"),
             (2023, "Landbouwgebruikspercelen_2023_-_Definitief_(extractie_28-03-2024)_GPKG.zip"),
             (2022, "Landbouwgebruikspercelen_2022_-_Definitief_(extractie_26-06-2023)_GPKG.zip"),
             (2021, "Landbouwgebruikspercelen_2021_-_Definitief_(extractie_15-03-2022)_GPKG.zip"),
@@ -35,7 +36,6 @@ class Converter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter):
 
     columns = {
         "geometry": "geometry",
-        "BT_BRON": "source",
         "BT_OMSCH": "typology",
         "GRAF_OPP": "metrics:area",
         "REF_ID": "id",
@@ -47,4 +47,4 @@ class Converter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter):
     }
     ec_mapping_csv = "be_vlg_2021.csv"
 
-    missing_schemas = {"properties": {"source": {"type": "string"}, "typology": {"type": "string"}}}
+    missing_schemas = {"properties": {"typology": {"type": "string"}}}

fiboa_cli/datasets/be_wal.py

@@ -41,10 +41,6 @@ class Converter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter):
     column_additions = {
         "determination:datetime": "2022-01-01T00:00:00Z",
     }
-    column_migrations = {
-        "crop_code": lambda col: col.str.extract(r"\.(\d+)$", expand=False),
-        "crop_name": lambda col: col.str.strip(),
-    }
     index_as_id = True
 
     def layer_filter(self, layer: str, uri: str) -> bool:
@@ -53,11 +49,14 @@ def layer_filter(self, layer: str, uri: str) -> bool:
     def file_migration(
         self, gdf: gpd.GeoDataFrame, path: str, uri: str, layer: str = None
     ) -> gpd.GeoDataFrame:
-        return gml_assure_columns(
+        gdf = gml_assure_columns(
             gdf,
             path,
             uri,
             layer,
             crop_name={"ElementPath": "specificLandUse@title", "Type": "String", "Width": 255},
             crop_code={"ElementPath": "specificLandUse@href", "Type": "String", "Width": 255},
         )
+        gdf["crop_code"] = gdf["crop_code"].str.extract(r"\.(\d+)$", expand=False)
+        gdf["crop_name"] = gdf["crop_name"].str.strip()
+        return gdf

fiboa_cli/datasets/br_conab.py

@@ -93,7 +93,7 @@ def migrate(self, gdf):
         gdf.loc[gdf["area_ha"] == 0, "area_ha"] = None
         gdf["cd_mun"] = gdf["cd_mun"].combine_first(gdf["CD_MUN"]).apply(fformat)
         gdf["nm_mun"] = gdf["nm_mun"].combine_first(gdf["NM_MUN"]).combine_first(gdf["NM_MUNIC"])
-        return gdf
+        return super().migrate(gdf)
 
     def get_data(self, paths, **kwargs):
         # Set invalid geometries to None in Cafe/MG/CAFE-MG_Safra_2017.zip

fiboa_cli/datasets/commons/hcat.py

@@ -3,6 +3,7 @@
 from typing import Optional
 
 import geopandas as gpd
+import numpy as np
 from vecorel_cli.vecorel.util import load_file
 
 HCAT_EXTENSION = "https://fiboa.org/hcat-extension/v0.3.0/schema.yaml"
@@ -17,7 +18,7 @@ class AddHCATMixin:
 
     ec_mapping_csv: Optional[str] = None  # TODO rename to hcat_mapping_csv
     mapping_file = None
-    ec_mapping: Optional[dict] = None  # TODO rename to hcat_mapping
+    ec_mapping: Optional[list[dict]] = None  # TODO rename to hcat_mapping
 
     hcat_columns = {
         "hcat:name_en": "hcat:name_en",
@@ -38,11 +39,11 @@ def convert(self, *args, **kwargs):
             )
         return super().convert(*args, **kwargs)
 
-    def get_code_column(self, gdf):
+    def get_code_column(self, gdf, code="crop:code"):
         try:
-            attribute = next(k for k, v in self.columns.items() if v == "crop:code")
+            attribute = next(k for k, v in self.columns.items() if v == code)
         except StopIteration:
-            raise Exception(f"Misssing crop:code column in converter {self.__class__.__name__}")
+            raise Exception(f"Misssing {code} column in converter {self.__class__.__name__}")
         col = gdf[attribute]
         # Should be corrected in original parser
         return col if col.dtype == "object" else col.astype(str)
@@ -56,15 +57,33 @@ def add_hcat(self, gdf):
 
             if self.ec_mapping is None:
                 self.ec_mapping = load_ec_mapping(self.ec_mapping_csv, url=self.mapping_file)
-            crop_code_col = self.get_code_column(gdf)
+
+            from_code = "original_code"
+            if from_code not in self.ec_mapping[0]:
+                # Some code lists have no code, only a crop_name
+                from_code = "original_name"
+                crop_code_col = self.get_code_column(gdf, "crop:name")
+            else:
+                crop_code_col = self.get_code_column(gdf)
 
             def map_to(attribute):
-                return {e["original_code"]: e[attribute] for e in self.ec_mapping}
+                return {e[from_code]: e[attribute] for e in self.ec_mapping}
 
+            col = None
             for k, v in zip(
                 self.hcat_columns.keys(), ("translated_name", "HCAT3_name", "HCAT3_code")
             ):
-                gdf[k] = crop_code_col.map(map_to(v))
+                if v in self.ec_mapping[0]:
+                    col = crop_code_col.map(map_to(v))
+                    gdf[k] = col
+                    assert np.unique(col[~col.isna()]).size > 1, "No HCAT crops mapped"
+
+            if col is not None and col.isna().any():
+                index = [
+                    k for k, v in self.columns.items() if v.startswith("crop:") and k in gdf.columns
+                ]
+                missing = gdf[col.isna()][index].drop_duplicates()
+                self.info(f"Missing codes in HCAT mapping:\n{missing}")
 
         if "crop:code_list" not in gdf.columns:
             gdf["crop:code_list"] = (
@@ -78,6 +97,8 @@ def migrate(self, gdf) -> gpd.GeoDataFrame:
 
 
 def ec_url(csv_file):
+    if csv_file.startswith("https://"):
+        return csv_file
     return f"https://raw.githubusercontent.com/maja601/EuroCrops/refs/heads/main/csvs/country_mappings/{csv_file}"
 
 

fiboa_cli/datasets/cz.py

@@ -2,10 +2,10 @@
 from vecorel_cli.conversion.admin import AdminConverterMixin
 
 from ..conversion.fiboa_converter import FiboaBaseConverter
-from .commons.ec import ec_url
+from .commons.hcat import AddHCATMixin
 
 
-class Converter(AdminConverterMixin, FiboaBaseConverter):
+class Converter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter):
     sources = "https://mze.gov.cz/public/app/eagriapp/Files/geoprostor_zadosti23_2024-08-01_202409261243_epsg4258.zip"
     id = "cz"
     short_name = "Czech"
@@ -24,8 +24,7 @@ class Converter(AdminConverterMixin, FiboaBaseConverter):
         # 'OKRES_NAZE': 'admin:subdivision_code',
     }
     column_migrations = {"DATUM_REP": lambda col: pd.to_datetime(col, format="%d.%m.%Y")}
-    extensions = {"https://fiboa.org/crop-extension/v0.2.0/schema.yaml"}
-    column_additions = {"crop:code_list": ec_url("cz_2023.csv")}
+    ec_mapping_csv = "cz_2023.csv"
     missing_schemas = {
         "properties": {
             "block_id": {"type": "string"},