Merge pull request #95 from ecmwf-projects/cdm-csv-tables

Read the new CSV CDM tables.

Author: garciampred

cdsobs/cdm/api.py

@@ -263,15 +263,17 @@ def open_asset(cdm_netcdf: str, decode_variables: bool = False) -> xarray.Datase
 
 
 def read_cdm_code_table(cdm_tables_location: Path, name: str) -> CDMCodeTable:
-    table_path = Path(cdm_tables_location, f"cdm-obs/tables/{name}.dat")
+    table_path = Path(cdm_tables_location, f"cdm-obs/tables/{name}.csv")
     table_data = pandas.read_csv(
         table_path,
-        delimiter="\t",
-        quoting=3,
+        delimiter=",",
+        quoting=0,
         dtype=str,
         na_filter=False,
         comment="#",
         index_col=0,
+        engine="python",
+        on_bad_lines="error",
     )
     return CDMCodeTable(name, table_data)
 

cdsobs/cdm/tables.py

@@ -179,12 +179,14 @@ def read_cdm_table(cdm_tables_location: Path, name: str) -> CDMTable:
     table_path = Path(cdm_tables_location, f"cdm-obs/table_definitions/{name}.csv")
     table_data = pandas.read_csv(
         table_path,
-        delimiter="\t",
-        quoting=3,
+        delimiter=",",
+        quoting=0,
         dtype=str,
         na_filter=False,
         comment="#",
         index_col="element_name",
+        engine="python",
+        on_bad_lines="error",
     ).drop("description", axis=1)
     # In some tables kind is called type
     if "kind" not in table_data:

tests/cli/test_app.py

@@ -92,6 +92,7 @@ def test_cli_make_cdm(tmp_path):
         tmp_path,
         "--source",
         "OzoneSonde",
+        "--disable-cdm-tag-check",
     ]
     result = runner.invoke(app, args, catch_exceptions=False)
     assert result.exit_code == 0