Merge pull request #2 from TicketSwap:tobias/move-fields-to-tap-config

move fields selection to tap config to fix flattening

Author: tobiascadee

meltano.yml

@@ -18,7 +18,16 @@ plugins:
     - structured-logging
 
     select:
-    - countries.cca3
+    - countries.*
+
+    config:
+      fields:
+      - name
+      - cca2
+      - cca3
+      - capital
+      - region
+
   loaders:
   - name: target-jsonl
     variant: andyh1203

tap_restcountries/client.py

@@ -52,9 +52,6 @@ def get_url_params(
         Returns:
             A dictionary of URL query parameters.
         """ 
-        selected_fields = [k[1] for k,v in self.metadata.items() if v.selected and len(k) == 2]
-        if len(selected_fields) > 10:
+        if len(self.config["fields"]) > 10:
             raise ValueError("The API has a maximum of 10 fields per request.")
-        elif len(selected_fields) == 0:
-            selected_fields = ["cca3"]
-        return {"fields": ",".join(selected_fields)}
+        return {"fields": ",".join(self.config["fields"])}

tap_restcountries/tap.py

@@ -21,7 +21,14 @@ class TapRestCountries(Tap):
 
     name = "tap-restcountries"
 
-    config_jsonschema = {}
+    config_jsonschema = th.PropertiesList(
+        th.Property(
+            "fields",
+            th.ArrayType(th.StringType),
+            required=True,
+            description="List of fields to include in the API response (max 10).",
+        ),
+    ).to_dict()
 
     @override
     def discover_streams(self) -> list[streams.RestCountriesStream]: