Merge pull request #31 from keboola/PST-2113-ondra

create manifest files for child tables

Author: ondrajodas

src/Keboola/DynamoDbExtractor/Extractor.php

@@ -73,11 +73,11 @@ public function actionRun(string $outputPath): void
                     $headers = $parser->parseAndWriteCsvFiles();
                     $export->cleanup();
 
-                    $options = new ManifestManager\Options\OutTable\ManifestOptions();
-                    $options->setIncremental($exportOptions['incremental']);
+                    foreach ($headers as $tableName => $tableHeaders) {
+                        $options = new ManifestManager\Options\OutTable\ManifestOptions();
+                        $options->setIncremental($exportOptions['incremental']);
 
-                    if (isset($headers[$webalizedExportName])) {
-                        foreach ($headers[$webalizedExportName] as $column) {
+                        foreach ($tableHeaders as $column) {
                             $column = trim($column, '"');
                             $isPrimaryKey = in_array($column, $exportOptions['primaryKey'] ?? [], true);
                             if ($isPrimaryKey) {
@@ -90,13 +90,13 @@ public function actionRun(string $outputPath): void
                                 $isPrimaryKey,
                             ));
                         }
-                    }
 
-                    $manifestManager->writeTableManifest(
-                        $webalizedExportName . '.csv',
-                        $options,
-                        $this->dataTypeSupport->usingLegacyManifest(),
-                    );
+                        $manifestManager->writeTableManifest(
+                            $tableName . '.csv',
+                            $options,
+                            $this->dataTypeSupport->usingLegacyManifest(),
+                        );
+                    }
                 } else {
                     $this->logger->info('No documents found for export ' . $exportOptions['name']);
                 }

tests/functional/run-scan-with-child-table/expected-code

@@ -0,0 +1 @@
+0

tests/functional/run-scan-with-child-table/expected-stdout

@@ -0,0 +1,2 @@
+Parsing "/tmp/run-%s/out/tables/10-movies.json"
+Done

tests/functional/run-scan-with-child-table/expected/data/out/files/.gitkeep

@@ -0,0 +1,10 @@
+"Transformers: Age of Extinction","d008089a1cb90697b3bbef6941c29624"
+"X-Men: Days of Future Past","f1f4064e7780cc73cefd9bf3e3ec5e5c"
+"Insidious: Chapter 2","0573c5cea38a536e2ea911e1d23ac083"
+"Now You See Me","4245e6dcfce5a9275511ec172690977e"
+"Prisoners","9b20dbf9a7f06306ef49368827ffa11b"
+"Rush","a214d738e894ec7c6b4137f5265daa9a"
+"The Hunger Games: Catching Fire","ef32318cfef9aa96c9ceb973a7b3a642"
+"This Is the End","94ecb1a4a5b002e7063f78d9b60cd041"
+"Thor: The Dark World","7f7968accdac206fed62393066f51388"
+"World War Z","084592b8967def33d25ef52abcc0b583"

tests/functional/run-scan-with-child-table/expected/data/out/tables/10-movies.csv

@@ -0,0 +1,29 @@
+{
+  "incremental": true,
+  "columns": [
+    "title",
+    "info"
+  ],
+  "column_metadata": {
+    "title": [
+      {
+        "key": "KBC.datatype.nullable",
+        "value": true
+      },
+      {
+        "key": "KBC.datatype.basetype",
+        "value": "string"
+      }
+    ],
+    "info": [
+      {
+        "key": "KBC.datatype.nullable",
+        "value": true
+      },
+      {
+        "key": "KBC.datatype.basetype",
+        "value": "string"
+      }
+    ]
+  }
+}

tests/functional/run-scan-with-child-table/expected/data/out/tables/10-movies.csv.manifest

@@ -0,0 +1,10 @@
+"","d008089a1cb90697b3bbef6941c29624"
+"","f1f4064e7780cc73cefd9bf3e3ec5e5c"
+"7.1","0573c5cea38a536e2ea911e1d23ac083"
+"7.3","4245e6dcfce5a9275511ec172690977e"
+"8.2","9b20dbf9a7f06306ef49368827ffa11b"
+"8.3","a214d738e894ec7c6b4137f5265daa9a"
+"","ef32318cfef9aa96c9ceb973a7b3a642"
+"7.2","94ecb1a4a5b002e7063f78d9b60cd041"
+"","7f7968accdac206fed62393066f51388"
+"7.1","084592b8967def33d25ef52abcc0b583"

tests/functional/run-scan-with-child-table/expected/data/out/tables/info.csv

@@ -0,0 +1,29 @@
+{
+  "incremental": true,
+  "columns": [
+    "rating",
+    "10-movies_pk"
+  ],
+  "column_metadata": {
+    "rating": [
+      {
+        "key": "KBC.datatype.nullable",
+        "value": true
+      },
+      {
+        "key": "KBC.datatype.basetype",
+        "value": "string"
+      }
+    ],
+    "10-movies_pk": [
+      {
+        "key": "KBC.datatype.nullable",
+        "value": true
+      },
+      {
+        "key": "KBC.datatype.basetype",
+        "value": "string"
+      }
+    ]
+  }
+}

tests/functional/run-scan-with-child-table/expected/data/out/tables/info.csv.manifest

@@ -0,0 +1,29 @@
+{
+  "parameters": {
+    "db": {
+      "endpoint": "http://dynamodb:8000",
+      "accessKeyId": "key",
+      "#secretAccessKey": "secret",
+      "regionName": "eu-central-1"
+    },
+    "exports": [
+      {
+        "id": 1,
+        "name": "10-movies",
+        "table": "Movies",
+        "enabled": true,
+        "incremental": true,
+        "mapping": {
+          "title": "title",
+          "info": {
+            "type": "table",
+            "destination": "info",
+            "tableMapping": {
+              "rating": "rating"
+            }
+          }
+        }
+      }
+    ]
+  }
+}