Merge pull request #12 from keboola/ujovlado-fix-missing-manifest-support

Fix missing manifest support

Author: ujovlado

composer.json

@@ -18,7 +18,8 @@
         "symfony/filesystem": "^4.0",
         "nette/utils": "^2.4",
         "keboola/csvmap": "^0.3",
-        "monolog/monolog": "^1.22"
+        "monolog/monolog": "^1.22",
+        "keboola/php-component": "^2.1"
     },
     "require-dev": {
         "phpunit/phpunit": "~5.3",

composer.lock

@@ -6,6 +6,7 @@
 use Nette\Utils\Strings;
 use Symfony\Component\Config\Definition\Processor;
 use Symfony\Component\Console\Output\OutputInterface;
+use Keboola\Component\Manifest\ManifestManager;
 
 class Extractor
 {
@@ -61,20 +62,28 @@ public function actionRun(string $outputPath): void
     {
         $this->validateExports($this->parameters['exports']);
 
+        $manifestManager = new ManifestManager($outputPath);
+
         foreach ($this->parameters['exports'] as $exportOptions) {
             $export = new Exporter($this->dynamoDbClient, $exportOptions, $outputPath, $this->consoleOutput);
 
             if ($export->hasEnabledExport()) {
                 $filename = $export->export();
                 if (file_exists($filename)) {
+                    $webalizedExportName = Strings::webalize($exportOptions['name']);
                     $parser = new Parser(
-                        Strings::webalize($exportOptions['name']),
+                        $webalizedExportName,
                         $filename,
                         $exportOptions['mapping'],
                         $this->consoleOutput
                     );
+
                     $parser->parseAndWriteCsvFiles();
                     $export->cleanup();
+
+                    $manifestManager->writeTableManifestFromArray($outputPath . '/' .$webalizedExportName . '.csv', [
+                        'incremental' => $exportOptions['incremental'],
+                    ]);
                 } else {
                     $this->consoleOutput->writeln('No documents found for export ' . $exportOptions['name']);
                 }

src/Keboola/DynamoDbExtractor/Extractor.php

@@ -5,7 +5,7 @@
 use Symfony\Component\Console\Application;
 use Symfony\Component\Console\Tester\CommandTester;
 
-class RunFullExportTest extends ExtractorTestCase
+class RunExportFromIndexTest extends ExtractorTestCase
 {
     /** @var string */
     protected $dataDir = '/tmp/run-export-from-index';
@@ -59,9 +59,11 @@ public function testRun()
         ]);
 
         $expectedFile = $this->dataDir . '/out/tables/10-movies.csv';
+        $expectedManifestFile = $expectedFile . '.manifest';
 
         $this->assertSame(0, $exitCode);
         $this->assertFileExists($expectedFile);
+        $this->assertFileExists($expectedManifestFile);
 
         $expectedCsv = <<<CSV
 "title","year","rating"
@@ -77,5 +79,10 @@ public function testRun()
 "X-Men: Days of Future Past","2014",""\n
 CSV;
         $this->assertEquals($expectedCsv, file_get_contents($expectedFile));
+
+        $expectedManifest = <<<JSON
+{"destination":"","primary_key":[],"delimiter":",","enclosure":"\"","columns":[],"incremental":true,"metadata":[],"column_metadata":[]}\n
+JSON;
+        $this->assertEquals($expectedManifest, file_get_contents($expectedManifestFile));
     }
 }

…DynamoDbExtractor/RunExportFromIndex.php …moDbExtractor/RunExportFromIndexTest.phptests/Keboola/DynamoDbExtractor/RunExportFromIndex.php renamed to tests/Keboola/DynamoDbExtractor/RunExportFromIndexTest.php tests/Keboola/DynamoDbExtractor/RunExportFromIndex.php renamed to tests/Keboola/DynamoDbExtractor/RunExportFromIndexTest.php

@@ -57,9 +57,11 @@ public function testRun()
         ]);
 
         $expectedFile = $this->dataDir . '/out/tables/10-movies.csv';
+        $expectedManifestFile = $expectedFile . '.manifest';
 
         $this->assertSame(0, $exitCode);
         $this->assertFileExists($expectedFile);
+        $this->assertFileExists($expectedManifestFile);
 
         $expectedCsv = <<<CSV
 "title","year","rating"
@@ -75,5 +77,10 @@ public function testRun()
 "World War Z","2013","7.1"\n
 CSV;
         $this->assertEquals($expectedCsv, file_get_contents($expectedFile));
+
+        $expectedManifest = <<<JSON
+{"destination":"","primary_key":[],"delimiter":",","enclosure":"\"","columns":[],"incremental":true,"metadata":[],"column_metadata":[]}\n
+JSON;
+        $this->assertEquals($expectedManifest, file_get_contents($expectedManifestFile));
     }
 }

tests/Keboola/DynamoDbExtractor/RunFullExportTest.php

@@ -29,7 +29,7 @@ protected function setUp()
         "name": "10-movies",
         "table": "Movies",
         "enabled": true,
-        "incremental": true,
+        "incremental": false,
         "dateFilter": {
           "field": "year",
           "format": "Y",
@@ -63,15 +63,22 @@ public function testRun()
         ]);
 
         $expectedFile = $this->dataDir . '/out/tables/10-movies.csv';
+        $expectedManifestFile = $expectedFile . '.manifest';
 
         $this->assertSame(0, $exitCode);
         $this->assertFileExists($expectedFile);
+        $this->assertFileExists($expectedManifestFile);
 
         $expectedCsv = <<<CSV
 "title","year","rating"
 "Transformers: Age of Extinction","2014",""
 "X-Men: Days of Future Past","2014",""\n
 CSV;
         $this->assertEquals($expectedCsv, file_get_contents($expectedFile));
+
+        $expectedManifest = <<<JSON
+{"destination":"","primary_key":[],"delimiter":",","enclosure":"\"","columns":[],"incremental":false,"metadata":[],"column_metadata":[]}\n
+JSON;
+        $this->assertEquals($expectedManifest, file_get_contents($expectedManifestFile));
     }
 }