Skip to content

Commit 959ed21

Browse files
ondrajodasondrajodas
committed
query test, config tests
1 parent bd9619e commit 959ed21

File tree

5 files changed

+328
-6
lines changed

5 files changed

+328
-6
lines changed

‎src/Keboola/DynamoDbExtractor/ConfigDefinition.php‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,11 @@ class ConfigDefinition implements ConfigurationInterface
1616

1717
private const QUERY_INVALID_NODES = ['dateFilter'];
1818

19-
private const SCAN_INVALID_NODES = ['KeyConditionExpression', 'ExpressionAttributeValues'];
19+
private const SCAN_INVALID_NODES = [
20+
'keyConditionExpression',
21+
'expressionAttributeNames',
22+
'expressionAttributeValues',
23+
];
2024

2125
public function getConfigTreeBuilder(): TreeBuilder
2226
{

‎src/Keboola/DynamoDbExtractor/ReadingAdapter/QueryReadingAdapter.php‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,14 @@ public function read(array $params): void
1313
{
1414
$marshaler = new Marshaler();
1515

16-
$params['KeyConditionExpression'] = $this->exportOptions['keyConditionExpression'];
17-
$params['ExpressionAttributeValues'] = $marshaler->marshalJson(
18-
(string) json_encode($this->exportOptions['expressionAttributeValues'])
19-
);
16+
if (!empty($this->exportOptions['keyConditionExpression'])) {
17+
$params['KeyConditionExpression'] = $this->exportOptions['keyConditionExpression'];
18+
}
19+
if (!empty($this->exportOptions['expressionAttributeValues'])) {
20+
$params['ExpressionAttributeValues'] = $marshaler->marshalJson(
21+
(string) json_encode($this->exportOptions['expressionAttributeValues'])
22+
);
23+
}
2024
if (!empty($this->exportOptions['expressionAttributeNames'])) {
2125
$params['ExpressionAttributeNames'] = $this->exportOptions['expressionAttributeNames'];
2226
}

‎tests/Keboola/DynamoDbExtractor/ConfigurationTest.php‎

Lines changed: 221 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,221 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
namespace Keboola\DynamoDbExtractor;
6+
7+
use Generator;
8+
use PHPUnit\Framework\TestCase;
9+
use Symfony\Component\Config\Definition\Exception\InvalidConfigurationException;
10+
use Symfony\Component\Config\Definition\Processor;
11+
12+
class ConfigurationTest extends TestCase
13+
{
14+
/**
15+
* @dataProvider validConfigDataProvider
16+
*/
17+
public function testValidConfig(array $config): void
18+
{
19+
$parameters = (new Processor)->processConfiguration(
20+
new ConfigDefinition,
21+
[$config['parameters']]
22+
);
23+
24+
self::assertEquals($config['parameters'], $parameters);
25+
}
26+
27+
/**
28+
* @dataProvider invalidConfigDataProvider
29+
*/
30+
public function testInvalidConfig(array $config, string $exceptionMessage): void
31+
{
32+
$this->expectException(InvalidConfigurationException::class);
33+
$this->expectExceptionMessage($exceptionMessage);
34+
(new Processor)->processConfiguration(
35+
new ConfigDefinition,
36+
[$config['parameters']]
37+
);
38+
}
39+
40+
public function validConfigDataProvider(): Generator
41+
{
42+
yield 'min-config' => [
43+
[
44+
'parameters' => [
45+
'db' => [
46+
'endpoint' => 'dynamoUrl',
47+
'accessKeyId' => 'key',
48+
'#secretAccessKey' => 'secret',
49+
'regionName' => 'eu-central-1',
50+
],
51+
'exports' => [
52+
[
53+
'id' => 1234,
54+
'name' => 'testName',
55+
'table' => 'testTable',
56+
'incremental' => false,
57+
'mode' => 'scan',
58+
'enabled' => true,
59+
'primaryKey' => [],
60+
'mapping' => [
61+
'id' => 'id',
62+
],
63+
],
64+
],
65+
],
66+
],
67+
];
68+
69+
yield 'query-reading' => [
70+
[
71+
'parameters' => [
72+
'db' => [
73+
'endpoint' => 'dynamoUrl',
74+
'accessKeyId' => 'key',
75+
'#secretAccessKey' => 'secret',
76+
'regionName' => 'eu-central-1',
77+
],
78+
'exports' => [
79+
[
80+
'id' => 1234,
81+
'name' => 'testName',
82+
'table' => 'testTable',
83+
'incremental' => false,
84+
'mode' => 'query',
85+
'keyConditionExpression' => 'acb',
86+
'expressionAttributeValues' => [],
87+
'expressionAttributeNames' => [],
88+
'enabled' => true,
89+
'primaryKey' => [],
90+
'mapping' => [
91+
'id' => 'id',
92+
],
93+
],
94+
],
95+
],
96+
],
97+
];
98+
}
99+
100+
public function invalidConfigDataProvider(): Generator
101+
{
102+
yield 'bad-scan-params-1' => [
103+
[
104+
'parameters' => [
105+
'db' => [
106+
'endpoint' => 'dynamoUrl',
107+
'accessKeyId' => 'key',
108+
'#secretAccessKey' => 'secret',
109+
'regionName' => 'eu-central-1',
110+
],
111+
'exports' => [
112+
[
113+
'id' => 1234,
114+
'name' => 'testName',
115+
'table' => 'testTable',
116+
'incremental' => false,
117+
'mode' => 'scan',
118+
'keyConditionExpression' => 'acb',
119+
'expressionAttributeNames' => [],
120+
'expressionAttributeValues' => [],
121+
'enabled' => true,
122+
'primaryKey' => [],
123+
'mapping' => [
124+
'id' => 'id',
125+
],
126+
],
127+
],
128+
],
129+
],
130+
'Node "keyConditionExpression" is not allowed for scan export.',
131+
];
132+
133+
yield 'bad-scan-params-2' => [
134+
[
135+
'parameters' => [
136+
'db' => [
137+
'endpoint' => 'dynamoUrl',
138+
'accessKeyId' => 'key',
139+
'#secretAccessKey' => 'secret',
140+
'regionName' => 'eu-central-1',
141+
],
142+
'exports' => [
143+
[
144+
'id' => 1234,
145+
'name' => 'testName',
146+
'table' => 'testTable',
147+
'incremental' => false,
148+
'mode' => 'scan',
149+
'expressionAttributeNames' => [],
150+
'expressionAttributeValues' => [],
151+
'enabled' => true,
152+
'primaryKey' => [],
153+
'mapping' => [
154+
'id' => 'id',
155+
],
156+
],
157+
],
158+
],
159+
],
160+
'Node "expressionAttributeNames" is not allowed for scan export.',
161+
];
162+
163+
yield 'bad-scan-params-3' => [
164+
[
165+
'parameters' => [
166+
'db' => [
167+
'endpoint' => 'dynamoUrl',
168+
'accessKeyId' => 'key',
169+
'#secretAccessKey' => 'secret',
170+
'regionName' => 'eu-central-1',
171+
],
172+
'exports' => [
173+
[
174+
'id' => 1234,
175+
'name' => 'testName',
176+
'table' => 'testTable',
177+
'incremental' => false,
178+
'mode' => 'scan',
179+
'expressionAttributeValues' => [],
180+
'enabled' => true,
181+
'primaryKey' => [],
182+
'mapping' => [
183+
'id' => 'id',
184+
],
185+
],
186+
],
187+
],
188+
],
189+
'Node "expressionAttributeValues" is not allowed for scan export.',
190+
];
191+
192+
yield 'bad-query-params' => [
193+
[
194+
'parameters' => [
195+
'db' => [
196+
'endpoint' => 'dynamoUrl',
197+
'accessKeyId' => 'key',
198+
'#secretAccessKey' => 'secret',
199+
'regionName' => 'eu-central-1',
200+
],
201+
'exports' => [
202+
[
203+
'id' => 1234,
204+
'name' => 'testName',
205+
'table' => 'testTable',
206+
'incremental' => false,
207+
'mode' => 'query',
208+
'dateFilter' => [],
209+
'enabled' => true,
210+
'primaryKey' => [],
211+
'mapping' => [
212+
'id' => 'id',
213+
],
214+
],
215+
],
216+
],
217+
],
218+
'Node "dateFilter" is not allowed for query export.',
219+
];
220+
}
221+
}

‎tests/Keboola/DynamoDbExtractor/RunQueryFullExportTest.php‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
namespace Keboola\DynamoDbExtractor;
6+
7+
use Symfony\Component\Console\Application;
8+
use Symfony\Component\Console\Tester\CommandTester;
9+
10+
class RunQueryFullExportTest extends ExtractorTestCase
11+
{
12+
protected string $dataDir = '/tmp/run';
13+
14+
protected function setUp(): void
15+
{
16+
parent::setUp();
17+
18+
$this->fs->dumpFile($this->dataDir . '/config.json', <<<JSON
19+
{
20+
"parameters": {
21+
"db": {
22+
"endpoint": "http://dynamodb:8000",
23+
"accessKeyId": "key",
24+
"#secretAccessKey": "secret",
25+
"regionName": "eu-central-1"
26+
},
27+
"exports": [
28+
{
29+
"id": 1,
30+
"name": "10-movies",
31+
"table": "Movies",
32+
"enabled": true,
33+
"incremental": true,
34+
"mode": "query",
35+
"keyConditionExpression": "#yr = :a",
36+
"expressionAttributeNames": {
37+
"#yr": "year"
38+
},
39+
"expressionAttributeValues": {
40+
":a": "2013"
41+
},
42+
"primaryKey": ["title", "year"],
43+
"mapping": {
44+
"title": "title",
45+
"year": "year",
46+
"info.rating": "rating"
47+
}
48+
}
49+
]
50+
}
51+
}
52+
JSON
53+
);
54+
}
55+
56+
public function testRun(): void
57+
{
58+
$application = new Application;
59+
$application->add(new RunCommand);
60+
61+
$command = $application->find('run');
62+
$commandTester = new CommandTester($command);
63+
64+
$exitCode = $commandTester->execute([
65+
'command' => $command->getName(),
66+
'data directory' => $this->dataDir,
67+
], ['capture_stderr_separately' => true]);
68+
69+
$expectedFile = $this->dataDir . '/out/tables/10-movies.csv';
70+
$expectedManifestFile = $expectedFile . '.manifest';
71+
$this->assertSame(0, $exitCode);
72+
$this->assertFileExists($expectedFile);
73+
$this->assertFileExists($expectedManifestFile);
74+
75+
$expectedCsv = <<<CSV
76+
"title","year","rating"
77+
"Insidious: Chapter 2","2013","7.1"
78+
"Now You See Me","2013","7.3"
79+
"Prisoners","2013","8.2"
80+
"Rush","2013","8.3"
81+
"The Hunger Games: Catching Fire","2013",""
82+
"This Is the End","2013","7.2"
83+
"Thor: The Dark World","2013",""
84+
"World War Z","2013","7.1"\n
85+
CSV;
86+
$this->assertEquals($expectedCsv, file_get_contents($expectedFile));
87+
88+
$expectedManifest = <<<JSON
89+
{"primary_key":["title","year"],"incremental":true}
90+
JSON;
91+
$this->assertEquals($expectedManifest, file_get_contents($expectedManifestFile));
92+
}
93+
}

tests/Keboola/DynamoDbExtractor/RunFullExportTest.php renamed to tests/Keboola/DynamoDbExtractor/RunScanFullExportTest.php

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
use Symfony\Component\Console\Application;
88
use Symfony\Component\Console\Tester\CommandTester;
99

10-
class RunFullExportTest extends ExtractorTestCase
10+
class RunScanFullExportTest extends ExtractorTestCase
1111
{
1212
protected string $dataDir = '/tmp/run';
1313

0 commit comments

Comments
 (0)