Skip to content

Commit 6472683

Browse files
luandy64judahrand
luandy64
and
judahrand
authored
Fix tests for get_standard_metadata (#134)
* Fix testing of get_standard_metadata * Fix maxDiff issue * Add docstring * Revert change to get_standard_metadata() * Update how asserts work * Fix missing import * Revert changes to breadcrumbs * Rename test_kp to has_pk * Add test numbers * Add missing test * Fix failing test Co-authored-by: Jude188 <[email protected]>
1 parent eeed831 commit 6472683

File tree

1 file changed

+80
-58
lines changed

1 file changed

+80
-58
lines changed

‎tests/test_metadata.py‎

Lines changed: 80 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
from pprint import pprint
22
import unittest
3-
from singer.metadata import get_standard_metadata
3+
from singer.metadata import get_standard_metadata, to_map
44

5-
def make_expected_metadata(base_obj, dict_of_extras):
5+
def make_expected_metadata(base_obj, dict_of_extras, has_pk=False):
66
metadata_value = {**base_obj}
77
metadata_value.update(dict_of_extras)
88

@@ -13,7 +13,7 @@ def make_expected_metadata(base_obj, dict_of_extras):
1313
},
1414
{
1515
'metadata': {
16-
'inclusion': 'available',
16+
'inclusion': 'automatic' if has_pk else 'available',
1717
},
1818
'breadcrumb': ('properties', 'id')
1919
},
@@ -33,9 +33,19 @@ def make_expected_metadata(base_obj, dict_of_extras):
3333

3434
class TestStandardMetadata(unittest.TestCase):
3535

36-
#maxDiff = None
37-
3836
def test_standard_metadata(self):
37+
"""
38+
There's four inputs we want to test: schema, key_properties, replication_method, valid_replication_keys.
39+
40+
When `schema` is a non-null input, we expect `"inclusion": "available"` metadata for the `()` breadcrumb.
41+
42+
When `key_properties` is a non-null input, we expect `table-key-properties` metadata for the `()` breadcrumb.
43+
44+
When `replication_method` is a non-null input, we expect `forced-replication-method` metadata for the `()` breadcrumb.
45+
46+
When `valid_replication_keys` is a non-null input, we expect `valid-replication-keys` metadata for the `()` breadcrumb.
47+
"""
48+
self.maxDiff = None
3949

4050
# Some contants shared by a number of expected metadata objects
4151
tap_stream_id = 'employees'
@@ -44,7 +54,7 @@ def test_standard_metadata(self):
4454
test_rk = ['id', 'created']
4555
metadata_kp = {'table-key-properties': ['id']}
4656
metadata_rm = {'forced-replication-method': 'INCREMENTAL'}
47-
metadata_rk = {'valid_replication_keys': ['id','created']}
57+
metadata_rk = {'valid-replication-keys': ['id','created']}
4858
schema_present_base_obj = {'inclusion': 'available'}
4959
test_schema = {
5060
'type': ['null', 'object'],
@@ -61,7 +71,7 @@ def test_standard_metadata(self):
6171
# dictionary of parameters for `get_standard_metadata()` and the
6272
# second element is the expected metadata
6373
test_variables = [
64-
(
74+
( # test_number=0
6575
{
6676
'schema': test_schema,
6777
'schema_name': tap_stream_id,
@@ -74,7 +84,7 @@ def test_standard_metadata(self):
7484
{'schema-name': tap_stream_id,}
7585
)
7686
),
77-
(
87+
( # test_number=1
7888
{
7989
'schema': test_schema,
8090
'schema_name': tap_stream_id,
@@ -84,11 +94,11 @@ def test_standard_metadata(self):
8494
},
8595
make_expected_metadata(
8696
schema_present_base_obj,
87-
{'valid_replication_keys': ['id','created'],
97+
{'valid-replication-keys': ['id','created'],
8898
'schema-name':tap_stream_id}
8999
)
90100
),
91-
(
101+
( # test_number=2
92102
{
93103
'schema': test_schema,
94104
'schema_name': tap_stream_id,
@@ -102,7 +112,7 @@ def test_standard_metadata(self):
102112
'schema-name':tap_stream_id}
103113
)
104114
),
105-
(
115+
( # test_number=3
106116
{
107117
'schema': test_schema,
108118
'schema_name': tap_stream_id,
@@ -112,12 +122,12 @@ def test_standard_metadata(self):
112122
},
113123
make_expected_metadata(
114124
schema_present_base_obj,
115-
{'valid_replication_keys': ['id','created'],
125+
{'valid-replication-keys': ['id','created'],
116126
'forced-replication-method': 'INCREMENTAL',
117127
'schema-name':tap_stream_id}
118128
)
119129
),
120-
(
130+
( # test_number=4
121131
{
122132
'schema': test_schema,
123133
'schema_name': tap_stream_id,
@@ -128,10 +138,11 @@ def test_standard_metadata(self):
128138
make_expected_metadata(
129139
schema_present_base_obj,
130140
{'table-key-properties': ['id'],
131-
'schema-name':tap_stream_id}
141+
'schema-name':tap_stream_id},
142+
has_pk=True
132143
)
133144
),
134-
(
145+
( # test_number=5
135146
{
136147
'schema': test_schema,
137148
'schema_name': tap_stream_id,
@@ -140,14 +151,14 @@ def test_standard_metadata(self):
140151
'valid_replication_keys': test_rk
141152
},
142153
make_expected_metadata(
143-
144154
schema_present_base_obj,
145155
{'table-key-properties': ['id'],
146-
'valid_replication_keys': ['id','created'],
147-
'schema-name':tap_stream_id}
156+
'valid-replication-keys': ['id','created'],
157+
'schema-name':tap_stream_id},
158+
has_pk=True
148159
)
149160
),
150-
(
161+
( # test_number=6
151162
{
152163
'schema': test_schema,
153164
'schema_name': tap_stream_id,
@@ -159,10 +170,11 @@ def test_standard_metadata(self):
159170
schema_present_base_obj,
160171
{'table-key-properties': ['id'],
161172
'forced-replication-method': 'INCREMENTAL',
162-
'schema-name':tap_stream_id}
173+
'schema-name':tap_stream_id},
174+
has_pk=True
163175
)
164176
),
165-
(
177+
( # test_number=7
166178
{
167179
'schema': test_schema,
168180
'schema_name': tap_stream_id,
@@ -174,25 +186,21 @@ def test_standard_metadata(self):
174186
schema_present_base_obj,
175187
{'table-key-properties': ['id'],
176188
'forced-replication-method': 'INCREMENTAL',
177-
'valid_replication_keys': ['id','created'],
178-
'schema-name':tap_stream_id}
189+
'valid-replication-keys': ['id','created'],
190+
'schema-name':tap_stream_id},
191+
has_pk=True
179192
)
180193
),
181-
(
194+
( # test_number=8
182195
{
183196
'schema': None,
184197
'key_properties': None,
185198
'replication_method': None,
186199
'valid_replication_keys': None
187200
},
188-
[
189-
{
190-
'metadata': {},
191-
'breadcrumb': []
192-
}
193-
]
201+
[]
194202
),
195-
(
203+
( # test_number=9
196204
{
197205
'schema': None,
198206
'key_properties': None,
@@ -202,14 +210,13 @@ def test_standard_metadata(self):
202210
[
203211
{
204212
'metadata': {
205-
'inclusion': 'available',
206-
'valid_replication_keys': ['id','created']
213+
'valid-replication-keys': ['id','created']
207214
},
208215
'breadcrumb': []
209216
}
210217
]
211218
),
212-
(
219+
( # test_number=10
213220
{
214221
'schema': None,
215222
'key_properties': None,
@@ -219,14 +226,13 @@ def test_standard_metadata(self):
219226
[
220227
{
221228
'metadata': {
222-
'inclusion': 'available',
223229
'forced-replication-method': 'INCREMENTAL'
224230
},
225231
'breadcrumb': []
226232
}
227233
]
228234
),
229-
(
235+
( # test_number=11
230236
{
231237
'schema': None,
232238
'key_properties': None,
@@ -236,15 +242,14 @@ def test_standard_metadata(self):
236242
[
237243
{
238244
'metadata': {
239-
'inclusion': 'available',
240245
'forced-replication-method': 'INCREMENTAL',
241-
'valid_replication_keys': ['id','created']
246+
'valid-replication-keys': ['id','created']
242247
},
243248
'breadcrumb': []
244249
}
245250
]
246251
),
247-
(
252+
( # test_number=12
248253
{
249254
'schema': None,
250255
'key_properties': test_kp,
@@ -254,14 +259,13 @@ def test_standard_metadata(self):
254259
[
255260
{
256261
'metadata': {
257-
'inclusion': 'available',
258262
'table-key-properties': ['id'],
259263
},
260264
'breadcrumb': []
261265
}
262266
]
263267
),
264-
(
268+
( # test_number=13
265269
{
266270
'schema': None,
267271
'key_properties': test_kp,
@@ -271,15 +275,31 @@ def test_standard_metadata(self):
271275
[
272276
{
273277
'metadata': {
274-
'inclusion': 'available',
275278
'table-key-properties': ['id'],
276-
'valid_replication_keys': ['id','created']
279+
'valid-replication-keys': ['id','created']
280+
},
281+
'breadcrumb': []
282+
}
283+
]
284+
),
285+
( # test_number=14
286+
{
287+
'schema': None,
288+
'key_properties': test_kp,
289+
'replication_method': test_rm,
290+
'valid_replication_keys': None
291+
},
292+
[
293+
{
294+
'metadata': {
295+
'table-key-properties': ['id'],
296+
'forced-replication-method': 'INCREMENTAL',
277297
},
278298
'breadcrumb': []
279299
}
280300
]
281301
),
282-
(
302+
( # test_number=15
283303
{
284304
'schema': None,
285305
'key_properties': test_kp,
@@ -289,26 +309,26 @@ def test_standard_metadata(self):
289309
[
290310
{
291311
'metadata': {
292-
'inclusion': 'available',
293312
'table-key-properties': ['id'],
294313
'forced-replication-method': 'INCREMENTAL',
295-
'valid_replication_keys': ['id','created']
314+
'valid-replication-keys': ['id','created']
296315
},
297316
'breadcrumb': []
298317
}
299318
]
300319
)
301320
]
302321

303-
for var in test_variables:
304-
function_params = var[0]
305-
expected_metadata = var[1]
322+
for i, var in enumerate(test_variables):
323+
with self.subTest(test_number=i):
324+
function_params = var[0]
325+
expected_metadata = var[1]
306326

307-
test_value = get_standard_metadata(**function_params)
327+
test_value = get_standard_metadata(**function_params)
308328

309-
for obj in expected_metadata:
310-
if obj in test_value:
311-
self.assertIn(obj, test_value)
329+
expected_value = to_map(expected_metadata)
330+
actual_value = to_map(test_value)
331+
self.assertDictEqual(expected_value, actual_value)
312332

313333
# Test one function call where the parameters are not splat in
314334
test_value = get_standard_metadata(test_schema,
@@ -320,11 +340,13 @@ def test_standard_metadata(self):
320340
expected_metadata = make_expected_metadata(schema_present_base_obj,
321341
{'table-key-properties': ['id'],
322342
'forced-replication-method': 'INCREMENTAL',
323-
'valid_replication_keys': ['id','created'],
324-
'schema-name':tap_stream_id})
325-
for obj in expected_metadata:
326-
if obj in test_value:
327-
self.assertIn(obj, test_value)
343+
'valid-replication-keys': ['id','created'],
344+
'schema-name':tap_stream_id},
345+
has_pk=True)
346+
self.assertDictEqual(
347+
to_map(expected_metadata),
348+
to_map(test_value)
349+
)
328350

329351
def test_empty_key_properties_are_written(self):
330352
mdata = get_standard_metadata(key_properties=[])

0 commit comments

Comments
 (0)