SAC-29354: Update _sdc_extra column schema to be nullable (#83)

skuttleman · dsprayberry · web-flow · commit e2e8c92d4a36 · 2025-10-23T11:11:27.000-04:00
* Update _sdc_extra column schema to be nullable

Co-authored-by: Dylan Sprayberry &lt;dylan.sprayberry@qlik.com&gt;

* Fix test

Co-authored-by: Dylan Sprayberry &lt;dylan.sprayberry@qlik.com&gt;

* Fix integration tests

Co-authored-by: Dylan Sprayberry &lt;dylan.sprayberry@qlik.com&gt;

---------

Co-authored-by: Dylan Sprayberry &lt;dylan.sprayberry@qlik.com&gt;
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,8 @@
 # Changelog
 
+## 2.2.1
+  * Fixes schema for _sdc_extra column [#83](https://github.com/singer-io/tap-s3-csv/pull/83)
+
 ## 2.2.0
   * Refactors JSONL support to use singer-encodings library [#82](https://github.com/singer-io/tap-s3-csv/pull/82)
 
diff --git a/setup.py b/setup.py
@@ -3,7 +3,7 @@
 from setuptools import setup
 
 setup(name='tap-s3-csv',
-      version='2.2.0',
+      version='2.2.1',
       description='Singer.io tap for extracting CSV files from S3',
       author='Stitch',
       url='https://singer.io',
diff --git a/tap_s3_csv/s3.py b/tap_s3_csv/s3.py
@@ -296,8 +296,10 @@ def get_sampled_schema_for_table(config, table_spec):
         SDC_SOURCE_BUCKET_COLUMN: {'type': 'string'},
         SDC_SOURCE_FILE_COLUMN: {'type': 'string'},
         SDC_SOURCE_LINENO_COLUMN: {'type': 'integer'},
-        SDC_EXTRA_COLUMN: {'type': 'array', 'items': {
-            'anyOf': [{'type': 'object', 'properties': {}}, {'type': 'string'}]}}
+        SDC_EXTRA_COLUMN: {
+            'type': ['null', 'array'],
+            'items': {'type': 'object', 'properties': {}}
+        }
     }
 
     data_schema = schema.generate_schema(samples)
diff --git a/tests/test_avro.py b/tests/test_avro.py
@@ -51,10 +51,8 @@ def test_run(self):
                 '_sdc_source_file': {'type': 'string', 'inclusion': 'available'},
                 '_sdc_source_lineno': {'type': 'integer', 'inclusion': 'available'},
                 '_sdc_extra': {
-                    'type': 'array',
-                    'items': {
-                        'anyOf': [{'type': 'object', 'properties': {}}, {'type': 'string'}]
-                    },
+                    'type': ['null', 'array'],
+                    'items': {'type': 'object', 'properties': {}},
                 'inclusion': 'available'}
             }
         }
diff --git a/tests/test_parquet.py b/tests/test_parquet.py
@@ -50,10 +50,8 @@ def test_run(self):
                 '_sdc_source_file': {'type': 'string', 'inclusion': 'available'},
                 '_sdc_source_lineno': {'type': 'integer', 'inclusion': 'available'},
                 '_sdc_extra': {
-                    'type': 'array',
-                    'items': {
-                        'anyOf': [{'type': 'object', 'properties': {}}, {'type': 'string'}]
-                    },
+                    'type': ['null', 'array'],
+                    'items': {'type': 'object', 'properties': {}},
                 'inclusion': 'available'}
             }
         }
diff --git a/tests/unittests/test_sampled_schema.py b/tests/unittests/test_sampled_schema.py
@@ -46,11 +46,8 @@ def test_schema_for_valid_samples(self, mock_valid_sample_files, mocked_get_inpu
                 '_sdc_source_file': {'type': 'string'},
                 '_sdc_source_lineno': {'type': 'integer'},
                 '_sdc_extra': {
-                    'type': 'array',
-                    'items': {
-                        'anyOf': [{'type': 'object', 'properties': {}},
-                                  {'type': 'string'}]
-                    }
+                    'type': ['null', 'array'],
+                    'items': {'type': 'object', 'properties': {}}
                 }
             }
         }

Original file line number	Diff line number	Diff line change
`@@ -51,10 +51,8 @@ def test_run(self):`
`51`	`51`	`'_sdc_source_file': {'type': 'string', 'inclusion': 'available'},`
`52`	`52`	`'_sdc_source_lineno': {'type': 'integer', 'inclusion': 'available'},`
`53`	`53`	`'_sdc_extra': {`
`54`		`- 'type': 'array',`
`55`		`- 'items': {`
`56`		`- 'anyOf': [{'type': 'object', 'properties': {}}, {'type': 'string'}]`
`57`		`- },`
	`54`	`+ 'type': ['null', 'array'],`
	`55`	`+ 'items': {'type': 'object', 'properties': {}},`
`58`	`56`	`'inclusion': 'available'}`
`59`	`57`	`}`
`60`	`58`	`}`
Original file line number	Diff line number	Diff line change
`@@ -50,10 +50,8 @@ def test_run(self):`
`50`	`50`	`'_sdc_source_file': {'type': 'string', 'inclusion': 'available'},`
`51`	`51`	`'_sdc_source_lineno': {'type': 'integer', 'inclusion': 'available'},`
`52`	`52`	`'_sdc_extra': {`
`53`		`- 'type': 'array',`
`54`		`- 'items': {`
`55`		`- 'anyOf': [{'type': 'object', 'properties': {}}, {'type': 'string'}]`
`56`		`- },`
	`53`	`+ 'type': ['null', 'array'],`
	`54`	`+ 'items': {'type': 'object', 'properties': {}},`
`57`	`55`	`'inclusion': 'available'}`
`58`	`56`	`}`
`59`	`57`	`}`
Original file line number	Diff line number	Diff line change
`@@ -46,11 +46,8 @@ def test_schema_for_valid_samples(self, mock_valid_sample_files, mocked_get_inpu`
`46`	`46`	`'_sdc_source_file': {'type': 'string'},`
`47`	`47`	`'_sdc_source_lineno': {'type': 'integer'},`
`48`	`48`	`'_sdc_extra': {`
`49`		`- 'type': 'array',`
`50`		`- 'items': {`
`51`		`- 'anyOf': [{'type': 'object', 'properties': {}},`
`52`		`- {'type': 'string'}]`
`53`		`- }`
	`49`	`+ 'type': ['null', 'array'],`
	`50`	`+ 'items': {'type': 'object', 'properties': {}}`
`54`	`51`	`}`
`55`	`52`	`}`
`56`	`53`	`}`