Merge pull request #79 from moj-analytical-services/write-glue-spec

Write glue spec

Author: isichei

CHANGELOG.md

@@ -4,6 +4,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## v2.2.0
+### Added
+- Fixed bug where glue_specific would not write to json or be a key in dictionary from TableMeta class `to_dict()` method.
+- Fixed bug where default table ddl templates would be overwritten causing mixed table definitions (see issue no. 80) for specific example and fix.
+- If meta has partition property if none or empty list then this property will no longer be passed to dict (and therefore not to json)
+- If meta has glue_specific property if none or empty dict then this property will no longer be passed to dict (and therefore not to json)
+
 ## v2.1.2
 ### Added
 - DatabaseMeta method function `test_column_types_align` now tests that all column types match across all tables in database object.

MANIFEST.in

@@ -3,7 +3,7 @@ include etl_manager/specs/glue_job.json
 include etl_manager/specs/base.json
 include etl_manager/specs/orc_specific.json
 include etl_manager/specs/csv_quoted_nodate_specific.json
-include etl_manager/specs/par_specific.json
+include etl_manager/specs/parquet_specific.json
 include etl_manager/specs/csv_specific.json
 include etl_manager/specs/json_specific.json
 include etl_manager/specs/regex_specific.json

etl_manager/meta.py

@@ -12,7 +12,7 @@
     _s3_resource,
     _remove_final_slash
 )
-from copy import copy
+import copy
 import string
 import json
 import os
@@ -27,8 +27,7 @@
     "csv_quoted_nodate":  json.load(pkg_resources.resource_stream(__name__, "specs/csv_quoted_nodate_specific.json")),
     "regex":  json.load(pkg_resources.resource_stream(__name__, "specs/regex_specific.json")),
     "orc":  json.load(pkg_resources.resource_stream(__name__, "specs/orc_specific.json")),
-    "par":  json.load(pkg_resources.resource_stream(__name__, "specs/par_specific.json")),
-    "parquet":  json.load(pkg_resources.resource_stream(__name__, "specs/par_specific.json")),
+    "parquet":  json.load(pkg_resources.resource_stream(__name__, "specs/parquet_specific.json")),
     "json": json.load(pkg_resources.resource_stream(__name__, "specs/json_specific.json"))
 }
 
@@ -47,7 +46,7 @@ def _get_spec(spec_name) :
     if spec_name not in _template :
         raise ValueError("spec_name/data_type requested ({}) is not a valid spec/data_type".format(spec_name))
 
-    return copy(_template[spec_name])
+    return copy.deepcopy(_template[spec_name])
 
 class TableMeta :
     """
@@ -273,9 +272,14 @@ def to_dict(self) :
             "description" : self.description,
             "data_format" : self.data_format,
             "columns" : self.columns,
-            "partitions" : self.partitions,
-            "location" : self.location
+            "location" : self.location,
         }
+        if bool(self.partitions) :
+            meta['partitions'] = self.partitions
+
+        if bool(self.glue_specific) :
+            meta['glue_specific'] = self.glue_specific
+            
         return meta
 
     def write_to_json(self, file_path) :

etl_manager/specs/par_specific.json etl_manager/specs/parquet_specific.jsonetl_manager/specs/par_specific.json renamed to etl_manager/specs/parquet_specific.json

@@ -65,6 +65,10 @@
         "examples": [
           "sop_full/"
         ]
+      },
+      "glue_specific" : {
+        "type" : ["null", "object"],
+        "title": "Dict used to add any additional table properties for glue catalogue. For an example see here: https://github.com/moj-analytical-services/etl_manager/blob/master/example/meta_data/db1/pay.json#L19"
       }
     }
   }

etl_manager/specs/table_schema.json

@@ -2,7 +2,7 @@
 
 setup(
     name='etl_manager',
-    version='2.1.2',
+    version='2.2.0',
     packages=find_packages(exclude=['tests*']),
     license='MIT',
     description='A python package to manage etl processes on AWS',

setup.py

@@ -5,7 +5,7 @@
 """
 
 import unittest
-from etl_manager.meta import DatabaseMeta, TableMeta, read_database_folder, read_table_json, _agnostic_to_glue_spark_dict, MetaColumnTypeMismatch
+from etl_manager.meta import DatabaseMeta, TableMeta, read_database_folder, read_table_json, _agnostic_to_glue_spark_dict, MetaColumnTypeMismatch, _get_spec
 from etl_manager.utils import _end_with_slash, _validate_string, _glue_client, read_json, _remove_final_slash
 from etl_manager.etl import GlueJob
 import boto3
@@ -158,6 +158,16 @@ def test_table_to_dict(self) :
 
         self.assertDictEqual(test_dict, expected_dict)
 
+        # Test file with glue specific
+        expected_dict2 = read_json('example/meta_data/db1/pay.json')
+        test_dict2 = db.table('pay').to_dict()
+        
+        # Null out schema as may need changing when on branch but still need to unit test
+        expected_dict2["$schema"] = ''
+        test_dict2["$schema"] = ''
+
+        self.assertDictEqual(test_dict2, expected_dict2)
+
     def test_db_table_names(self) :
         db = read_database_folder('example/meta_data/db1/')
         t = all(t in ['teams', 'employees', 'pay'] for t in db.table_names)
@@ -189,6 +199,13 @@ def test_glue_specific_table(self):
         glue_def = t.glue_table_definition("db_path")
         self.assertTrue(t.glue_table_definition("db_path")["Parameters"]['skip.header.line.count'] == '1')
 
+    def test_glue_table_definition_doesnt_overwrite_base_spec(self) :
+        expected_dict = _get_spec('base')
+        db = read_database_folder('example/meta_data/db1/')
+        glue_def_dump = db.table('pay').glue_table_definition()
+
+        self.assertDictEqual(expected_dict, _get_spec('base'))
+        
     def test_add_remove_table(self) :
         db = read_database_folder('example/meta_data/db1/')
         self.assertRaises(ValueError, db.remove_table, 'not_a_table')