refactor: Extract SQL from source code to its own files. (#171)

* refactor: extract sql files

* refactor: Change Dataset to follow folder structure convention

Co-authored-by: ijemmy

Author: ijemmy

core/.projen/tasks.json

@@ -12,7 +12,7 @@ import { LambdaInvoke, AthenaStartQueryExecution } from '@aws-cdk/aws-stepfuncti
 import { Construct, Arn, Aws, Stack, Duration, ArnFormat } from '@aws-cdk/core';
 import { AwsCustomResource, AwsCustomResourcePolicy, PhysicalResourceId } from '@aws-cdk/custom-resources';
 import { PreBundledFunction } from '../common/pre-bundled-function';
-import { Dataset } from '../dataset';
+import { Dataset } from '../datasets/dataset';
 import { SingletonBucket } from '../singleton-bucket';
 import { SingletonGlueDatabase } from '../singleton-glue-database';
 import { SynchronousAthenaQuery } from '../synchronous-athena-query';

core/src/data-generator/data-generator.ts

@@ -2,14 +2,14 @@
 // SPDX-License-Identifier: MIT-0
 
 import { Location } from '@aws-cdk/aws-s3';
-import { retailCustomerCreate, retailCustomerCreateTarget, retailCustomerGenerate } from './datasets/retail-customer';
-import { retailCustomerAddressCreate, retailCustomerAddressCreateTarget, retailCustomerAddressGenerate } from './datasets/retail-customer-address';
-import { retailItemCreate, retailItemGenerate } from './datasets/retail-item';
-import { retailPromoCreate, retailPromoGenerate } from './datasets/retail-promo';
-import { retailStoreCreate, retailStoreGenerate } from './datasets/retail-store';
-import { retailStoreSaleCreate, retailStoreSaleGenerate } from './datasets/retail-store-sale';
-import { retailWarehouseCreate, retailWarehouseGenerate } from './datasets/retail-warehouse';
-import { retailWebSaleCreate, retailWebSaleGenerate } from './datasets/retail-web-sale';
+import { retailCustomerCreate, retailCustomerCreateTarget, retailCustomerGenerate } from './retail-customer';
+import { retailCustomerAddressCreate, retailCustomerAddressCreateTarget, retailCustomerAddressGenerate } from './retail-customer-address';
+import { retailItemCreate, retailItemGenerate } from './retail-item';
+import { retailPromoCreate, retailPromoGenerate } from './retail-promo';
+import { retailStoreCreate, retailStoreGenerate } from './retail-store';
+import { retailStoreSaleCreate, retailStoreSaleGenerate } from './retail-store-sale';
+import { retailWarehouseCreate, retailWarehouseGenerate } from './retail-warehouse';
+import { retailWebSaleCreate, retailWebSaleGenerate } from './retail-web-sale';
 
 export interface DatasetProps {
   /**

core/src/dataset.ts core/src/datasets/dataset.tscore/src/dataset.ts renamed to core/src/datasets/dataset.ts

@@ -0,0 +1 @@
+export { Dataset, DatasetProps } from './dataset';

core/src/datasets/index.ts

@@ -0,0 +1,9 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: MIT-0
+
+import { readFileSync } from 'fs';
+const ENCODING = 'utf8';
+
+export function readSqlFile(path: string) {
+  return readFileSync(path, ENCODING);
+}

core/src/datasets/read-sql-file.ts

@@ -0,0 +1,23 @@
+CREATE EXTERNAL TABLE IF NOT EXISTS {{DATABASE}}.{{TABLE}}(
+  address_id string,
+  city string,
+  county string,
+  state string,
+  zip string,
+  country string,
+  gmt_offset string,
+  location_type string,
+  street string,
+  address_datetime string
+)
+ROW FORMAT DELIMITED 
+  FIELDS TERMINATED BY ',' 
+STORED AS INPUTFORMAT 
+  'org.apache.hadoop.mapred.TextInputFormat' 
+OUTPUTFORMAT 
+  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
+LOCATION
+  's3://{{BUCKET}}/{{KEY}}/'
+TBLPROPERTIES (
+  'skip.header.line.count'='1'
+)

core/src/datasets/resources/retail-customer-address/create-target.sql

@@ -0,0 +1,18 @@
+CREATE EXTERNAL TABLE IF NOT EXISTS {{DATABASE}}.{{TABLE}}(
+  address_id string,
+  city string,
+  county string,
+  state string,
+  zip string,
+  country string,
+  gmt_offset string,
+  location_type string,
+  street string,
+  address_datetime string
+)
+ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
+LOCATION
+  's3://{{BUCKET}}/{{KEY}}/'
+TBLPROPERTIES (
+  'skip.header.line.count'='1'
+);

core/src/datasets/resources/retail-customer-address/create.sql

@@ -0,0 +1,16 @@
+INSERT INTO {{DATABASE}}.{{TARGET_TABLE}} (
+  SELECT
+    address_id,
+    city,
+    county,
+    state,
+    zip,
+    country,
+    gmt_offset,
+    location_type,
+    street,
+    to_iso8601(date_add('second', {{OFFSET}}, from_iso8601_timestamp(address_datetime))) as address_datetime
+  FROM {{DATABASE}}.{{SOURCE_TABLE}}
+  WHERE address_datetime
+    BETWEEN '{{MIN}}' AND '{{MAX}}'
+)

core/src/datasets/resources/retail-customer-address/generate.sql

@@ -0,0 +1,31 @@
+CREATE EXTERNAL TABLE IF NOT EXISTS {{DATABASE}}.{{TABLE}}(
+  customer_id string,
+  salutation string,
+  first_name string,
+  last_name string,
+  birth_country string,
+  email_address string,
+  birth_date string,
+  gender string,
+  marital_status string,
+  education_status string,
+  purchase_estimate bigint,
+  credit_rating string,
+  buy_potential string,
+  vehicle_count bigint,
+  lower_bound bigint,
+  upper_bound bigint,
+  address_id string,
+  customer_datetime string
+)
+ROW FORMAT DELIMITED 
+  FIELDS TERMINATED BY ',' 
+STORED AS INPUTFORMAT 
+  'org.apache.hadoop.mapred.TextInputFormat' 
+OUTPUTFORMAT 
+  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
+LOCATION
+  's3://{{BUCKET}}/{{KEY}}/'
+TBLPROPERTIES (
+  'skip.header.line.count'='1'
+)

core/src/datasets/resources/retail-customer/create-target.sql

@@ -0,0 +1,26 @@
+CREATE EXTERNAL TABLE IF NOT EXISTS {{DATABASE}}.{{TABLE}}(
+  customer_id string,
+  salutation string,
+  first_name string,
+  last_name string,
+  birth_country string,
+  email_address string,
+  birth_date string,
+  gender string,
+  marital_status string,
+  education_status string,
+  purchase_estimate bigint,
+  credit_rating string,
+  buy_potential string,
+  vehicle_count bigint,
+  lower_bound bigint,
+  upper_bound bigint,
+  address_id string,
+  customer_datetime string
+)
+ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
+LOCATION
+  's3://{{BUCKET}}/{{KEY}}/'
+TBLPROPERTIES (
+  'skip.header.line.count'='1'
+)