Skip to content

Commit 7c25d8e

Browse files
Tian-2017Tian-2017
authored
Remove the abandoned modules - g-sheet ingestion (#2040)
* dpf-226 remove the testing modules * based on Daro's message, these are not in use anymore
1 parent 1391d23 commit 7c25d8e

8 files changed

+0
-504
lines changed

‎terraform/etl/08-google-sheets-imports.tf‎

Lines changed: 0 additions & 198 deletions
Original file line numberDiff line numberDiff line change
@@ -1,178 +1,3 @@
1-
module "repairs_dlo" {
2-
count = local.is_live_environment ? 1 : 0
3-
4-
source = "../modules/google-sheets-glue-job"
5-
is_production_environment = local.is_production_environment
6-
identifier_prefix = local.short_identifier_prefix
7-
is_live_environment = local.is_live_environment
8-
glue_scripts_bucket_id = module.glue_scripts_data_source.bucket_id
9-
helper_module_key = data.aws_s3_object.helpers.key
10-
pydeequ_zip_key = data.aws_s3_object.pydeequ.key
11-
glue_catalog_database_name = module.department_housing_repairs_data_source.raw_zone_catalog_database_name
12-
glue_temp_storage_bucket_url = module.glue_temp_storage_data_source.bucket_url
13-
glue_crawler_excluded_blobs = local.glue_crawler_excluded_blobs
14-
google_sheets_import_script_key = aws_s3_object.google_sheets_import_script.key
15-
bucket_id = module.raw_zone_data_source.bucket_id
16-
sheets_credentials_name = data.aws_ssm_parameter.sheets_credentials_housing_name.value
17-
google_sheets_document_id = "1i9q42Kkbugwi4f2S4zdyid2ZjoN1XLjuYvqYqfHyygs"
18-
google_sheets_worksheet_name = "Form responses 1"
19-
department = module.department_housing_repairs_data_source
20-
dataset_name = "repairs-dlo"
21-
enable_glue_trigger = false
22-
create_workflow = true
23-
spark_ui_output_storage_id = module.spark_ui_output_storage_data_source.bucket_id
24-
}
25-
26-
module "repairs_herts_heritage" {
27-
count = local.is_live_environment ? 1 : 0
28-
29-
source = "../modules/google-sheets-glue-job"
30-
is_production_environment = local.is_production_environment
31-
identifier_prefix = local.short_identifier_prefix
32-
is_live_environment = local.is_live_environment
33-
glue_scripts_bucket_id = module.glue_scripts_data_source.bucket_id
34-
helper_module_key = data.aws_s3_object.helpers.key
35-
pydeequ_zip_key = data.aws_s3_object.pydeequ.key
36-
glue_catalog_database_name = module.department_housing_repairs_data_source.raw_zone_catalog_database_name
37-
glue_temp_storage_bucket_url = module.glue_temp_storage_data_source.bucket_url
38-
glue_crawler_excluded_blobs = local.glue_crawler_excluded_blobs
39-
google_sheets_import_script_key = aws_s3_object.google_sheets_import_script.key
40-
bucket_id = module.raw_zone_data_source.bucket_id
41-
sheets_credentials_name = data.aws_ssm_parameter.sheets_credentials_housing_name.value
42-
google_sheets_document_id = "1Psw8i2qooASPLjaBfGKNX7upiX7BeQSiMeJ8dQngSJI"
43-
google_sheets_worksheet_name = "Form responses 1"
44-
department = module.department_housing_repairs_data_source
45-
dataset_name = "repairs-herts-heritage"
46-
enable_glue_trigger = false
47-
create_workflow = true
48-
spark_ui_output_storage_id = module.spark_ui_output_storage_data_source.bucket_id
49-
}
50-
51-
module "repairs_avonline" {
52-
count = local.is_live_environment ? 1 : 0
53-
54-
source = "../modules/google-sheets-glue-job"
55-
is_production_environment = local.is_production_environment
56-
identifier_prefix = local.short_identifier_prefix
57-
is_live_environment = local.is_live_environment
58-
glue_scripts_bucket_id = module.glue_scripts_data_source.bucket_id
59-
helper_module_key = data.aws_s3_object.helpers.key
60-
pydeequ_zip_key = data.aws_s3_object.pydeequ.key
61-
glue_catalog_database_name = module.department_housing_repairs_data_source.raw_zone_catalog_database_name
62-
glue_temp_storage_bucket_url = module.glue_temp_storage_data_source.bucket_url
63-
glue_crawler_excluded_blobs = local.glue_crawler_excluded_blobs
64-
google_sheets_import_script_key = aws_s3_object.google_sheets_import_script.key
65-
bucket_id = module.raw_zone_data_source.bucket_id
66-
sheets_credentials_name = data.aws_ssm_parameter.sheets_credentials_housing_name.value
67-
google_sheets_document_id = "1nM99bPaOPvg5o_cz9_yJR6jlnMB0oSHdFhAMKQkPJi4"
68-
google_sheets_worksheet_name = "Form responses 1"
69-
department = module.department_housing_repairs_data_source
70-
dataset_name = "repairs-avonline"
71-
enable_glue_trigger = false
72-
create_workflow = true
73-
spark_ui_output_storage_id = module.spark_ui_output_storage_data_source.bucket_id
74-
}
75-
76-
module "repairs_alpha_track" {
77-
count = local.is_live_environment ? 1 : 0
78-
79-
source = "../modules/google-sheets-glue-job"
80-
is_production_environment = local.is_production_environment
81-
identifier_prefix = local.short_identifier_prefix
82-
is_live_environment = local.is_live_environment
83-
glue_scripts_bucket_id = module.glue_scripts_data_source.bucket_id
84-
helper_module_key = data.aws_s3_object.helpers.key
85-
pydeequ_zip_key = data.aws_s3_object.pydeequ.key
86-
glue_catalog_database_name = module.department_housing_repairs_data_source.raw_zone_catalog_database_name
87-
glue_temp_storage_bucket_url = module.glue_temp_storage_data_source.bucket_url
88-
glue_crawler_excluded_blobs = local.glue_crawler_excluded_blobs
89-
google_sheets_import_script_key = aws_s3_object.google_sheets_import_script.key
90-
bucket_id = module.raw_zone_data_source.bucket_id
91-
sheets_credentials_name = data.aws_ssm_parameter.sheets_credentials_housing_name.value
92-
google_sheets_document_id = "1cbeVvMuNNinVQDeVfsUWalRpY6zK9oZPa3ebLtLSiAc"
93-
google_sheets_worksheet_name = "Form responses 1"
94-
department = module.department_housing_repairs_data_source
95-
dataset_name = "repairs-alpha-track"
96-
enable_glue_trigger = false
97-
create_workflow = true
98-
spark_ui_output_storage_id = module.spark_ui_output_storage_data_source.bucket_id
99-
}
100-
101-
module "repairs_stannah" {
102-
count = local.is_live_environment ? 1 : 0
103-
104-
source = "../modules/google-sheets-glue-job"
105-
is_production_environment = local.is_production_environment
106-
identifier_prefix = local.short_identifier_prefix
107-
is_live_environment = local.is_live_environment
108-
glue_scripts_bucket_id = module.glue_scripts_data_source.bucket_id
109-
helper_module_key = data.aws_s3_object.helpers.key
110-
pydeequ_zip_key = data.aws_s3_object.pydeequ.key
111-
glue_catalog_database_name = module.department_housing_repairs_data_source.raw_zone_catalog_database_name
112-
glue_temp_storage_bucket_url = module.glue_temp_storage_data_source.bucket_url
113-
glue_crawler_excluded_blobs = local.glue_crawler_excluded_blobs
114-
google_sheets_import_script_key = aws_s3_object.google_sheets_import_script.key
115-
bucket_id = module.raw_zone_data_source.bucket_id
116-
sheets_credentials_name = data.aws_ssm_parameter.sheets_credentials_housing_name.value
117-
google_sheets_document_id = "1CpC_Dn4aM8MSFb5a6HJ_FEsVYcahRsis9YIATcfArhw"
118-
google_sheets_worksheet_name = "Form responses 1"
119-
department = module.department_housing_repairs_data_source
120-
dataset_name = "repairs-stannah"
121-
enable_glue_trigger = false
122-
create_workflow = true
123-
spark_ui_output_storage_id = module.spark_ui_output_storage_data_source.bucket_id
124-
}
125-
126-
module "repairs_purdy" {
127-
count = local.is_live_environment ? 1 : 0
128-
129-
source = "../modules/google-sheets-glue-job"
130-
is_production_environment = local.is_production_environment
131-
identifier_prefix = local.short_identifier_prefix
132-
is_live_environment = local.is_live_environment
133-
glue_scripts_bucket_id = module.glue_scripts_data_source.bucket_id
134-
helper_module_key = data.aws_s3_object.helpers.key
135-
pydeequ_zip_key = data.aws_s3_object.pydeequ.key
136-
glue_catalog_database_name = module.department_housing_repairs_data_source.raw_zone_catalog_database_name
137-
glue_temp_storage_bucket_url = module.glue_temp_storage_data_source.bucket_url
138-
glue_crawler_excluded_blobs = local.glue_crawler_excluded_blobs
139-
google_sheets_import_script_key = aws_s3_object.google_sheets_import_script.key
140-
bucket_id = module.raw_zone_data_source.bucket_id
141-
sheets_credentials_name = data.aws_ssm_parameter.sheets_credentials_housing_name.value
142-
google_sheets_document_id = "1-PpKPnaPMA6AogsNXT5seqQk3VUB-naFnFJYhROkl2o"
143-
google_sheets_worksheet_name = "FormresponsesPUR"
144-
department = module.department_housing_repairs_data_source
145-
dataset_name = "repairs-purdy"
146-
enable_glue_trigger = false
147-
create_workflow = true
148-
spark_ui_output_storage_id = module.spark_ui_output_storage_data_source.bucket_id
149-
}
150-
151-
module "repairs_axis" {
152-
count = local.is_live_environment ? 1 : 0
153-
154-
source = "../modules/google-sheets-glue-job"
155-
is_production_environment = local.is_production_environment
156-
identifier_prefix = local.short_identifier_prefix
157-
is_live_environment = local.is_live_environment
158-
glue_scripts_bucket_id = module.glue_scripts_data_source.bucket_id
159-
helper_module_key = data.aws_s3_object.helpers.key
160-
pydeequ_zip_key = data.aws_s3_object.pydeequ.key
161-
glue_catalog_database_name = module.department_housing_repairs_data_source.raw_zone_catalog_database_name
162-
glue_temp_storage_bucket_url = module.glue_temp_storage_data_source.bucket_url
163-
glue_crawler_excluded_blobs = local.glue_crawler_excluded_blobs
164-
google_sheets_import_script_key = aws_s3_object.google_sheets_import_script.key
165-
bucket_id = module.raw_zone_data_source.bucket_id
166-
sheets_credentials_name = data.aws_ssm_parameter.sheets_credentials_housing_name.value
167-
google_sheets_document_id = "1aDWO9ZAVar377jiYTXkZzDCIckCqbhppOW23B85hFsA"
168-
google_sheets_worksheet_name = "Form responses 1"
169-
department = module.department_housing_repairs_data_source
170-
dataset_name = "repairs-axis"
171-
enable_glue_trigger = false
172-
create_workflow = true
173-
spark_ui_output_storage_id = module.spark_ui_output_storage_data_source.bucket_id
174-
}
175-
1761
module "parking_spreadsheet_estate_permit_limits" {
1772
count = local.is_live_environment ? 1 : 0
1783
source = "../modules/google-sheets-glue-job"
@@ -217,29 +42,6 @@ module "parking_spreadsheet_parkmap_restrictions_report" {
21742
spark_ui_output_storage_id = module.spark_ui_output_storage_data_source.bucket_id
21843
}
21944

220-
module "sandbox_estates_round_crew_data" {
221-
count = local.is_live_environment ? 1 : 0
222-
223-
source = "../modules/google-sheets-glue-job"
224-
is_production_environment = local.is_production_environment
225-
identifier_prefix = local.short_identifier_prefix
226-
is_live_environment = local.is_live_environment
227-
glue_scripts_bucket_id = module.glue_scripts_data_source.bucket_id
228-
helper_module_key = data.aws_s3_object.helpers.key
229-
pydeequ_zip_key = data.aws_s3_object.pydeequ.key
230-
glue_catalog_database_name = module.department_sandbox_data_source.raw_zone_catalog_database_name
231-
glue_temp_storage_bucket_url = module.glue_temp_storage_data_source.bucket_url
232-
glue_crawler_excluded_blobs = local.glue_crawler_excluded_blobs
233-
google_sheets_import_script_key = aws_s3_object.google_sheets_import_script.key
234-
bucket_id = module.raw_zone_data_source.bucket_id
235-
google_sheets_document_id = "1C5Afb_4qz2_7m7xPXAW9g40nWDt1jrdla3TBARAlCEM"
236-
google_sheets_worksheet_name = "EstatesRoundCrewData_07032022"
237-
department = module.department_sandbox_data_source
238-
dataset_name = "estate_round_crew_data"
239-
enable_glue_trigger = false
240-
spark_ui_output_storage_id = module.spark_ui_output_storage_data_source.bucket_id
241-
}
242-
24345
module "housing_patches" {
24446
count = local.is_live_environment ? 1 : 0
24547
source = "../modules/google-sheets-glue-job"

‎terraform/etl/26-aws-glue-job-repairs-avonline.tf‎

Lines changed: 0 additions & 30 deletions
This file was deleted.

‎terraform/etl/27-aws-glue-job-repairs-axis.tf‎

Lines changed: 0 additions & 30 deletions
This file was deleted.

0 commit comments

Comments
 (0)