Skip to content

Commit 3eec8f8

Browse files
joaquimsnjoaquimsn
committed
build: 🏗️ terraform improvements and CI
- Update Terraforma sintaxe - Organized terraform file structure - Added new variable with default value to replace static parameter in main.tf - deploy megalista metadata to bucket with terraform - Create Git Action to validate terrafom files
1 parent 7598763 commit 3eec8f8

File tree

5 files changed

+123
-53
lines changed

5 files changed

+123
-53
lines changed

‎.github/workflows/terraform.yml‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
name: Terraform Validate
2+
3+
on: ['push']
4+
5+
jobs:
6+
terraform-actions:
7+
name: Workflow
8+
runs-on: ubuntu-latest
9+
defaults:
10+
run:
11+
working-directory: ./terraform
12+
steps:
13+
- name: Checkout Repository
14+
uses: actions/checkout@master
15+
16+
- name: HashiCorp - Setup Terraform
17+
uses: hashicorp/[email protected]
18+
with:
19+
terraform_version: 0.14.6
20+
21+
- name: Terraform Init
22+
id: init
23+
run: terraform init
24+
continue-on-error: true
25+
26+
- name: Terraform Fmt
27+
id: fmt
28+
run: terraform fmt -check -diff
29+
continue-on-error: true
30+
31+
- name: Terraform Validate
32+
id: validate
33+
run: terraform validate -no-color
34+
continue-on-error: false

‎terraform/external.tf‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#create detaflow metadata
2+
resource "null_resource" "bucket_megalista_metadata" {
3+
provisioner "local-exec" {
4+
command = "sh ./scripts/deploy_cloud.sh ${data.google_client_config.current.project} ${var.bucket_name} ${var.region}"
5+
}
6+
7+
depends_on = [google_storage_bucket.my_storage]
8+
}

‎terraform/main.tf‎

Lines changed: 10 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -1,52 +1,13 @@
1-
variable "bucket_name" {
2-
type = string
3-
description = "Google Cloud Storage Bucket to create"
4-
}
5-
6-
variable "bq_ops_dataset" {
7-
type = string
8-
description = "Auxliary bigquery dataset for Megalista operations to create"
9-
}
10-
11-
variable "developer_token" {
12-
type = string
13-
description = "Google Ads developer Token"
14-
}
15-
16-
variable "client_id" {
17-
type = string
18-
description = "OAuth Client Id"
19-
}
20-
21-
variable "client_secret" {
22-
type = string
23-
description = "OAuth Client Secret"
24-
}
25-
26-
variable "access_token" {
27-
type = string
28-
description = "Access Token"
29-
}
30-
31-
variable "refresh_token" {
32-
type = string
33-
description = "Refresh Token"
34-
}
35-
36-
variable "setup_sheet_id" {
37-
type = string
38-
description = "Setup Sheet Id"
39-
}
40-
411
data "google_client_config" "current" {
422
}
433

444
data "google_client_openid_userinfo" "me" {
455
}
466

477
resource "google_bigquery_dataset" "dataset" {
48-
dataset_id = "${var.bq_ops_dataset}"
49-
location = "US"
8+
dataset_id = var.bq_ops_dataset
9+
location = var.location
10+
description = "Auxliary bigquery dataset for Megalista operations to create"
5011
delete_contents_on_destroy = true
5112
}
5213

@@ -56,7 +17,7 @@ locals {
5617
"jobName": "megalist_daily",
5718
"parameters": {
5819
"developer_token": "${var.developer_token}",
59-
"client_id": "${var.client_id}",
20+
"client_id": "${var.client_id},
6021
"client_secret": "${var.client_secret}",
6122
"access_token": "${var.access_token}",
6223
"refresh_token": "${var.refresh_token}",
@@ -65,17 +26,17 @@ locals {
6526
},
6627
"environment": {
6728
"tempLocation": "gs://${var.bucket_name}/tmp",
68-
"zone": "us-central1-f"
29+
"zone": "${var.zone}"
6930
}
7031
}
7132
EOF
7233
}
7334

7435
resource "google_storage_bucket" "my_storage" {
7536
name = var.bucket_name
76-
location = "US"
37+
location = var.location
7738
force_destroy = true
78-
bucket_policy_only = true
39+
uniform_bucket_level_access = true
7940
}
8041

8142
resource "google_project_iam_member" "cloudscheduler-creator" {
@@ -160,13 +121,13 @@ resource "google_cloud_scheduler_job" "megalista_job" {
160121
name = "megalista_job"
161122
description = "Daily Runner for Megalista"
162123
schedule = "0 0 * * *"
163-
time_zone = "America/Sao_Paulo"
124+
time_zone = local.time_zone
164125
attempt_deadline = "320s"
165-
region = "us-central1"
126+
region = var.region
166127

167128
http_target {
168129
http_method = "POST"
169-
uri = "https://dataflow.googleapis.com/v1b3/projects/${data.google_client_config.current.project}/templates:launch?gcsPath=gs://${var.bucket_name}/templates/megalist"
130+
uri = "https://dataflow.googleapis.com/v1b3/projects/${data.google_client_config.current.project}/templates:launch?gcsPath=gs://${var.bucket_name}/templates/megalista"
170131
body = base64encode(local.scheduler_body)
171132
oauth_token {
172133
service_account_email = google_service_account.sa.email

‎deploy_cloud.sh‎ renamed to ‎terraform/scripts/deploy_cloud.sh‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,14 @@ if [ $# != 3 ]; then
1919
exit 1
2020
fi
2121

22+
echo "Move to megalist_dataflow folder"
23+
cd ..
2224
cd megalist_dataflow
23-
gcloud config set project $1
25+
echo "Configuration GCP project in gcloud"
26+
gcloud config set project "$1"
27+
echo "Build Dataflow metadata"
2428
python3 -m pip install --user -q -r requirements.txt
25-
python3 -m main --runner DataflowRunner --project $1 --gcp_project_id $1 --temp_location gs://$2/tmp/ --region $3 --setup_file ./setup.py --template_location gs://$2/templates/megalist --num_workers 1 --autoscaling_algorithm=NONE
26-
gsutil cp megalist_metadata gs://$2/templates/megalist_metadata
27-
cd ..
29+
python3 -m main --runner DataflowRunner --project "$1" --gcp_project_id "$1" --temp_location"gs://$2/tmp/" --region "$3" --setup_file ./setup.py --template_location "gs://$2/templates/megalista" --num_workers 1 --autoscaling_algorithm=NONE
30+
echo "Copy megalista_medata to bucket $2"
31+
gsutil cp megalist_metadata "gs://$2/templates/megalista_metadata"
32+
cd ..

‎terraform/variables.tf‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
locals {
2+
time_zone = "America/Sao_Paulo"
3+
}
4+
5+
variable "bucket_name" {
6+
type = string
7+
description = "Google Cloud Storage Bucket to create"
8+
}
9+
10+
variable "bq_ops_dataset" {
11+
type = string
12+
description = "Auxliary bigquery dataset for Megalista operations to create"
13+
}
14+
15+
variable "developer_token" {
16+
type = string
17+
description = "Google Ads developer Token"
18+
}
19+
20+
variable "client_id" {
21+
type = string
22+
description = "OAuth Client Id"
23+
}
24+
25+
variable "client_secret" {
26+
type = string
27+
description = "OAuth Client Secret"
28+
}
29+
30+
variable "access_token" {
31+
type = string
32+
description = "Access Token"
33+
}
34+
35+
variable "refresh_token" {
36+
type = string
37+
description = "Refresh Token"
38+
}
39+
40+
variable "setup_sheet_id" {
41+
type = string
42+
description = "Setup Sheet Id"
43+
}
44+
45+
variable "location" {
46+
type = string
47+
description = "GCP location https://cloud.google.com/compute/docs/regions-zones?hl=pt-br default us"
48+
default = "us"
49+
}
50+
51+
variable "region" {
52+
type = string
53+
description = "GCP region https://cloud.google.com/compute/docs/regions-zones?hl=pt-br default us-central1"
54+
default = "us-central1"
55+
}
56+
57+
variable "zone" {
58+
type = string
59+
description = "GCP zone https://cloud.google.com/compute/docs/regions-zones?hl=pt-br default us-central1"
60+
default = "us-central1-f"
61+
}
62+

0 commit comments

Comments
 (0)