Skip to content

Commit 545bb7c

Browse files
jpraynaudjpraynaud
authored
Merge pull request #1222 from input-output-hk/jpraynaud/1200-use-compression-parameters-ci-cd
Use compression parameters in the terraform deployments
2 parents 5cf7cfa + 8e870be commit 545bb7c

File tree

9 files changed

+340
-282
lines changed

9 files changed

+340
-282
lines changed

‎.github/workflows/actions/deploy-terraform-infrastructure/action.yml‎

Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,191 @@
1+
name: deploy-terraform-infrastructure
2+
description: |
3+
Deploy the infrastructure of a Mithril network with terraform.
4+
inputs:
5+
dry_run:
6+
description: Dry run will apply the terraform infrastructure, just plan it.
7+
required: true
8+
default: "true"
9+
terraform_backend_bucket:
10+
description: terraform backend bucket used to store terraform state.
11+
required: true
12+
environment_prefix:
13+
description: Mithril network environment prefix.
14+
required: true
15+
environment:
16+
description: Mithril network environment name.
17+
required: true
18+
cardano_network:
19+
description: Cardano network name.
20+
required: true
21+
google_region:
22+
description: Google Cloud region name.
23+
required: true
24+
google_zone:
25+
description: Google Cloud zone name.
26+
required: true
27+
google_machine_type:
28+
description: Google Cloud VM name.
29+
required: true
30+
google_compute_instance_data_disk_size:
31+
description: Google Cloud attached data disk size in GB.
32+
required: true
33+
google_application_credentials:
34+
description: Google Cloud application credentials (service account).
35+
required: true
36+
mithril_api_domain:
37+
description: Mithril network api domain root.
38+
required: true
39+
mithril_image_id:
40+
description: Mithril Docker image id to deploy.
41+
required: true
42+
mithril_protocol_parameters:
43+
description: Mithril protocol parameters.
44+
required: true
45+
mithril_signers:
46+
description: Mithril signers settings.
47+
required: true
48+
mithril_genesis_secret_key:
49+
description: Mithril genesis secret key (only for test networks).
50+
required: false
51+
mithril_genesis_verification_key_url:
52+
description: Mithril genesis verification key location.
53+
required: true
54+
mithril_era_reader_address_url:
55+
description: Mithril era reader address location.
56+
required: true
57+
mithril_era_reader_verification_key_url:
58+
description: Mithril era reader verification key url.
59+
required: true
60+
mithril_era_reader_secret_key:
61+
description: Mithril era reader secret key (onlye for test networks).
62+
required: false
63+
mithril_aggregator_snapshot_compression_algorithm:
64+
description: Mithril aggregator snapshot compression algorithm.
65+
required: false
66+
mithril_aggregator_zstandard_parameters_level:
67+
description: Mithril aggregator snapshot zstandard compression level.
68+
required: false
69+
mithril_aggregator_zstandard_parameters_workers:
70+
description: Mithril aggregator snapshot zstandard number of workers.
71+
required: false
72+
prometheus_auth_username:
73+
description: Prometheus metrics endpoint username.
74+
required: false
75+
prometheus_auth_password:
76+
description: Prometheus metrics endpoint password.
77+
required: false
78+
prometheus_ingest_host:
79+
description: Prometheus ingester endpoint location.
80+
required: false
81+
prometheus_ingest_username:
82+
description: Prometheus ingester endpoint username.
83+
required: false
84+
prometheus_ingest_password:
85+
description: Prometheus ingester endpoint password.
86+
required: false
87+
loki_auth_username:
88+
description: Loki metrics endpoint username.
89+
required: false
90+
loki_auth_password:
91+
description: Loki metrics endpoint password.
92+
required: false
93+
loki_ingest_host:
94+
description: Loki ingester endpoint location.
95+
required: false
96+
loki_ingest_username:
97+
description: Loki ingester endpoint username.
98+
required: false
99+
loki_ingest_password:
100+
description: Loki ingester endpoint password.
101+
required: false
102+
103+
runs:
104+
using: "composite"
105+
steps:
106+
- name: Checkout sources
107+
uses: actions/checkout@v3
108+
109+
- name: Prepare service account credentials
110+
shell: bash
111+
working-directory: mithril-infra
112+
run: |
113+
echo '${{ inputs.google_application_credentials}}' > ./google-application-credentials.json
114+
chmod u+x ./assets/tools/utils/google-credentials-public-key.sh
115+
./assets/tools/utils/google-credentials-public-key.sh ./google-application-credentials.json ./assets/ssh_keys curry
116+
117+
- name: Prepare terraform variables
118+
shell: bash
119+
working-directory: mithril-infra
120+
run: |
121+
cat > ./env.variables.tfvars << EOF
122+
environment_prefix = "${{ inputs.environment_prefix }}"
123+
cardano_network = "${{ inputs.cardano_network }}"
124+
google_region = "${{ inputs.google_region }}"
125+
google_zone = "${{ inputs.google_zone }}"
126+
google_machine_type = "${{ inputs.google_machine_type }}"
127+
google_compute_instance_data_disk_size = "${{ inputs.google_compute_instance_data_disk_size }}"
128+
google_service_credentials_json_file = "./google-application-credentials.json"
129+
mithril_api_domain = "${{ inputs.mithril_api_domain }}"
130+
mithril_image_id = "${{ inputs.mithril_image_id }}"
131+
mithril_genesis_verification_key_url = "${{ inputs.mithril_genesis_verification_key_url }}"
132+
mithril_genesis_secret_key = "${{ inputs.mithril_genesis_secret_key }}"
133+
mithril_protocol_parameters = ${{ fromJSON(inputs.mithril_protocol_parameters) }}
134+
mithril_signers = ${{ fromJSON(inputs.mithril_signers) }}
135+
mithril_era_reader_adapter_type = "cardano-chain"
136+
mithril_era_reader_address_url = "${{ inputs.mithril_era_reader_address_url }}"
137+
mithril_era_reader_verification_key_url = "${{ inputs.mithril_era_reader_verification_key_url }}"
138+
mithril_era_reader_secret_key = "${{ inputs.mithril_era_reader_secret_key }}"
139+
mithril_aggregator_snapshot_compression_algorithm = "${{ inputs.mithril_aggregator_snapshot_compression_algorithm }}"
140+
mithril_aggregator_zstandard_parameters_level = "${{ inputs.mithril_aggregator_zstandard_parameters_level }}"
141+
mithril_aggregator_zstandard_parameters_workers = "${{ inputs.mithril_aggregator_zstandard_parameters_workers }}"
142+
prometheus_auth_username = "${{ inputs.prometheus_auth_username }}"
143+
prometheus_auth_password = "${{ inputs.prometheus_auth_password }}"
144+
prometheus_ingest_host = "${{ inputs.prometheus_ingest_host }}"
145+
prometheus_ingest_username = "${{ inputs.prometheus_ingest_username }}"
146+
prometheus_ingest_password = "${{ inputs.prometheus_ingest_password }}"
147+
loki_auth_username = "${{ inputs.loki_auth_username }}"
148+
loki_auth_password = "${{ inputs.loki_auth_password }}"
149+
loki_ingest_host = "${{ inputs.loki_ingest_host }}"
150+
loki_ingest_username = "${{ inputs.loki_ingest_username }}"
151+
loki_ingest_password = "${{ inputs.loki_ingest_password }}"
152+
EOF
153+
terraform fmt ./env.variables.tfvars
154+
cat ./env.variables.tfvars
155+
156+
- name: Setup Terraform
157+
uses: hashicorp/setup-terraform@v2
158+
with:
159+
terraform_wrapper: false
160+
161+
- name: Init Terraform
162+
shell: bash
163+
working-directory: mithril-infra
164+
run: |
165+
GOOGLE_APPLICATION_CREDENTIALS=./google-application-credentials.json terraform init -backend-config="bucket=${{ inputs.terraform_backend_bucket }}" -backend-config="prefix=terraform/mithril-${{ inputs.environment }}"
166+
167+
- name: Check Terraform
168+
shell: bash
169+
working-directory: mithril-infra
170+
run: terraform fmt -check
171+
172+
- name: Terraform Plan
173+
if: inputs.dry_run == 'true'
174+
shell: bash
175+
working-directory: mithril-infra
176+
run: |
177+
GOOGLE_APPLICATION_CREDENTIALS=./google-application-credentials.json terraform plan --var-file=./env.variables.tfvars
178+
179+
- name: Terraform Apply
180+
shell: bash
181+
working-directory: mithril-infra
182+
if: inputs.dry_run == 'false'
183+
run: |
184+
GOOGLE_APPLICATION_CREDENTIALS=./google-application-credentials.json terraform apply -auto-approve --var-file=./env.variables.tfvars
185+
186+
- name: Cleanup
187+
shell: bash
188+
working-directory: mithril-infra
189+
run: |
190+
rm -f ./env.variables.tfvars
191+
rm -f ./google-application-credentials.json

‎.github/workflows/ci.yml‎

Lines changed: 43 additions & 95 deletions
Original file line numberDiff line numberDiff line change
@@ -438,106 +438,54 @@ jobs:
438438
google_zone: europe-west1-b
439439
google_machine_type: e2-highmem-4
440440
google_compute_instance_data_disk_size: 250
441-
441+
environment: ${{ matrix.environment }}
442442
runs-on: ubuntu-22.04
443-
444443
needs:
445444
- docker-mithril
446-
447-
environment: ${{ matrix.environment }}
448-
449-
env:
450-
GOOGLE_APPLICATION_CREDENTIALS: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }}
451-
GENESIS_SECRET_KEY: ${{ secrets.GENESIS_SECRET_KEY }}
452-
GENESIS_VERIFICATION_KEY_URL: ${{ vars.GENESIS_VERIFICATION_KEY_URL }}
453-
ERA_READER_ADDRESS_URL: ${{ vars.ERA_READER_ADDRESS_URL }}
454-
ERA_READER_VERIFICATION_KEY_URL: ${{ vars.ERA_READER_VERIFICATION_KEY_URL }}
455-
ERA_READER_SECRET_KEY: ${{ secrets.ERA_READER_SECRET_KEY }}
456-
PROMETHEUS_AUTH_USERNAME: ${{ secrets.PROMETHEUS_AUTH_USERNAME }}
457-
PROMETHEUS_AUTH_PASSWORD: ${{ secrets.PROMETHEUS_AUTH_PASSWORD }}
458-
PROMETHEUS_INGEST_HOST: ${{ vars.PROMETHEUS_INGEST_HOST }}
459-
PROMETHEUS_INGEST_USERNAME: ${{ secrets.PROMETHEUS_INGEST_USERNAME }}
460-
PROMETHEUS_INGEST_PASSWORD: ${{ secrets.PROMETHEUS_INGEST_PASSWORD }}
461-
LOKI_AUTH_USERNAME: ${{ secrets.LOKI_AUTH_USERNAME }}
462-
LOKI_AUTH_PASSWORD: ${{ secrets.LOKI_AUTH_PASSWORD }}
463-
LOKI_INGEST_HOST: ${{ vars.LOKI_INGEST_HOST }}
464-
LOKI_INGEST_USERNAME: ${{ secrets.LOKI_INGEST_USERNAME }}
465-
LOKI_INGEST_PASSWORD: ${{ secrets.LOKI_INGEST_PASSWORD }}
466-
467445
defaults:
468446
run:
469447
working-directory: mithril-infra
470-
471448
steps:
449+
- name: Checkout sources
450+
uses: actions/checkout@v3
472451

473-
- name: Checkout sources
474-
uses: actions/checkout@v3
475-
476-
- name: Get Docker image id
477-
run: echo "DOCKER_IMAGE_ID=${{ github.base_ref || github.ref_name }}-$(echo ${{ github.sha }} | cut -c1-7)" >> $GITHUB_ENV
478-
479-
- name: Prepare service account credentials
480-
run: |
481-
echo '${{ env.GOOGLE_APPLICATION_CREDENTIALS}}' > ./google-application-credentials.json
482-
chmod u+x ./assets/tools/utils/google-credentials-public-key.sh
483-
./assets/tools/utils/google-credentials-public-key.sh ./google-application-credentials.json ./assets/ssh_keys curry
484-
485-
- name: Prepare terraform variables
486-
run: |
487-
cat > ./env.variables.tfvars << EOF
488-
environment_prefix = "${{ matrix.environment_prefix }}"
489-
cardano_network = "${{ matrix.cardano_network }}"
490-
google_region = "${{ matrix.google_region }}"
491-
google_zone = "${{ matrix.google_zone }}"
492-
google_machine_type = "${{ matrix.google_machine_type }}"
493-
google_compute_instance_data_disk_size = "${{ matrix.google_compute_instance_data_disk_size }}"
494-
google_service_credentials_json_file = "./google-application-credentials.json"
495-
mithril_api_domain = "${{ matrix.mithril_api_domain }}"
496-
mithril_image_id = "${{ env.DOCKER_IMAGE_ID }}"
497-
mithril_genesis_verification_key_url = "${{ env.GENESIS_VERIFICATION_KEY_URL }}"
498-
mithril_genesis_secret_key = "${{ env.GENESIS_SECRET_KEY }}"
499-
mithril_protocol_parameters = ${{ matrix.mithril_protocol_parameters }}
500-
mithril_era_reader_adapter_type = "cardano-chain"
501-
mithril_era_reader_address_url = "${{ env.ERA_READER_ADDRESS_URL }}"
502-
mithril_era_reader_verification_key_url = "${{ env.ERA_READER_VERIFICATION_KEY_URL }}"
503-
mithril_era_reader_secret_key = "${{ env.ERA_READER_SECRET_KEY }}"
504-
mithril_signers = ${{ matrix.mithril_signers }}
505-
prometheus_auth_username = "${{ env.PROMETHEUS_AUTH_USERNAME }}"
506-
prometheus_auth_password = "${{ env.PROMETHEUS_AUTH_PASSWORD }}"
507-
prometheus_ingest_host = "${{ env.PROMETHEUS_INGEST_HOST }}"
508-
prometheus_ingest_username = "${{ env.PROMETHEUS_INGEST_USERNAME }}"
509-
prometheus_ingest_password = "${{ env.PROMETHEUS_INGEST_PASSWORD }}"
510-
loki_auth_username = "${{ env.LOKI_AUTH_USERNAME }}"
511-
loki_auth_password = "${{ env.LOKI_AUTH_PASSWORD }}"
512-
loki_ingest_host = "${{ env.LOKI_INGEST_HOST }}"
513-
loki_ingest_username = "${{ env.LOKI_INGEST_USERNAME }}"
514-
loki_ingest_password = "${{ env.LOKI_INGEST_PASSWORD }}"
515-
EOF
516-
terraform fmt ./env.variables.tfvars
517-
cat ./env.variables.tfvars
518-
519-
- name: Setup Terraform
520-
uses: hashicorp/setup-terraform@v2
521-
with:
522-
terraform_wrapper: false
523-
524-
- name: Init Terraform
525-
run: |
526-
GOOGLE_APPLICATION_CREDENTIALS=./google-application-credentials.json terraform init -backend-config="bucket=${{ matrix.terraform_backend_bucket }}" -backend-config="prefix=terraform/mithril-${{ matrix.environment }}"
527-
528-
- name: Check Terraform
529-
run: terraform fmt -check
530-
531-
- name: Terraform Plan
532-
run: |
533-
GOOGLE_APPLICATION_CREDENTIALS=./google-application-credentials.json terraform plan --var-file=./env.variables.tfvars
534-
535-
- name: Terraform Apply
536-
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
537-
run: |
538-
GOOGLE_APPLICATION_CREDENTIALS=./google-application-credentials.json terraform apply -auto-approve --var-file=./env.variables.tfvars
539-
540-
- name: Cleanup
541-
run: |
542-
rm -f ./env.variables.tfvars
543-
rm -f ./google-application-credentials.json
452+
- name: Get Docker image id
453+
run: echo "DOCKER_IMAGE_ID=${{ github.base_ref || github.ref_name }}-$(echo ${{ github.sha }} | cut -c1-7)" >> $GITHUB_ENV
454+
455+
- name: ${{ env.DEPLOY == 'true' && 'Apply' || 'Plan' }} terraform infrastructure
456+
uses: ./.github/workflows/actions/deploy-terraform-infrastructure
457+
env:
458+
DEPLOY: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
459+
with:
460+
dry_run: ${{ env.DEPLOY == 'true' && 'false' || 'true' }}
461+
terraform_backend_bucket: ${{ matrix.terraform_backend_bucket }}
462+
environment_prefix: ${{ matrix.environment_prefix }}
463+
environment: ${{ matrix.environment }}
464+
cardano_network: ${{ matrix.cardano_network }}
465+
google_region: ${{ matrix.google_region }}
466+
google_zone: ${{ matrix.google_zone }}
467+
google_machine_type: ${{ matrix.google_machine_type }}
468+
google_compute_instance_data_disk_size: ${{ matrix.google_compute_instance_data_disk_size }}
469+
google_application_credentials: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }}
470+
mithril_api_domain: ${{ matrix.mithril_api_domain }}
471+
mithril_image_id: ${{ env.DOCKER_IMAGE_ID }}
472+
mithril_protocol_parameters: ${{ toJSON(matrix.mithril_protocol_parameters) }}
473+
mithril_signers: ${{ toJSON(matrix.mithril_signers) }}
474+
mithril_genesis_secret_key: ${{ secrets.GENESIS_SECRET_KEY }}
475+
mithril_genesis_verification_key_url: ${{ vars.GENESIS_VERIFICATION_KEY_URL }}
476+
mithril_era_reader_address_url: ${{ vars.ERA_READER_ADDRESS_URL }}
477+
mithril_era_reader_verification_key_url: ${{ vars.ERA_READER_VERIFICATION_KEY_URL }}
478+
mithril_era_reader_secret_key: ${{ secrets.ERA_READER_SECRET_KEY }}
479+
mithril_aggregator_snapshot_compression_algorithm: ${{ vars.AGGREGATOR_SNAPSHOT_COMPRESSION_ALGORITHM }}
480+
mithril_aggregator_zstandard_parameters_level: ${{ vars.AGGREGATOR_SNAPSHOT_ZSTANDARD_LEVEL }}
481+
mithril_aggregator_zstandard_parameters_workers: ${{ vars.AGGREGATOR_SNAPSHOT_ZSTANDARD_WORKERS }}
482+
prometheus_auth_username: ${{ secrets.PROMETHEUS_AUTH_USERNAME }}
483+
prometheus_auth_password: ${{ secrets.PROMETHEUS_AUTH_PASSWORD }}
484+
prometheus_ingest_host: ${{ vars.PROMETHEUS_INGEST_HOST }}
485+
prometheus_ingest_username: ${{ secrets.PROMETHEUS_INGEST_USERNAME }}
486+
prometheus_ingest_password: ${{ secrets.PROMETHEUS_INGEST_PASSWORD }}
487+
loki_auth_username: ${{ secrets.LOKI_AUTH_USERNAME }}
488+
loki_auth_password: ${{ secrets.LOKI_AUTH_PASSWORD }}
489+
loki_ingest_host: ${{ vars.LOKI_INGEST_HOST }}
490+
loki_ingest_username: ${{ secrets.LOKI_INGEST_USERNAME }}
491+
loki_ingest_password: ${{ secrets.LOKI_INGEST_PASSWORD }}

0 commit comments

Comments
 (0)