Skip to content

Commit 9bdde2b

Browse files
jpraynaudjpraynaud
committed
Use snapshot compression parameters in CI/CD
1 parent 1edbc6b commit 9bdde2b

File tree

8 files changed

+75
-27
lines changed

8 files changed

+75
-27
lines changed

‎.github/workflows/actions/deploy-terraform-infrastructure/action.yml‎

Lines changed: 39 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,15 @@ inputs:
6060
mithril_era_reader_secret_key:
6161
description: Mithril era reader secret key (onlye for test networks).
6262
required: false
63+
mithril_aggregator_snapshot_compression_algorithm:
64+
description: Mithril aggregator snapshot compression algorithm.
65+
required: false
66+
mithril_aggregator_zstandard_parameters_level:
67+
description: Mithril aggregator snapshot zstandard compression level.
68+
required: false
69+
mithril_aggregator_zstandard_parameters_workers:
70+
description: Mithril aggregator snapshot zstandard number of workers.
71+
required: false
6372
prometheus_auth_username:
6473
description: Prometheus metrics endpoint username.
6574
required: false
@@ -110,33 +119,36 @@ runs:
110119
working-directory: mithril-infra
111120
run: |
112121
cat > ./env.variables.tfvars << EOF
113-
environment_prefix = "${{ inputs.environment_prefix }}"
114-
cardano_network = "${{ inputs.cardano_network }}"
115-
google_region = "${{ inputs.google_region }}"
116-
google_zone = "${{ inputs.google_zone }}"
117-
google_machine_type = "${{ inputs.google_machine_type }}"
118-
google_compute_instance_data_disk_size = "${{ inputs.google_compute_instance_data_disk_size }}"
119-
google_service_credentials_json_file = "./google-application-credentials.json"
120-
mithril_api_domain = "${{ inputs.mithril_api_domain }}"
121-
mithril_image_id = "${{ inputs.mithril_image_id }}"
122-
mithril_genesis_verification_key_url = "${{ inputs.mithril_genesis_verification_key_url }}"
123-
mithril_genesis_secret_key = "${{ inputs.mithril_genesis_secret_key }}"
124-
mithril_protocol_parameters = ${{ fromJSON(inputs.mithril_protocol_parameters) }}
125-
mithril_era_reader_adapter_type = "cardano-chain"
126-
mithril_era_reader_address_url = "${{ inputs.mithril_era_reader_address_url }}"
127-
mithril_era_reader_verification_key_url = "${{ inputs.mithril_era_reader_verification_key_url }}"
128-
mithril_era_reader_secret_key = "${{ inputs.mithril_era_reader_secret_key }}"
129-
mithril_signers = ${{ fromJSON(inputs.mithril_signers) }}
130-
prometheus_auth_username = "${{ inputs.prometheus_auth_username }}"
131-
prometheus_auth_password = "${{ inputs.prometheus_auth_password }}"
132-
prometheus_ingest_host = "${{ inputs.prometheus_ingest_host }}"
133-
prometheus_ingest_username = "${{ inputs.prometheus_ingest_username }}"
134-
prometheus_ingest_password = "${{ inputs.prometheus_ingest_password }}"
135-
loki_auth_username = "${{ inputs.loki_auth_username }}"
136-
loki_auth_password = "${{ inputs.loki_auth_password }}"
137-
loki_ingest_host = "${{ inputs.loki_ingest_host }}"
138-
loki_ingest_username = "${{ inputs.loki_ingest_username }}"
139-
loki_ingest_password = "${{ inputs.loki_ingest_password }}"
122+
environment_prefix = "${{ inputs.environment_prefix }}"
123+
cardano_network = "${{ inputs.cardano_network }}"
124+
google_region = "${{ inputs.google_region }}"
125+
google_zone = "${{ inputs.google_zone }}"
126+
google_machine_type = "${{ inputs.google_machine_type }}"
127+
google_compute_instance_data_disk_size = "${{ inputs.google_compute_instance_data_disk_size }}"
128+
google_service_credentials_json_file = "./google-application-credentials.json"
129+
mithril_api_domain = "${{ inputs.mithril_api_domain }}"
130+
mithril_image_id = "${{ inputs.mithril_image_id }}"
131+
mithril_genesis_verification_key_url = "${{ inputs.mithril_genesis_verification_key_url }}"
132+
mithril_genesis_secret_key = "${{ inputs.mithril_genesis_secret_key }}"
133+
mithril_protocol_parameters = ${{ fromJSON(inputs.mithril_protocol_parameters) }}
134+
mithril_signers = ${{ fromJSON(inputs.mithril_signers) }}
135+
mithril_era_reader_adapter_type = "cardano-chain"
136+
mithril_era_reader_address_url = "${{ inputs.mithril_era_reader_address_url }}"
137+
mithril_era_reader_verification_key_url = "${{ inputs.mithril_era_reader_verification_key_url }}"
138+
mithril_era_reader_secret_key = "${{ inputs.mithril_era_reader_secret_key }}"
139+
mithril_aggregator_snapshot_compression_algorithm = "${{ inputs.mithril_aggregator_snapshot_compression_algorithm }}"
140+
mithril_aggregator_zstandard_parameters_level = "${{ inputs.mithril_aggregator_zstandard_parameters_level }}"
141+
mithril_aggregator_zstandard_parameters_workers = "${{ inputs.mithril_aggregator_zstandard_parameters_workers }}"
142+
prometheus_auth_username = "${{ inputs.prometheus_auth_username }}"
143+
prometheus_auth_password = "${{ inputs.prometheus_auth_password }}"
144+
prometheus_ingest_host = "${{ inputs.prometheus_ingest_host }}"
145+
prometheus_ingest_username = "${{ inputs.prometheus_ingest_username }}"
146+
prometheus_ingest_password = "${{ inputs.prometheus_ingest_password }}"
147+
loki_auth_username = "${{ inputs.loki_auth_username }}"
148+
loki_auth_password = "${{ inputs.loki_auth_password }}"
149+
loki_ingest_host = "${{ inputs.loki_ingest_host }}"
150+
loki_ingest_username = "${{ inputs.loki_ingest_username }}"
151+
loki_ingest_password = "${{ inputs.loki_ingest_password }}"
140152
EOF
141153
terraform fmt ./env.variables.tfvars
142154
cat ./env.variables.tfvars

‎.github/workflows/ci.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -476,6 +476,9 @@ jobs:
476476
mithril_era_reader_address_url: ${{ vars.ERA_READER_ADDRESS_URL }}
477477
mithril_era_reader_verification_key_url: ${{ vars.ERA_READER_VERIFICATION_KEY_URL }}
478478
mithril_era_reader_secret_key: ${{ secrets.ERA_READER_SECRET_KEY }}
479+
mithril_aggregator_snapshot_compression_algorithm: ${{ vars.AGGREGATOR_SNAPSHOT_COMPRESSION_ALGORITHM }}
480+
mithril_aggregator_zstandard_parameters_level: ${{ vars.AGGREGATOR_SNAPSHOT_ZSTANDARD_LEVEL }}
481+
mithril_aggregator_zstandard_parameters_workers: ${{ vars.AGGREGATOR_SNAPSHOT_ZSTANDARD_WORKERS }}
479482
prometheus_auth_username: ${{ secrets.PROMETHEUS_AUTH_USERNAME }}
480483
prometheus_auth_password: ${{ secrets.PROMETHEUS_AUTH_PASSWORD }}
481484
prometheus_ingest_host: ${{ vars.PROMETHEUS_INGEST_HOST }}

‎.github/workflows/pre-release.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -201,6 +201,9 @@ jobs:
201201
mithril_era_reader_address_url: ${{ vars.ERA_READER_ADDRESS_URL }}
202202
mithril_era_reader_verification_key_url: ${{ vars.ERA_READER_VERIFICATION_KEY_URL }}
203203
mithril_era_reader_secret_key: ${{ secrets.ERA_READER_SECRET_KEY }}
204+
mithril_aggregator_snapshot_compression_algorithm: ${{ vars.AGGREGATOR_SNAPSHOT_COMPRESSION_ALGORITHM }}
205+
mithril_aggregator_zstandard_parameters_level: ${{ vars.AGGREGATOR_SNAPSHOT_ZSTANDARD_LEVEL }}
206+
mithril_aggregator_zstandard_parameters_workers: ${{ vars.AGGREGATOR_SNAPSHOT_ZSTANDARD_WORKERS }}
204207
prometheus_auth_username: ${{ secrets.PROMETHEUS_AUTH_USERNAME }}
205208
prometheus_auth_password: ${{ secrets.PROMETHEUS_AUTH_PASSWORD }}
206209
prometheus_ingest_host: ${{ vars.PROMETHEUS_INGEST_HOST }}

‎.github/workflows/release.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -191,6 +191,9 @@ jobs:
191191
mithril_genesis_verification_key_url: ${{ vars.GENESIS_VERIFICATION_KEY_URL }}
192192
mithril_era_reader_address_url: ${{ vars.ERA_READER_ADDRESS_URL }}
193193
mithril_era_reader_verification_key_url: ${{ vars.ERA_READER_VERIFICATION_KEY_URL }}
194+
mithril_aggregator_snapshot_compression_algorithm: ${{ vars.AGGREGATOR_SNAPSHOT_COMPRESSION_ALGORITHM }}
195+
mithril_aggregator_zstandard_parameters_level: ${{ vars.AGGREGATOR_SNAPSHOT_ZSTANDARD_LEVEL }}
196+
mithril_aggregator_zstandard_parameters_workers: ${{ vars.AGGREGATOR_SNAPSHOT_ZSTANDARD_WORKERS }}
194197
prometheus_auth_username: ${{ secrets.PROMETHEUS_AUTH_USERNAME }}
195198
prometheus_auth_password: ${{ secrets.PROMETHEUS_AUTH_PASSWORD }}
196199
prometheus_ingest_host: ${{ vars.PROMETHEUS_INGEST_HOST }}

‎mithril-infra/assets/docker/docker-compose-aggregator-auth.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,9 @@ services:
5656
- SNAPSHOT_STORE_TYPE=local
5757
- SNAPSHOT_UPLOADER_TYPE=gcp
5858
- SNAPSHOT_BUCKET_NAME=${SNAPSHOT_BUCKET_NAME}
59+
- SNAPSHOT_COMPRESSION_ALGORITHM=${SNAPSHOT_COMPRESSION_ALGORITHM}
60+
- ZSTANDARD_PARAMETERS__LEVEL=${ZSTANDARD_PARAMETERS__LEVEL}
61+
- ZSTANDARD_PARAMETERS__NUMBER_OF_WORKERS=${ZSTANDARD_PARAMETERS__NUMBER_OF_WORKERS}
5962
- DATA_STORES_DIRECTORY=/mithril-aggregator/mithril/stores
6063
- STORE_RETENTION_LIMIT=5
6164
- CARDANO_NODE_SOCKET_PATH=/ipc/node.socket

‎mithril-infra/assets/docker/docker-compose-aggregator-noauth.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,9 @@ services:
5656
- SNAPSHOT_STORE_TYPE=local
5757
- SNAPSHOT_UPLOADER_TYPE=gcp
5858
- SNAPSHOT_BUCKET_NAME=${SNAPSHOT_BUCKET_NAME}
59+
- SNAPSHOT_COMPRESSION_ALGORITHM=${SNAPSHOT_COMPRESSION_ALGORITHM}
60+
- ZSTANDARD_PARAMETERS__LEVEL=${ZSTANDARD_PARAMETERS__LEVEL}
61+
- ZSTANDARD_PARAMETERS__NUMBER_OF_WORKERS=${ZSTANDARD_PARAMETERS__NUMBER_OF_WORKERS}
5962
- DATA_STORES_DIRECTORY=/mithril-aggregator/mithril/stores
6063
- STORE_RETENTION_LIMIT=5
6164
- CARDANO_NODE_SOCKET_PATH=/ipc/node.socket

‎mithril-infra/mithril.aggregator.tf‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,9 @@ EOT
4646
"export AGGREGATOR_HOST=${local.mithril_aggregator_host}",
4747
"export GOOGLE_APPLICATION_CREDENTIALS_JSON='${local.google_cloud_storage_credentials_json}'",
4848
"export SNAPSHOT_BUCKET_NAME='${google_storage_bucket.cloud_storage.name}'",
49+
"export SNAPSHOT_COMPRESSION_ALGORITHM=${var.mithril_aggregator_snapshot_compression_algorithm}",
50+
"export ZSTANDARD_PARAMETERS__LEVEL=${var.mithril_aggregator_zstandard_parameters_level}",
51+
"export ZSTANDARD_PARAMETERS__NUMBER_OF_WORKERS=${var.mithril_aggregator_zstandard_parameters_workers}",
4952
"export GENESIS_VERIFICATION_KEY=$(wget -q -O - ${var.mithril_genesis_verification_key_url})",
5053
"export GENESIS_SECRET_KEY='${var.mithril_genesis_secret_key}'",
5154
"export PROTOCOL_PARAMETERS__K='${var.mithril_protocol_parameters.k}'",

‎mithril-infra/variables.tf‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,24 @@ variable "mithril_container_logging_driver" {
160160
default = "json-file"
161161
}
162162

163+
variable "mithril_aggregator_snapshot_compression_algorithm" {
164+
type = string
165+
description = "The compression algorithm of the snapshot archive"
166+
default = "zstandard"
167+
}
168+
169+
variable "mithril_aggregator_zstandard_parameters_level" {
170+
type = string
171+
description = "Zstandard compression level parameter"
172+
default = "9"
173+
}
174+
175+
variable "mithril_aggregator_zstandard_parameters_workers" {
176+
type = string
177+
description = "Zstandard number of workers parameter"
178+
default = "4"
179+
}
180+
163181
variable "mithril_aggregator_auth_username" {
164182
type = string
165183
description = "The username for authentication on the mithril aggregator"

0 commit comments

Comments
 (0)