Skip to content

Commit 3b71450

Browse files
sghosh23sghosh23
authored
WPB-20580: Fix resource unavailavility and make sure a clean build with deploy via CI (#815)
* Add fallback localtion and resoruces * postgres-external values * fix lint issue * add demo values * try make a faster deployment process * Set default upload name in deploy workflow for test only * fix lint issue * try again * fix script to get the supplied s3 hash * try again * WIP-1 * add logging * add nix env for deploy only workflow * one more time * try parallel deployment and copy from local assets * run again * replace bitnami with bitnamilegacy * fix: bintami/nginx fetch issue * fix chart with no image issue * fix hanging issues * try skip this chart * fix grep no match issue * try another fix to append images correctly * try with aws-ingress issue * keep it simple * try fix the count * fix jq issue * use yq * fix: patch chart images * Update ephemeral database and redis values * update offline workflow and scripts * fix demo build * fix lint and celanup
1 parent 8903e76 commit 3b71450

File tree

14 files changed

+656
-122
lines changed

14 files changed

+656
-122
lines changed

‎.github/workflows/offline.yml‎

Lines changed: 143 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,14 @@ on:
1111
- '*.md'
1212
- '**/*.md'
1313
jobs:
14-
offline:
15-
name: Prepare offline package
16-
# Useful to skip expensive CI when writing docs
14+
# Build default profile and create local assets
15+
build-default:
16+
name: Build default profile
1717
if: "!contains(github.event.head_commit.message, 'skip ci')"
1818
runs-on:
1919
group: wire-server-deploy
20+
outputs:
21+
upload_name: ${{ steps.upload_name.outputs.UPLOAD_NAME }}
2022
steps:
2123
- uses: actions/checkout@v2
2224
with:
@@ -32,60 +34,177 @@ jobs:
3234

3335
- name: Get upload name
3436
id: upload_name
35-
run: |
36-
# FIXME: Tag with a nice release name using the github tag...
37-
# SOURCE_TAG=${GITHUB_REF#refs/tags/}
38-
echo ::set-output name=UPLOAD_NAME::$GITHUB_SHA
39-
# echo ::set-output name=UPLOAD_NAME::${SOURCE_TAG:-$GITHUB_SHA}
37+
run: echo "UPLOAD_NAME=$GITHUB_SHA" >> $GITHUB_OUTPUT
4038

41-
# deafult profile build
39+
# default profile build
4240
- name: Process the default profile build
4341
run: ./offline/default-build/build.sh
4442
env:
4543
GPG_PRIVATE_KEY: '${{ secrets.GPG_PRIVATE_KEY }}'
4644
DOCKER_LOGIN: '${{ secrets.DOCKER_LOGIN }}'
4745

48-
- name: Copy default build assets tarball to S3 and clean up
46+
# Upload the assets to be shared with other jobs
47+
- name: Upload build artifacts
48+
uses: actions/upload-artifact@v4
49+
with:
50+
name: default-build-assets
51+
path: offline/default-build/output/assets.tgz
52+
retention-days: 1
53+
54+
# Upload to S3 in parallel with deployment
55+
upload-s3:
56+
name: Upload default build to S3
57+
if: "!contains(github.event.head_commit.message, 'skip ci')"
58+
needs: build-default
59+
runs-on:
60+
group: wire-server-deploy
61+
steps:
62+
- uses: actions/checkout@v2
63+
with:
64+
submodules: true
65+
66+
- name: Download build artifacts
67+
uses: actions/download-artifact@v4
68+
with:
69+
name: default-build-assets
70+
path: offline/default-build/output/
71+
72+
- name: Copy default build assets tarball to S3
4973
run: |
50-
# Upload tarball for each profile by specifying their OUTPUT_TAR path
51-
aws s3 cp offline/default-build/output/assets.tgz s3://public.wire.com/artifacts/wire-server-deploy-static-${{ steps.upload_name.outputs.UPLOAD_NAME }}.tgz
52-
echo "Uploaded to: https://s3-$AWS_REGION.amazonaws.com/public.wire.com/artifacts/wire-server-deploy-static-${{ steps.upload_name.outputs.UPLOAD_NAME }}.tgz"
53-
# removing everything except assets.tgz as it is not required anymore in the further builds
54-
find offline/default-build/output/ -mindepth 1 -maxdepth 1 ! -name 'assets.tgz' -exec rm -r {} +
74+
aws s3 cp offline/default-build/output/assets.tgz s3://public.wire.com/artifacts/wire-server-deploy-static-${{ needs.build-default.outputs.upload_name }}.tgz
75+
echo "Uploaded to: https://s3-$AWS_REGION.amazonaws.com/public.wire.com/artifacts/wire-server-deploy-static-${{ needs.build-default.outputs.upload_name }}.tgz"
5576
env:
5677
AWS_ACCESS_KEY_ID: '${{ secrets.AWS_ACCESS_KEY_ID }}'
5778
AWS_SECRET_ACCESS_KEY: '${{ secrets.AWS_SECRET_ACCESS_KEY }}'
5879
AWS_REGION: "eu-west-1"
5980

81+
# Deploy to Hetzner in parallel with S3 upload
82+
deploy-hetzner:
83+
name: Deploy default build to Hetzner
84+
if: "!contains(github.event.head_commit.message, 'skip ci')"
85+
needs: build-default
86+
runs-on:
87+
group: wire-server-deploy
88+
steps:
89+
- uses: actions/checkout@v2
90+
with:
91+
submodules: true
92+
- uses: cachix/install-nix-action@v27
93+
- uses: cachix/cachix-action@v15
94+
with:
95+
name: wire-server
96+
signingKey: "${{ secrets.CACHIX_SIGNING_KEY }}"
97+
98+
- name: Install nix environment
99+
run: nix-env -f default.nix -iA env
100+
101+
- name: Download build artifacts
102+
uses: actions/download-artifact@v4
103+
with:
104+
name: default-build-assets
105+
path: offline/default-build/output/
106+
107+
- name: Install terraform
108+
uses: hashicorp/setup-terraform@v3
109+
with:
110+
terraform_version: "^1.3.7"
111+
terraform_wrapper: false
112+
113+
- name: Deploy offline environment to hetzner
114+
run: ./offline/cd-with-retry.sh
115+
env:
116+
HCLOUD_TOKEN: '${{ secrets.HCLOUD_TOKEN }}'
117+
118+
# Build container in parallel
119+
build-container:
120+
name: Build container
121+
if: "!contains(github.event.head_commit.message, 'skip ci')"
122+
needs: build-default
123+
runs-on:
124+
group: wire-server-deploy
125+
steps:
126+
- uses: actions/checkout@v2
127+
with:
128+
submodules: true
129+
- uses: cachix/install-nix-action@v27
130+
- uses: cachix/cachix-action@v15
131+
with:
132+
name: wire-server
133+
signingKey: "${{ secrets.CACHIX_SIGNING_KEY }}"
134+
60135
- name: Build and upload wire-server-deploy container
61136
run: |
62137
container_image=$(nix-build --no-out-link -A container)
63138
skopeo copy --retry-times 10 --dest-creds "$DOCKER_LOGIN" \
64139
docker-archive:"$container_image" \
65-
"docker://quay.io/wire/wire-server-deploy:${{ steps.upload_name.outputs.UPLOAD_NAME }}"
140+
"docker://quay.io/wire/wire-server-deploy:${{ needs.build-default.outputs.upload_name }}"
66141
env:
67142
DOCKER_LOGIN: '${{ secrets.DOCKER_LOGIN }}'
68143

69-
# demo profile build
144+
# Build demo profile
145+
build-demo:
146+
name: Build demo profile
147+
if: "!contains(github.event.head_commit.message, 'skip ci')"
148+
runs-on:
149+
group: wire-server-deploy
150+
steps:
151+
- uses: actions/checkout@v2
152+
with:
153+
submodules: true
154+
- uses: cachix/install-nix-action@v27
155+
- uses: cachix/cachix-action@v15
156+
with:
157+
name: wire-server
158+
signingKey: "${{ secrets.CACHIX_SIGNING_KEY }}"
159+
160+
- name: Install nix environment
161+
run: nix-env -f default.nix -iA env
162+
163+
- name: Get upload name
164+
id: upload_name
165+
run: echo "UPLOAD_NAME=$GITHUB_SHA" >> $GITHUB_OUTPUT
166+
70167
- name: Process the demo profile build
71168
run: ./offline/demo-build/build.sh
72169
env:
73170
GPG_PRIVATE_KEY: '${{ secrets.GPG_PRIVATE_KEY }}'
74171
DOCKER_LOGIN: '${{ secrets.DOCKER_LOGIN }}'
75172

76-
- name: Copy demo build assets tarball to S3 and clean up
173+
- name: Copy demo build assets tarball to S3
77174
run: |
78-
# Upload tarball for each profile by specifying their OUTPUT_TAR path
79175
aws s3 cp offline/demo-build/output/assets.tgz s3://public.wire.com/artifacts/wire-server-deploy-static-demo-${{ steps.upload_name.outputs.UPLOAD_NAME }}.tgz
80176
echo "Uploaded to: https://s3-$AWS_REGION.amazonaws.com/public.wire.com/artifacts/wire-server-deploy-static-demo-${{ steps.upload_name.outputs.UPLOAD_NAME }}.tgz"
81-
# remove the assets from the build to optimize the space on the server
82-
rm -rf offline/demo-build/output/*
83177
env:
84178
AWS_ACCESS_KEY_ID: '${{ secrets.AWS_ACCESS_KEY_ID }}'
85179
AWS_SECRET_ACCESS_KEY: '${{ secrets.AWS_SECRET_ACCESS_KEY }}'
86180
AWS_REGION: "eu-west-1"
87181

88-
# min profile build
182+
- name: Cleanup demo build assets
183+
run: rm -rf offline/demo-build/output/
184+
185+
# Build min profile
186+
build-min:
187+
name: Build min profile
188+
if: "!contains(github.event.head_commit.message, 'skip ci')"
189+
runs-on:
190+
group: wire-server-deploy
191+
steps:
192+
- uses: actions/checkout@v2
193+
with:
194+
submodules: true
195+
- uses: cachix/install-nix-action@v27
196+
- uses: cachix/cachix-action@v15
197+
with:
198+
name: wire-server
199+
signingKey: "${{ secrets.CACHIX_SIGNING_KEY }}"
200+
201+
- name: Install nix environment
202+
run: nix-env -f default.nix -iA env
203+
204+
- name: Get upload name
205+
id: upload_name
206+
run: echo "UPLOAD_NAME=$GITHUB_SHA" >> $GITHUB_OUTPUT
207+
89208
- name: Process the min profile build
90209
run: ./offline/min-build/build.sh
91210
env:
@@ -94,30 +213,12 @@ jobs:
94213

95214
- name: Copy min build assets tarball to S3
96215
run: |
97-
# Upload tarball for each profile by specifying their OUTPUT_TAR path
98216
aws s3 cp offline/min-build/output/assets.tgz s3://public.wire.com/artifacts/wire-server-deploy-static-min-${{ steps.upload_name.outputs.UPLOAD_NAME }}.tgz
99217
echo "Uploaded to: https://s3-$AWS_REGION.amazonaws.com/public.wire.com/artifacts/wire-server-deploy-static-min-${{ steps.upload_name.outputs.UPLOAD_NAME }}.tgz"
100-
# remove the archives from the build to optimize the space on the server
101-
rm -rf offline/min-build/output/*
102218
env:
103219
AWS_ACCESS_KEY_ID: '${{ secrets.AWS_ACCESS_KEY_ID }}'
104220
AWS_SECRET_ACCESS_KEY: '${{ secrets.AWS_SECRET_ACCESS_KEY }}'
105221
AWS_REGION: "eu-west-1"
106222

107-
- name: Install terraform
108-
uses: hashicorp/setup-terraform@v3
109-
with:
110-
terraform_version: "^1.3.7"
111-
terraform_wrapper: false
112-
113-
- name: Deploy offline environment to hetzner
114-
run: |
115-
./offline/cd.sh
116-
env:
117-
HCLOUD_TOKEN: '${{ secrets.HCLOUD_TOKEN }}'
118-
119-
#- name: Clean up hetzner environment; just in case
120-
# if: always()
121-
# run: (cd terraform/examples/wire-server-deploy-offline-hetzner ; terraform init && terraform destroy -auto-approve)
122-
# env:
123-
# HCLOUD_TOKEN: '${{ secrets.HCLOUD_TOKEN }}'
223+
- name: Cleanup min build assets
224+
run: rm -rf offline/min-build/output/

‎changelog.d/3-deploy-builds/optimize-default-build-deploy-process‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
Changed: Optimize Wire offline deployment pipeline with parallel job execution and S3 direct downloads
2+
Added: Retry logic with progressive server type fallbacks for Hetzner Cloud resource availability issues
3+
Changed: Implement parallel terraform operations (15 parallelism) and fast SSH connection multiplexing
4+
Changed: Move ansible execution directly to adminhost for faster private network connectivity
5+
Changed: Reduce CI deployment time from 60+ minutes to ~30-40 minutes through parallel builds and optimized deployment process

‎nix/scripts/list-helm-containers.sh‎

100644100755
Lines changed: 46 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
# those.
66
# In cases where no container image tag has been specified, it'll use `latest`.
77
# The list is sorted and deduplicated, then printed to stdout.
8-
set -eou pipefail
8+
set -euo pipefail
99

1010
VALUES_DIR=""
1111
HELM_IMAGE_TREE_FILE=""
@@ -74,14 +74,51 @@ function optionally_complain() {
7474
images=""
7575
# For each helm chart passed in from stdin, use the example values to
7676
# render the charts, and assemble the list of images this would fetch.
77+
chart_count=0
7778
while IFS= read -r chart; do
78-
echo "Running helm template on chart ${chart}…" >&2
79-
current_images=$(helm template --debug "${chart}" \
80-
--set federate.dtls.tls.key=emptyString \
81-
--set federate.dtls.tls.crt=emptyString \
82-
$( [[ -f "${VALUES_DIR}"/$(basename "${chart}")/"${VALUES_TYPE}"-values.example.yaml ]] && echo "-f ${VALUES_DIR}/$(basename "${chart}")/${VALUES_TYPE}-values.example.yaml" ) \
83-
$( [[ -f "${VALUES_DIR}"/$(basename "${chart}")/"${VALUES_TYPE}"-secrets.example.yaml ]] && echo "-f ${VALUES_DIR}/$(basename "${chart}")/${VALUES_TYPE}-secrets.example.yaml" ) \
84-
| yq -r '..|.image? | select(.)' | optionally_complain | sort -u)
79+
chart_count=$((chart_count + 1))
80+
echo "[$chart_count] Running helm template on chart ${chart}…" >&2
81+
set +e # Temporarily disable exit on error
82+
# Determine values file to use (prod first, then demo as fallback)
83+
values_file=""
84+
if [[ -f "${VALUES_DIR}"/$(basename "${chart}")/"${VALUES_TYPE}"-values.example.yaml ]]; then
85+
values_file="${VALUES_DIR}/$(basename "${chart}")/${VALUES_TYPE}-values.example.yaml"
86+
elif [[ -f "${VALUES_DIR}"/$(basename "${chart}")/demo-values.example.yaml ]]; then
87+
values_file="${VALUES_DIR}/$(basename "${chart}")/demo-values.example.yaml"
88+
echo "Using demo values for $(basename $chart) (no ${VALUES_TYPE} values found)" >&2
89+
fi
90+
91+
# Determine secrets file to use
92+
secrets_file=""
93+
if [[ -f "${VALUES_DIR}"/$(basename "${chart}")/"${VALUES_TYPE}"-secrets.example.yaml ]]; then
94+
secrets_file="${VALUES_DIR}/$(basename "${chart}")/${VALUES_TYPE}-secrets.example.yaml"
95+
elif [[ -f "${VALUES_DIR}"/$(basename "${chart}")/demo-secrets.example.yaml ]]; then
96+
secrets_file="${VALUES_DIR}/$(basename "${chart}")/demo-secrets.example.yaml"
97+
fi
98+
99+
raw_images=$(helm template "${chart}" \
100+
$( [[ -n "$values_file" ]] && echo "-f $values_file" ) \
101+
$( [[ -n "$secrets_file" ]] && echo "-f $secrets_file" ) \
102+
2>&1 | yq -r '..|.image?' | grep -v "^null$" | grep -v "^---$" | grep -v "^$" || true)
103+
104+
helm_exit_code=$?
105+
set -e # Re-enable exit on error
106+
107+
if [[ $helm_exit_code -ne 0 ]]; then
108+
echo "ERROR: Failed to process chart $(basename $chart)" >&2
109+
echo "Chart path: $chart" >&2
110+
echo "Values file: ${values_file:-none}" >&2
111+
echo "Secrets file: ${secrets_file:-none}" >&2
112+
echo "Try running: helm template $chart $([ -n "$values_file" ] && echo "-f $values_file") $([ -n "$secrets_file" ] && echo "-f $secrets_file")" >&2
113+
raw_images=""
114+
fi
115+
116+
# Process extracted images
117+
if [[ -n "$raw_images" ]]; then
118+
current_images=$(echo "$raw_images" | grep -v "^$" | optionally_complain | sort -u)
119+
else
120+
current_images=""
121+
fi
85122

86123
images+="$current_images\n"
87124
if [[ -n "$current_images" ]]; then
@@ -90,4 +127,4 @@ while IFS= read -r chart; do
90127
append_chart_entry "$(basename $chart)" "$image_array" "${HELM_IMAGE_TREE_FILE}"
91128
fi
92129
done
93-
echo -e "$images" | grep . | sort -u
130+
echo -e "$images" | grep . | sort -u || true

0 commit comments

Comments
 (0)