Skip to content

Commit 3c00387

Browse files
EgorPopelyaevEgorPopelyaev
authored
[Release|CI/CD] Github pipeline to publish polkadot deb package (#6640)
This pipeline should replace a manual action done on the `cleamroom` server to publish the `polkadot` deb package to our apt repo with the pipeline triggered from the new paritytech-release org. Right now, this is done manually by running the [add-packages.sh](https://github.com/paritytech/cleanroom/blob/master/ansible/roles/parity-repos/files/add-packages.sh) script on the `cleanroom` machine. What is done under the hood: - Pipeline downloads `polakdot` deb package from S3, that was prebuilt in the [Build release rc pipeline](https://github.com/paritytech/polkadot-sdk/blob/master/.github/workflows/release-build-rc.yml) - Prepares and syncs local apt repository - Adds and signs deb package to it using `reprepro` - Uploads new deb package to the distributed repo Closes: paritytech/release-engineering#239
1 parent 8216235 commit 3c00387

File tree

3 files changed

+222
-2
lines changed

3 files changed

+222
-2
lines changed

‎.github/scripts/common/lib.sh‎

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -237,15 +237,44 @@ fetch_release_artifacts() {
237237
popd > /dev/null
238238
}
239239

240-
# Fetch the release artifacts like binary and signatures from S3. Assumes the ENV are set:
240+
# Fetch deb package from S3. Assumes the ENV are set:
241241
# - RELEASE_ID
242242
# - GITHUB_TOKEN
243243
# - REPO in the form paritytech/polkadot
244-
fetch_release_artifacts_from_s3() {
244+
fetch_debian_package_from_s3() {
245245
BINARY=$1
246246
echo "Version : $VERSION"
247247
echo "Repo : $REPO"
248248
echo "Binary : $BINARY"
249+
echo "Tag : $RELEASE_TAG"
250+
OUTPUT_DIR=${OUTPUT_DIR:-"./release-artifacts/${BINARY}"}
251+
echo "OUTPUT_DIR : $OUTPUT_DIR"
252+
253+
URL_BASE=$(get_s3_url_base $BINARY)
254+
echo "URL_BASE=$URL_BASE"
255+
256+
URL=$URL_BASE/$RELEASE_TAG/x86_64-unknown-linux-gnu/${BINARY}_${VERSION}_amd64.deb
257+
258+
mkdir -p "$OUTPUT_DIR"
259+
pushd "$OUTPUT_DIR" > /dev/null
260+
261+
echo "Fetching deb package..."
262+
263+
echo "Fetching %s" "$URL"
264+
curl --progress-bar -LO "$URL" || echo "Missing $URL"
265+
266+
pwd
267+
ls -al --color
268+
popd > /dev/null
269+
270+
}
271+
272+
# Fetch the release artifacts like binary and signatures from S3. Assumes the ENV are set:
273+
# - RELEASE_ID
274+
# - GITHUB_TOKEN
275+
# - REPO in the form paritytech/polkadot
276+
fetch_release_artifacts_from_s3() {
277+
BINARY=$1
249278
OUTPUT_DIR=${OUTPUT_DIR:-"./release-artifacts/${BINARY}"}
250279
echo "OUTPUT_DIR : $OUTPUT_DIR"
251280

‎.github/scripts/release/distributions‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
Origin: Parity
2+
Label: Parity
3+
Codename: release
4+
Architectures: amd64
5+
Components: main
6+
Description: Apt repository for software made by Parity Technologies Ltd.
7+
SignWith: 90BD75EBBB8E95CB3DA6078F94A4029AB4B35DAE
8+
9+
Origin: Parity
10+
Label: Parity Staging
11+
Codename: staging
12+
Architectures: amd64
13+
Components: main
14+
Description: Staging distribution for Parity Technologies Ltd. packages
15+
SignWith: 90BD75EBBB8E95CB3DA6078F94A4029AB4B35DAE
16+
17+
Origin: Parity
18+
Label: Parity stable2407
19+
Codename: stable2407
20+
Architectures: amd64
21+
Components: main
22+
Description: Apt repository for software made by Parity Technologies Ltd.
23+
SignWith: 90BD75EBBB8E95CB3DA6078F94A4029AB4B35DAE
24+
25+
Origin: Parity
26+
Label: Parity stable2409
27+
Codename: stable2409
28+
Architectures: amd64
29+
Components: main
30+
Description: Apt repository for software made by Parity Technologies Ltd.
31+
SignWith: 90BD75EBBB8E95CB3DA6078F94A4029AB4B35DAE
32+
33+
Origin: Parity
34+
Label: Parity stable2412
35+
Codename: stable2412
36+
Architectures: amd64
37+
Components: main
38+
Description: Apt repository for software made by Parity Technologies Ltd.
39+
SignWith: 90BD75EBBB8E95CB3DA6078F94A4029AB4B35DAE

‎.github/workflows/release-40_publish-deb-package.yml‎

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
name: Release - Publish polakdot deb package
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
tag:
7+
description: Current final release tag in the format polakdot-stableYYMM or polkadot-stable-YYMM-X
8+
default: polkadot-stable2412
9+
required: true
10+
type: string
11+
12+
distribution:
13+
description: Distribution where to publish deb package (release, staging, stable2407, etc)
14+
default: staging
15+
required: true
16+
type: string
17+
18+
jobs:
19+
check-synchronization:
20+
uses: paritytech-release/sync-workflows/.github/workflows/check-syncronization.yml@main
21+
22+
validate-inputs:
23+
needs: [check-synchronization]
24+
if: ${{ needs.check-synchronization.outputs.checks_passed }} == 'true'
25+
runs-on: ubuntu-latest
26+
outputs:
27+
release_tag: ${{ steps.validate_inputs.outputs.release_tag }}
28+
29+
steps:
30+
- name: Checkout sources
31+
uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
32+
33+
- name: Validate inputs
34+
id: validate_inputs
35+
run: |
36+
. ./.github/scripts/common/lib.sh
37+
38+
RELEASE_TAG=$(validate_stable_tag ${{ inputs.tag }})
39+
echo "release_tag=${RELEASE_TAG}" >> $GITHUB_OUTPUT
40+
41+
42+
fetch-artifacts-from-s3:
43+
runs-on: ubuntu-latest
44+
needs: [validate-inputs]
45+
env:
46+
REPO: ${{ github.repository }}
47+
RELEASE_TAG: ${{ needs.validate-inputs.outputs.release_tag }}
48+
outputs:
49+
VERSION: ${{ steps.fetch_artifacts_from_s3.outputs.VERSION }}
50+
51+
steps:
52+
- name: Checkout sources
53+
uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
54+
55+
- name: Fetch rc artifacts or release artifacts from s3 based on version
56+
id: fetch_artifacts_from_s3
57+
run: |
58+
. ./.github/scripts/common/lib.sh
59+
60+
VERSION="$(get_polkadot_node_version_from_code)"
61+
echo "VERSION=${VERSION}" >> $GITHUB_OUTPUT
62+
63+
fetch_debian_package_from_s3 polkadot
64+
65+
- name: Upload artifacts
66+
uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
67+
with:
68+
name: release-artifacts
69+
path: release-artifacts/polkadot/*.deb
70+
71+
publish-deb-package:
72+
runs-on: ubuntu-latest
73+
needs: [fetch-artifacts-from-s3]
74+
environment: release
75+
env:
76+
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
77+
AWS_DEB_PATH: "s3://releases-package-repos/deb"
78+
LOCAL_DEB_REPO_PATH: ${{ github.workspace }}/deb
79+
VERSION: ${{ needs.fetch-artifacts-from-s3.outputs.VERSION }}
80+
81+
steps:
82+
- name: Install pgpkkms
83+
run: |
84+
# Install pgpkms that is used to sign built artifacts
85+
python3 -m pip install "pgpkms @ git+https://github.com/paritytech-release/pgpkms.git@1f8555426662ac93a3849480a35449f683b1c89f"
86+
echo "PGPKMS_REPREPRO_PATH=$(which pgpkms-reprepro)" >> $GITHUB_ENV
87+
88+
- name: Install awscli
89+
run: |
90+
python3 -m pip install awscli
91+
which aws
92+
93+
- name: Checkout sources
94+
uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
95+
96+
- name: Import gpg keys
97+
shell: bash
98+
run: |
99+
. ./.github/scripts/common/lib.sh
100+
101+
import_gpg_keys
102+
103+
- name: Download artifacts
104+
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
105+
with:
106+
name: release-artifacts
107+
path: release-artifacts
108+
109+
- name: Setup local deb repo
110+
run: |
111+
sudo apt-get install -y reprepro
112+
which reprepro
113+
114+
sed -i "s|^SignWith:.*|SignWith: ! ${PGPKMS_REPREPRO_PATH}|" ${{ github.workspace }}/.github/scripts/release/distributions
115+
116+
mkdir -p ${{ github.workspace }}/deb/conf
117+
cp ${{ github.workspace }}/.github/scripts/release/distributions ${{ github.workspace }}/deb/conf/distributions
118+
cat ${{ github.workspace }}/deb/conf/distributions
119+
120+
- name: Sync local deb repo
121+
env:
122+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_RELEASE_ACCESS_KEY_ID }}
123+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_RELEASE_SECRET_ACCESS_KEY }}
124+
run: |
125+
# Download the current state of the deb repo
126+
aws s3 sync "$AWS_DEB_PATH/db" "$LOCAL_DEB_REPO_PATH/db"
127+
aws s3 sync "$AWS_DEB_PATH/pool" "$LOCAL_DEB_REPO_PATH/pool"
128+
aws s3 sync "$AWS_DEB_PATH/dists" "$LOCAL_DEB_REPO_PATH/dists"
129+
130+
- name: Add deb package to local repo
131+
env:
132+
PGP_KMS_KEY: ${{ secrets.PGP_KMS_KEY }}
133+
PGP_KMS_HASH: ${{ secrets.PGP_KMS_HASH }}
134+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
135+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
136+
run: |
137+
# Add the new deb to the repo
138+
reprepro -b "$LOCAL_DEB_REPO_PATH" includedeb "${{ inputs.distribution }}" "release-artifacts/polkadot_${VERSION}_amd64.deb"
139+
140+
- name: Upload updated deb repo
141+
env:
142+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_RELEASE_ACCESS_KEY_ID }}
143+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_RELEASE_SECRET_ACCESS_KEY }}
144+
run: |
145+
# Upload the updated repo - dists and pool should be publicly readable
146+
aws s3 sync "$LOCAL_DEB_REPO_PATH/pool" "$AWS_DEB_PATH/pool" --acl public-read
147+
aws s3 sync "$LOCAL_DEB_REPO_PATH/dists" "$AWS_DEB_PATH/dists" --acl public-read
148+
aws s3 sync "$LOCAL_DEB_REPO_PATH/db" "$AWS_DEB_PATH/db"
149+
aws s3 sync "$LOCAL_DEB_REPO_PATH/conf" "$AWS_DEB_PATH/conf"
150+
151+
# Invalidate caches to make sure latest files are served
152+
aws cloudfront create-invalidation --distribution-id E36FKEYWDXAZYJ --paths '/deb/*'

0 commit comments

Comments
 (0)