Skip to content

Commit f90be15

Browse files
igorpecovnikigorpecovnik
committed
Experiments
1 parent 194a0ab commit f90be15

File tree

2 files changed

+259
-0
lines changed

2 files changed

+259
-0
lines changed

‎.github/workflows/mirror.yml

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
# .github/workflows/mirror.yml
2+
name: "Mirror repository artifacts"
3+
on:
4+
repository_dispatch:
5+
types: ["Mirror"]
6+
7+
env:
8+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
9+
PULL_REPOSITORY: "${{ github.event.client_payload.pull_repository }}" # username/repository
10+
CDN_TAG: "${{ github.event.client_payload.cdn_tag }}" # folder on server
11+
PULL_TAG: "" # leave empty for latest
12+
13+
jobs:
14+
15+
prepare:
16+
name: "Split JSON into parts"
17+
runs-on: "ubuntu-24.04"
18+
outputs:
19+
matrix: ${{steps.json.outputs.JSON_CONTENT}}
20+
steps:
21+
- name: "Install SSH key"
22+
uses: shimataro/ssh-key-action@v2
23+
with:
24+
key: "${{ secrets.KEY_UPLOAD }}"
25+
known_hosts: "${{ secrets.KNOWN_HOSTS_ARMBIAN_UPLOAD }}"
26+
if_key_exists: replace
27+
- name: "Get latest release TAG"
28+
run: |
29+
echo "LATEST=$(gh release list --repo https://github.com/${{ env.PULL_REPOSITORY }} --json isLatest,tagName | jq -r '.[] | select(.isLatest == true) | .tagName')" >> $GITHUB_ENV
30+
- name: "Get upload servers"
31+
run: |
32+
curl -H "Authorization: Token ${{ secrets.NETBOX_TOKEN }}" \
33+
-H "Accept: application/json; indent=4" \
34+
"https://stuff.armbian.com/netbox/api/virtualization/virtual-machines/?limit=500&name__empty=false&status=active" | \
35+
jq '.results[] | select(.tags.[].name == "cache") | {name, custom_fields}' > servers.json
36+
- name: "Get release artefacts"
37+
run: |
38+
gh release view --json \
39+
assets --repo https://github.com/${{ env.PULL_REPOSITORY }} | \
40+
jq '.assets[] | { "tag": "'${{ env.LATEST }}'","cdn_tag": "'${{ env.CDN_TAG }}'", name: .name, url: .url, size: .size}' | \
41+
jq -s > artifacts.json
42+
- name: "Split JSON file into parts"
43+
run: |
44+
cat >> "split.py" <<- EOT
45+
import json
46+
def split_json(input_file, output_prefix, chunk_size):
47+
# Open and load the large JSON file
48+
with open(input_file, 'r') as infile:
49+
data = json.load(infile)
50+
# Split the data into chunks
51+
total_chunks = len(data) // chunk_size + (1 if len(data) % chunk_size != 0 else 0)
52+
for i in range(total_chunks):
53+
chunk = data[i * chunk_size:(i + 1) * chunk_size]
54+
output_filename = f"{output_prefix}_{i + 1}.json"
55+
# Write each chunk into a separate JSON file
56+
with open(output_filename, 'w') as outfile:
57+
json.dump(chunk, outfile, indent=4)
58+
split_json('artifacts.json', 'part', 200)
59+
EOT
60+
python3 split.py
61+
- name: "Upload JSON parts"
62+
uses: actions/upload-artifact@v4
63+
with:
64+
name: parts
65+
path: |
66+
part*
67+
servers.json
68+
if-no-files-found: ignore
69+
compression-level: 9
70+
- name: "Make JSON"
71+
id: json
72+
run: |
73+
echo 'JSON_CONTENT<<EOF' >> $GITHUB_OUTPUT
74+
find * -name "part*.json" -type f -exec stat --format '{"latest": "'${{ env.LATEST }}'","cdn_tag": "'${{ env.CDN_TAG }}'","name": "%n"}' {} \; | jq -s . >> $GITHUB_OUTPUT
75+
echo 'EOF' >> $GITHUB_OUTPUT
76+
gradle:
77+
needs: prepare
78+
name: "JSON: ${{ matrix.name }}"
79+
strategy:
80+
fail-fast: false
81+
matrix:
82+
include: ${{ fromJson(needs.prepare.outputs.matrix) }}
83+
uses: armbian/armbian.github.io/.github/workflows/reusable.yml@mirror
84+
with:
85+
name: "${{ matrix.name }}"
86+
cdntag: "${{ matrix.cdn_tag }}"
87+
latest: "${{ matrix.latest }}"
88+
secrets:
89+
KEY_UPLOAD: ${{ secrets.KEY_UPLOAD }}
90+
KNOWN_HOSTS_ARMBIAN_UPLOAD: ${{ secrets.KNOWN_HOSTS_ARMBIAN_UPLOAD }}
91+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
92+
93+
cleanup:
94+
needs: gradle
95+
name: "Cleaning"
96+
runs-on: "ubuntu-24.04"
97+
steps:
98+
- name: "Install SSH key"
99+
uses: shimataro/ssh-key-action@v2
100+
with:
101+
key: "${{ secrets.KEY_UPLOAD }}"
102+
known_hosts: "${{ secrets.KNOWN_HOSTS_ARMBIAN_UPLOAD }}"
103+
if_key_exists: replace
104+
- name: Download ${{ matrix.name }}
105+
uses: actions/download-artifact@v4
106+
with:
107+
name: parts
108+
pattern: part*
109+
merge-multiple: true
110+
- name: "Make JSON"
111+
id: json
112+
run: |
113+
LATEST=$(cat part*.json | jq -r '.[].tag' | sort | uniq)
114+
CDN_TAG=$(cat part*.json | jq -r '.[].cdn_tag' | sort | uniq)
115+
echo "Delete all except $LATEST on $CDN_TAG"
116+
tempfolder=$(mktemp -d)
117+
rsync --delete -e \
118+
"ssh -p 10023 -o StrictHostKeyChecking=accept-new" \
119+
-rvP ${tempfolder}/ "[email protected]:/storage/www/cache/${CDN_TAG}" --exclude=${LATEST}
120+
121+
- uses: geekyeggo/delete-artifact@v5
122+
with:
123+
name: |
124+
parts*

‎.github/workflows/reusable.yml

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
# .github/workflows/reusable.yml
2+
name: "Reusable Workflow for Mirroring"
3+
on:
4+
workflow_call:
5+
inputs:
6+
name:
7+
description: 'Name'
8+
required: true
9+
type: string
10+
cdntag:
11+
description: 'Folder on CDN'
12+
required: true
13+
type: string
14+
latest:
15+
description: 'Release tag'
16+
required: true
17+
type: string
18+
secrets:
19+
KEY_UPLOAD:
20+
required: true
21+
KNOWN_HOSTS_ARMBIAN_UPLOAD:
22+
required: true
23+
GH_TOKEN:
24+
required: true
25+
26+
jobs:
27+
example-job:
28+
name: "Download"
29+
runs-on: ubuntu-latest
30+
outputs:
31+
DEPLOYMENT_MATRIX: "${{ steps.files.outputs.DEPLOYMENT_MATRIX }}"
32+
steps:
33+
- name: Download ${{ matrix.name }}
34+
uses: actions/download-artifact@v4
35+
with:
36+
name: parts
37+
pattern: part*
38+
- name: "Get devices from database ${{ inputs.name }}"
39+
id: files
40+
run: |
41+
delimiter="$(openssl rand -hex 8)"
42+
echo "DEPLOYMENT_MATRIX<<${delimiter}" >> "${GITHUB_OUTPUT}"
43+
sed -i -e '$a\' "${{ inputs.name }}"
44+
cat "${{ inputs.name }}" | jq >> "${GITHUB_OUTPUT}"
45+
echo "${delimiter}" >> "${GITHUB_OUTPUT}"
46+
47+
gradle:
48+
name: "${{ matrix.base.name }}"
49+
runs-on: "ubuntu-24.04"
50+
needs: example-job
51+
if: ${{ needs.example-job.outputs.DEPLOYMENT_MATRIX != '[]' }}
52+
timeout-minutes: 20
53+
strategy:
54+
max-parallel: 16
55+
fail-fast: false
56+
matrix:
57+
base: ${{ fromJSON(needs.example-job.outputs.DEPLOYMENT_MATRIX) }}
58+
steps:
59+
60+
- name: Download ${{ matrix.name }}
61+
uses: actions/download-artifact@v4
62+
with:
63+
name: parts
64+
pattern: servers.json
65+
66+
- name: Check API rate limits
67+
env:
68+
GH_TOKEN: ${{ secrets.GH_TOKEN }}
69+
run: |
70+
71+
while true
72+
do
73+
API_CALLS_TOTAL=$(gh api -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" /rate_limit | jq -r '.rate.limit')
74+
API_CALLS_LEFT=$(gh api -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" /rate_limit | jq -r '.rate.remaining')
75+
PERCENT=$(( API_CALLS_LEFT * 100 / API_CALLS_TOTAL ))
76+
if (( $PERCENT > 20 )); then
77+
echo "API rate in good shape $PERCENT % free"
78+
exit 0
79+
fi
80+
echo "API rate lower then 20%, sleping 10m"
81+
sleep 10m
82+
done
83+
# show current api rate
84+
curl -s -H "Accept: application/vnd.github.v3+json" -H "Authorization: token ${{ secrets.ACCESS_TOKEN }}" https://api.github.com/rate_limit
85+
86+
- name: Install SSH key
87+
uses: shimataro/ssh-key-action@v2
88+
with:
89+
key: ${{ secrets.KEY_UPLOAD }}
90+
known_hosts: ${{ secrets.KNOWN_HOSTS_ARMBIAN_UPLOAD }}
91+
if_key_exists: replace
92+
93+
- name: "Download artifact ${{ matrix.base.name }} from Git"
94+
run: |
95+
96+
curl \
97+
--progress-bar \
98+
--create-dir \
99+
--output-dir ${{ inputs.cdntag }}/${{ inputs.latest }} \
100+
-o ${{ matrix.base.name }} \
101+
-L ${{ matrix.base.url }}
102+
ls -l
103+
ls -l ${{ inputs.cdntag }}
104+
chmod -R 755 ${{ inputs.cdntag }}
105+
106+
- name: "Upload artifact to CDN"
107+
run: |
108+
109+
# we use servers.json definitions that was build in main workflow
110+
# so we don't call API for every file
111+
for row in $( cat servers.json | jq -r '@base64'); do
112+
# Decode the base64 encoded JSON and extract values
113+
_jq() {
114+
echo ${row} | base64 --decode | jq -r ${1}
115+
}
116+
# Extract values from each item
117+
id=$(_jq '.id')
118+
name=$(_jq '.name')
119+
path=$(_jq '.custom_fields.path')
120+
port=$(_jq '.custom_fields.port')
121+
download_path_archive=$(_jq '.custom_fields.download_path_archive')
122+
download_path_debs=$(_jq '.custom_fields.download_path_debs')
123+
download_path_images=$(_jq '.custom_fields.download_path_images')
124+
known_hosts=$(_jq '.custom_fields.known_hosts')
125+
path=$(_jq '.custom_fields.path')
126+
port=$(_jq '.custom_fields.port')
127+
runners=$(_jq '.custom_fields.runners')
128+
username=$(_jq '.custom_fields.username')
129+
weight=$(_jq '.custom_fields.weight')
130+
131+
# rsync
132+
rsync --progress -e \
133+
"ssh -p ${port} -o StrictHostKeyChecking=accept-new" \
134+
-ArvP . "${username}@${name}:${path}/cache" --exclude='servers.json' --exclude='part*.json'
135+
done

0 commit comments

Comments
 (0)