Skip to content

Commit f168209

Browse files
djdefidjdefi
authored
Merge branch 'master' into docker-multi
2 parents 443c8aa + cfaa9a7 commit f168209

22 files changed

+765
-104
lines changed

‎.github/workflows/backup.yml

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
name: Backup GHES instance and save to Azure
2+
run-name: "${{ github.actor }} - Backup GHES instance and save to Azure"
3+
4+
on:
5+
workflow_call:
6+
inputs:
7+
github-hostname:
8+
description: GitHub Hostname to backup
9+
required: true
10+
type: string
11+
backup-name:
12+
description: The name of the backup to be saved in Azure storage
13+
required: false
14+
default: ""
15+
type: string
16+
secrets:
17+
BACKUP_SSH_KEY:
18+
description: SSH key to access the GitHub Enterprise instance
19+
required: true
20+
INTERNAL_ACTIONS_DX_BOT_ACCOUNT_TOKEN:
21+
description: Token for the internal actions dx bot account
22+
required: true
23+
AZURE_USERNAME:
24+
description: Azure service principal username
25+
required: false
26+
AZURE_PASSWORD:
27+
description: Azure service principal password
28+
required: false
29+
AZURE_TENANT_ID:
30+
description: Azure tenant ID
31+
required: false
32+
AZURE_SUBSCRIPTION_ID:
33+
description: Azure subscription ID
34+
required: false
35+
AZURE_ACCOUNT_NAME:
36+
description: Azure storage account name
37+
required: false
38+
AZURE_CONTAINER_NAME:
39+
description: Azure storage container name
40+
required: false
41+
CONNECTIONSTRING:
42+
description: Azure storage connection string
43+
required: false
44+
45+
46+
jobs:
47+
build:
48+
runs-on: ubuntu-latest
49+
steps:
50+
- uses: actions/checkout@v3
51+
with:
52+
repository: github/backup-utils-private
53+
token: "${{ secrets.INTERNAL_ACTIONS_DX_BOT_ACCOUNT_TOKEN }}"
54+
- run: docker build . --file Dockerfile --tag backup-utils
55+
- run: docker save backup-utils -o backup-utils.tar
56+
- uses: actions/upload-artifact@v3
57+
with:
58+
name: backup-utils
59+
path: backup-utils.tar
60+
61+
backup-utils-backup:
62+
needs: build
63+
runs-on:
64+
group: larger-hosted-public-runners
65+
labels: ubuntu-latest-xl
66+
env:
67+
SSH_KEY: ${{ secrets.BACKUP_SSH_KEY }}
68+
steps:
69+
- uses: actions/download-artifact@v3
70+
with:
71+
name: backup-utils
72+
- name: Load docker container
73+
run: docker load -i backup-utils.tar
74+
- uses: actions/checkout@v3
75+
- name: Create backup directory
76+
run: mkdir "$HOME/ghe-backup-data"
77+
- name: set up ssh SSH_KEY
78+
run: echo -e "${SSH_KEY}\n" > "$HOME/backup"
79+
- name: set up ssh key permissions
80+
run: chmod 0600 "$HOME/backup"
81+
- name: change version
82+
run: echo "3.8.0" > "$HOME/version"
83+
84+
- name: Perform backup
85+
run: |
86+
docker run -e "GHE_HOSTNAME=${{ inputs.github-hostname }}" \
87+
-e "GHE_DATA_DIR=/data" \
88+
-e "GHE_EXTRA_SSH_OPTS=-p 122 -i /ghe-ssh/id_rsa -o ServerAliveInterval=30 -o ServerAliveCountMax=12000 -o StrictHostKeyChecking=no" \
89+
-e "GHE_NUM_SNAPSHOTS=15" \
90+
-v "$HOME/ghe-backup-data:/data" \
91+
-v "$HOME/backup:/ghe-ssh/id_rsa" \
92+
-v "$HOME/version:/backup-utils/share/github-backup-utils/version" \
93+
--rm \
94+
backup-utils ghe-backup
95+
- name: Check the backup file
96+
run: |
97+
current=$(readlink "$HOME/ghe-backup-data/current")
98+
sudo tar -czvf "${{ inputs.backup-name }}.tar.gz" -C "$HOME/ghe-backup-data/$current" .
99+
100+
- name: Login to Azure
101+
if: ${{ inputs.backup-name }} != ""
102+
run: |
103+
az login \
104+
--service-principal \
105+
-u "${{ secrets.AZURE_USERNAME }}" \
106+
-p "${{ secrets.AZURE_PASSWORD }}" \
107+
--tenant "${{ secrets.AZURE_TENANT_ID }}"
108+
az account set --subscription "${{ secrets.AZURE_SUBSCRIPTION_ID }}"
109+
110+
- name: Upload backup to Azure
111+
if: ${{ inputs.backup-name }} != ""
112+
run: |
113+
az storage blob upload \
114+
--account-name "${{ secrets.AZURE_ACCOUNT_NAME }}" \
115+
--container-name "${{ secrets.AZURE_CONTAINER_NAME }}" \
116+
--name "${{ inputs.backup-name }}.tar.gz" \
117+
--file "${{ inputs.backup-name }}.tar.gz" \
118+
--connection-string "${{ secrets.CONNECTIONSTRING }}"

‎.github/workflows/lint.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ jobs:
1717
# Full git history is needed to get a proper list of changed files within `super-linter`
1818
fetch-depth: 0
1919
- name: Lint Code Base
20-
uses: github/super-linter@v4
20+
uses: github/super-linter@v5
2121
env:
2222
VALIDATE_ALL_CODEBASE: false
2323
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

‎.github/workflows/restore.yml

Lines changed: 93 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
name: Restore Dataset
2-
run-name: ${{ github.actor }} retrieving data-sets
2+
run-name: ${{ github.actor }} retrieving data-sets
33
on:
44
workflow_dispatch:
55
inputs:
@@ -19,32 +19,34 @@ on:
1919
required: false
2020
type: string
2121
default: 'master'
22-
version:
23-
description: 'GHES Version of dataset'
22+
version:
23+
description: 'Version of the dataset to restore (3.8/3.9)'
2424
required: false
25+
default: '3.8'
2526
type: string
26-
default: "3.8.0"
27+
2728
workflow_call:
2829
inputs:
29-
size:
30-
description: 'Size of the dataset to restore'
31-
required: false
32-
type: string
33-
default: 'small'
3430
hostname:
3531
description: 'Hostname of the server'
3632
required: true
3733
type: string
34+
size:
35+
description: 'Size of the dataset to restore (small/medium)'
36+
required: false
37+
default: 'small'
38+
type: string
39+
version:
40+
description: 'Version of the dataset to restore (3.8/3.9)'
41+
required: false
42+
default: '3.8'
43+
type: string
3844
ref:
3945
description: 'Branch ref to use'
4046
required: false
4147
type: string
4248
default: 'master'
43-
version:
44-
description: 'GHES Version of dataset'
45-
required: false
46-
type: string
47-
default: "3.8.0"
49+
4850
jobs:
4951
build:
5052
runs-on: ubuntu-latest
@@ -53,30 +55,98 @@ jobs:
5355
with:
5456
repository: github/backup-utils-private
5557
ref: ${{ inputs.ref }}
56-
- run: docker build . --file Dockerfile --tag backup-utils
58+
token: "${{ secrets.INTERNAL_ACTIONS_DX_BOT_ACCOUNT_TOKEN }}"
59+
60+
- run: docker build . --file Dockerfile --tag backup-utils
5761
- run: docker save backup-utils -o backup-utils.tar
62+
5863
- uses: actions/upload-artifact@v3
5964
with:
6065
name: backup-utils
6166
path: backup-utils.tar
67+
6268
restore:
6369
needs: build
64-
runs-on: ubuntu-latest
70+
runs-on:
71+
group: larger-hosted-public-runners
72+
labels: ubuntu-latest-xl
6573
env:
6674
SSH_KEY: ${{ secrets.BACKUP_SSH_KEY }}
6775
steps:
6876
- uses: actions/download-artifact@v3
6977
with:
7078
name: backup-utils
71-
- name: Load docker container
79+
80+
- name: Load docker container
7281
run: docker load -i backup-utils.tar
73-
- uses: actions/checkout@v3
74-
with:
75-
repository: github/ghes-data
76-
ref: main
77-
- run: tar zxvf ghes-data/data/backup/${{ inputs.size }}/v3.8.0/${{ inputs.size }}-refined.tar.gz
78-
- run: ls -al ghes-data/data/backup/${{ inputs.size }}
7982

83+
- name: Find backup file version
84+
id: file
85+
run: |
86+
version="${{ inputs.version }}"
87+
size="${{ inputs.size }}"
88+
V3_6_COMPATIBLE="3.6 3.7"
89+
V3_8_COMPATIBLE="3.8 3.9 3.10"
90+
if echo "$V3_8_COMPATIBLE" | grep -q -w "$version"; then
91+
echo "Version $version is acceptable by 3.8 backup"
92+
file_version=3.8
93+
elif echo "$V3_6_COMPATIBLE" | grep -q -w "$version"; then
94+
echo "Version $version is acceptable by 3.6 backup"
95+
file_version=3.6
96+
else
97+
echo "Version $version is not acceptable"
98+
exit 1
99+
fi
100+
101+
echo "version=$file_version" >> "$GITHUB_OUTPUT"
102+
echo "name=v$file_version-$size.tar.gz" >> "$GITHUB_OUTPUT"
103+
104+
- name: Download from blob storage
105+
run: |
106+
mkdir ghes-data
107+
az storage blob download \
108+
--account-name ghesresults \
109+
--container-name ghes-data \
110+
--name "${{ steps.file.outputs.name }}" \
111+
--file "ghes-data/${{ steps.file.outputs.name }}" \
112+
--connection-string "${{ secrets.CONNECTIONSTRING }}"
113+
114+
- name: Unzip backup and setup symlink
115+
run: |
116+
mkdir "$HOME/ghe-backup-data"
117+
dir_name=$(date +%s)
118+
mkdir "$HOME/ghe-backup-data/$dir_name"
119+
120+
tar -xvf "ghes-data/${{ steps.file.outputs.name }}" -C "$HOME/ghe-backup-data/$dir_name"
121+
122+
ln -s "$dir_name" "$HOME/ghe-backup-data/current"
123+
124+
- name: set up ssh SSH_KEY
125+
run: echo -e "${SSH_KEY}\n" > "$HOME/backup"
126+
127+
- name: set up ssh key permissions
128+
run: chmod 0600 "$HOME/backup"
129+
130+
- name: change version
131+
run: echo "${{ inputs.version }}.0" > "$HOME/version"
132+
133+
- name: Prepare for restore
134+
run: ssh -p122 -i "$HOME/backup" -o StrictHostKeyChecking=no admin@${{ inputs.hostname }} "ghe-maintenance -s"
135+
136+
- name: Restore data to instance
137+
run: |
138+
docker run -e "GHE_HOSTNAME=${{ inputs.hostname }}" \
139+
-e "GHE_DATA_DIR=/data" \
140+
-e "GHE_EXTRA_SSH_OPTS=-p 122 -i /ghe-ssh/id_rsa -o ServerAliveInterval=30 -o ServerAliveCountMax=12000 -o StrictHostKeyChecking=no" \
141+
-e "GHE_NUM_SNAPSHOTS=15" \
142+
-v "$HOME/ghe-backup-data:/data" \
143+
-v "$HOME/backup:/ghe-ssh/id_rsa" \
144+
-v "$HOME/version:/backup-utils/share/github-backup-utils/version" \
145+
--rm \
146+
backup-utils ghe-restore ${{ inputs.hostname }}
147+
148+
- name: Reset maintenance mode after restore
149+
run: ssh -p122 -i "$HOME/backup" -o StrictHostKeyChecking=no admin@${{ inputs.hostname }} "ghe-maintenance -u"
80150

81151

82152

‎backup.config-example

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,3 +95,6 @@ GHE_NUM_SNAPSHOTS=10
9595
# When running an external mysql database, run this script to trigger a MySQL restore
9696
# rather than attempting to backup via backup-utils directly.
9797
#EXTERNAL_DATABASE_RESTORE_SCRIPT="/bin/false"
98+
99+
# If set to 'yes', Pages data will be included in backup and restore. Defaults to 'yes'
100+
#GHE_BACKUP_PAGES=no

‎bin/ghe-backup

Lines changed: 33 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,30 @@ if [ -n "$GHE_ALLOW_REPLICA_BACKUP" ]; then
155155
echo "Warning: backing up a high availability replica may result in inconsistent or unreliable backups."
156156
fi
157157

158+
# Output system information of the backup host
159+
160+
# If /etc/issue.net exists, use it to get the OS version
161+
if [ -f /etc/issue.net ]; then
162+
echo "Running on: $(cat /etc/issue.net)"
163+
else
164+
echo "Running on: Unknown OS"
165+
fi
166+
167+
# If nproc command exists, use it to get the number of CPUs
168+
if command -v nproc >/dev/null 2>&1; then
169+
echo "CPUs: $(nproc)"
170+
else
171+
echo "CPUs: Unknown"
172+
fi
173+
174+
# If the free command exists, use it to get the memory details
175+
if command -v free >/dev/null 2>&1; then
176+
echo "Memory $(free -m | grep '^Mem:' | awk '{print "total/used/free+share/buff/cache: " $2 "/" $3 "/" $4 "+" $5 "/" $6 "/" $7}')"
177+
else
178+
echo "Memory: Unknown"
179+
fi
180+
181+
158182
# Log backup start message in /var/log/syslog on remote instance
159183
ghe_remote_logger "Starting backup from $(hostname) with backup-utils v$BACKUP_UTILS_VERSION in snapshot $GHE_SNAPSHOT_TIMESTAMP ..."
160184

@@ -214,10 +238,15 @@ commands+=("
214238
echo \"$cmd_title\"
215239
ghe-backup-repositories || printf %s \"repositories \" >> \"$failures_file\"")
216240

217-
cmd_title=$(log_info "Backing up GitHub Pages artifacts ...")
218-
commands+=("
219-
echo \"$cmd_title\"
220-
ghe-backup-pages || printf %s \"pages \" >> \"$failures_file\"")
241+
# Pages backups are skipped only if GHE_BACKUP_PAGES is explicitly set to 'no' to guarantee backward compatibility.
242+
# If a customer upgrades backup-utils but keeps the config file from a previous version, Pages backups still work as expected.
243+
244+
if [ "$GHE_BACKUP_PAGES" != "no" ]; then
245+
cmd_title=$(log_info "Backing up GitHub Pages artifacts ...")
246+
commands+=("
247+
echo \"$cmd_title\"
248+
ghe-backup-pages || printf %s \"pages \" >> \"$failures_file\"")
249+
fi
221250

222251
cmd_title=$(log_info "Backing up storage data ...")
223252
commands+=("

0 commit comments

Comments
 (0)