Skip to content

Commit 61f7184

Browse files
jeluhujeluhu
committed
Merge 3.9.0 from backup-utils-private
2 parents 7a3f383 + f336730 commit 61f7184

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

65 files changed

+1671
-346
lines changed

‎.github/workflows/backup.yml

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
name: Backup GHES instance and save to Azure
2+
run-name: "${{ github.actor }} - Backup GHES instance and save to Azure"
3+
4+
on:
5+
workflow_call:
6+
inputs:
7+
github-hostname:
8+
description: GitHub Hostname to backup
9+
required: true
10+
type: string
11+
backup-name:
12+
description: The name of the backup to be saved in Azure storage
13+
required: false
14+
default: ""
15+
type: string
16+
secrets:
17+
BACKUP_SSH_KEY:
18+
description: SSH key to access the GitHub Enterprise instance
19+
required: true
20+
INTERNAL_ACTIONS_DX_BOT_ACCOUNT_TOKEN:
21+
description: Token for the internal actions dx bot account
22+
required: true
23+
AZURE_USERNAME:
24+
description: Azure service principal username
25+
required: false
26+
AZURE_PASSWORD:
27+
description: Azure service principal password
28+
required: false
29+
AZURE_TENANT_ID:
30+
description: Azure tenant ID
31+
required: false
32+
AZURE_SUBSCRIPTION_ID:
33+
description: Azure subscription ID
34+
required: false
35+
AZURE_ACCOUNT_NAME:
36+
description: Azure storage account name
37+
required: false
38+
AZURE_CONTAINER_NAME:
39+
description: Azure storage container name
40+
required: false
41+
CONNECTIONSTRING:
42+
description: Azure storage connection string
43+
required: false
44+
45+
46+
jobs:
47+
build:
48+
runs-on: ubuntu-latest
49+
steps:
50+
- uses: actions/checkout@v3
51+
with:
52+
repository: github/backup-utils-private
53+
token: "${{ secrets.INTERNAL_ACTIONS_DX_BOT_ACCOUNT_TOKEN }}"
54+
- run: docker build . --file Dockerfile --tag backup-utils
55+
- run: docker save backup-utils -o backup-utils.tar
56+
- uses: actions/upload-artifact@v3
57+
with:
58+
name: backup-utils
59+
path: backup-utils.tar
60+
61+
backup-utils-backup:
62+
needs: build
63+
runs-on:
64+
group: larger-hosted-public-runners
65+
labels: ubuntu-latest-xl
66+
env:
67+
SSH_KEY: ${{ secrets.BACKUP_SSH_KEY }}
68+
steps:
69+
- uses: actions/download-artifact@v3
70+
with:
71+
name: backup-utils
72+
- name: Load docker container
73+
run: docker load -i backup-utils.tar
74+
- uses: actions/checkout@v3
75+
- name: Create backup directory
76+
run: mkdir "$HOME/ghe-backup-data"
77+
- name: set up ssh SSH_KEY
78+
run: echo -e "${SSH_KEY}\n" > "$HOME/backup"
79+
- name: set up ssh key permissions
80+
run: chmod 0600 "$HOME/backup"
81+
- name: change version
82+
run: echo "3.8.0" > "$HOME/version"
83+
84+
- name: Perform backup
85+
run: |
86+
docker run -e "GHE_HOSTNAME=${{ inputs.github-hostname }}" \
87+
-e "GHE_DATA_DIR=/data" \
88+
-e "GHE_EXTRA_SSH_OPTS=-p 122 -i /ghe-ssh/id_rsa -o ServerAliveInterval=30 -o ServerAliveCountMax=12000 -o StrictHostKeyChecking=no" \
89+
-e "GHE_NUM_SNAPSHOTS=15" \
90+
-v "$HOME/ghe-backup-data:/data" \
91+
-v "$HOME/backup:/ghe-ssh/id_rsa" \
92+
-v "$HOME/version:/backup-utils/share/github-backup-utils/version" \
93+
--rm \
94+
backup-utils ghe-backup
95+
- name: Check the backup file
96+
run: |
97+
current=$(readlink "$HOME/ghe-backup-data/current")
98+
sudo tar -czvf "${{ inputs.backup-name }}.tar.gz" -C "$HOME/ghe-backup-data/$current" .
99+
100+
- name: Login to Azure
101+
if: ${{ inputs.backup-name }} != ""
102+
run: |
103+
az login \
104+
--service-principal \
105+
-u "${{ secrets.AZURE_USERNAME }}" \
106+
-p "${{ secrets.AZURE_PASSWORD }}" \
107+
--tenant "${{ secrets.AZURE_TENANT_ID }}"
108+
az account set --subscription "${{ secrets.AZURE_SUBSCRIPTION_ID }}"
109+
110+
- name: Upload backup to Azure
111+
if: ${{ inputs.backup-name }} != ""
112+
run: |
113+
az storage blob upload \
114+
--account-name "${{ secrets.AZURE_ACCOUNT_NAME }}" \
115+
--container-name "${{ secrets.AZURE_CONTAINER_NAME }}" \
116+
--name "${{ inputs.backup-name }}.tar.gz" \
117+
--file "${{ inputs.backup-name }}.tar.gz" \
118+
--connection-string "${{ secrets.CONNECTIONSTRING }}"

‎.github/workflows/backuprestore.yml

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
name: Backup and Restore
2+
run-name: ${{ github.actor }} running backup and restore operation
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
hostname:
7+
description: 'Hostname'
8+
required: true
9+
type: string
10+
ref:
11+
description: 'Ref'
12+
required: false
13+
type: string
14+
default: 'master'
15+
workflow_call:
16+
inputs:
17+
hostname:
18+
description: 'Hostname'
19+
required: true
20+
type: string
21+
ref:
22+
description: 'Ref'
23+
required: false
24+
type: string
25+
default: 'master'
26+
27+
jobs:
28+
build:
29+
runs-on: ubuntu-latest
30+
steps:
31+
- uses: actions/checkout@v3
32+
with:
33+
repository: github/backup-utils-private
34+
ref: ${{ inputs.ref }}
35+
- run: docker build . --file Dockerfile --tag backup-utils
36+
- run: docker save backup-utils -o backup-utils.tar
37+
- uses: actions/upload-artifact@v3
38+
with:
39+
name: backup-utils
40+
path: backup-utils.tar
41+
backup-utils-backup-and-restore:
42+
needs: build
43+
runs-on: ubuntu-latest
44+
env:
45+
SSH_KEY: ${{ secrets.BACKUP_SSH_KEY }}
46+
steps:
47+
- uses: actions/download-artifact@v3
48+
with:
49+
name: backup-utils
50+
- name: Load docker container
51+
run: docker load -i backup-utils.tar
52+
- uses: actions/checkout@v3
53+
- name: Create backup directory
54+
run: mkdir $HOME/ghe-backup-data
55+
- name: set up ssh SSH_KEY
56+
run: echo -e "${SSH_KEY}\n" > $HOME/backup
57+
- name: set up ssh key permissions
58+
run: chmod 0600 $HOME/backup
59+
- name: change version
60+
run: echo "3.8.0" > $HOME/version
61+
- name: Perform backup
62+
run: |
63+
docker run -e "GHE_HOSTNAME=${{ inputs.hostname }}" \
64+
-e "GHE_DATA_DIR=/data" \
65+
-e "GHE_EXTRA_SSH_OPTS=-p 122 -i /ghe-ssh/id_rsa -o ServerAliveInterval=30 -o ServerAliveCountMax=12000 -o StrictHostKeyChecking=no" \
66+
-e "GHE_NUM_SNAPSHOTS=15" \
67+
-v "$HOME/ghe-backup-data:/data" \
68+
-v "$HOME/backup:/ghe-ssh/id_rsa" \
69+
-v "$HOME/version:/backup-utils/share/github-backup-utils/version" \
70+
--rm \
71+
backup-utils ghe-backup
72+
- name: Prepare for restore
73+
run: ssh -p122 -i $HOME/backup -o StrictHostKeyChecking=no admin@${{ inputs.hostname }} "ghe-maintenance -s"
74+
- name: Restore data to instance
75+
run: |
76+
docker run -e "GHE_HOSTNAME=${{ inputs.hostname }}" \
77+
-e "GHE_DATA_DIR=/data" \
78+
-e "GHE_EXTRA_SSH_OPTS=-p 122 -i /ghe-ssh/id_rsa -o ServerAliveInterval=30 -o ServerAliveCountMax=12000 -o StrictHostKeyChecking=no" \
79+
-e "GHE_NUM_SNAPSHOTS=15" \
80+
-v "$HOME/ghe-backup-data:/data" \
81+
-v "$HOME/backup:/ghe-ssh/id_rsa" \
82+
-v "$HOME/version:/backup-utils/share/github-backup-utils/version" \
83+
--rm \
84+
backup-utils ghe-restore ${{ inputs.hostname }}
85+
- name: Reset maintenance mode after restore
86+
run: ssh -p122 -i $HOME/backup -o StrictHostKeyChecking=no admin@${{ inputs.hostname }} "ghe-maintenance -u"
87+

‎.github/workflows/docker-image.yml

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,12 +14,15 @@ jobs:
1414

1515
steps:
1616
- uses: actions/checkout@v3
17-
- name: Build the Debian Docker image
17+
- name: Build the Ubuntu Docker image
1818
run: docker build . --file Dockerfile --tag backup-utils:${GITHUB_RUN_ID}
1919
- name: Build the Alpine Docker image
2020
run: docker build . --file Dockerfile.alpine --tag backup-utils-alpine:${GITHUB_RUN_ID}
21-
- name: Run tests in Debian Docker image
22-
run: docker run backup-utils:${GITHUB_RUN_ID} ghe-backup --version
21+
- name: Run tests in Ubuntu Docker image
22+
run: |
23+
docker run backup-utils:${GITHUB_RUN_ID} ghe-backup --version
24+
docker run backup-utils:${GITHUB_RUN_ID} rsync --version
2325
- name: Run tests in Alpine Docker image
24-
run: docker run backup-utils-alpine:${GITHUB_RUN_ID} ghe-backup --version
25-
26+
run: |
27+
docker run backup-utils-alpine:${GITHUB_RUN_ID} ghe-backup --version
28+
docker run backup-utils-alpine:${GITHUB_RUN_ID} rsync --version

‎.github/workflows/main.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ jobs:
1010
# macos-latest references are kept here for historical purposes. removed macos-latest from the
1111
#matrix as it is not a typical case for users and causes a lot of friction with other linux-based
1212
# installs. Recommend developing on codespaces or using an ubuntu container.
13-
os: ['ubuntu-22.04', 'ubuntu-20.04', 'ubuntu-18.04']
13+
os: ['ubuntu-22.04', 'ubuntu-20.04']
1414
fail-fast: false
1515
runs-on: ${{ matrix.os }}
1616
steps:

‎.github/workflows/restore.yml

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
name: Restore Dataset
2+
run-name: ${{ github.actor }} retrieving data-sets
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
size:
7+
description: 'Size of the dataset to restore'
8+
required: true
9+
type: choice
10+
options:
11+
- 'small'
12+
- 'medium'
13+
hostname:
14+
description: 'Hostname of the server'
15+
required: true
16+
type: string
17+
ref:
18+
description: 'Branch ref to use'
19+
required: false
20+
type: string
21+
default: 'master'
22+
version:
23+
description: 'Version of the dataset to restore (3.8/3.9)'
24+
required: false
25+
default: '3.8'
26+
type: string
27+
28+
workflow_call:
29+
inputs:
30+
hostname:
31+
description: 'Hostname of the server'
32+
required: true
33+
type: string
34+
size:
35+
description: 'Size of the dataset to restore (small/medium)'
36+
required: false
37+
default: 'small'
38+
type: string
39+
version:
40+
description: 'Version of the dataset to restore (3.8/3.9)'
41+
required: false
42+
default: '3.8'
43+
type: string
44+
ref:
45+
description: 'Branch ref to use'
46+
required: false
47+
type: string
48+
default: 'master'
49+
50+
jobs:
51+
build:
52+
runs-on: ubuntu-latest
53+
steps:
54+
- uses: actions/checkout@v3
55+
with:
56+
repository: github/backup-utils-private
57+
ref: ${{ inputs.ref }}
58+
token: "${{ secrets.INTERNAL_ACTIONS_DX_BOT_ACCOUNT_TOKEN }}"
59+
60+
- run: docker build . --file Dockerfile --tag backup-utils
61+
- run: docker save backup-utils -o backup-utils.tar
62+
63+
- uses: actions/upload-artifact@v3
64+
with:
65+
name: backup-utils
66+
path: backup-utils.tar
67+
68+
restore:
69+
needs: build
70+
runs-on:
71+
group: larger-hosted-public-runners
72+
labels: ubuntu-latest-xl
73+
env:
74+
SSH_KEY: ${{ secrets.BACKUP_SSH_KEY }}
75+
steps:
76+
- uses: actions/download-artifact@v3
77+
with:
78+
name: backup-utils
79+
80+
- name: Load docker container
81+
run: docker load -i backup-utils.tar
82+
83+
- name: Find backup file version
84+
id: file
85+
run: |
86+
version="${{ inputs.version }}"
87+
size="${{ inputs.size }}"
88+
V3_6_COMPATIBLE="3.6 3.7"
89+
V3_8_COMPATIBLE="3.8 3.9 3.10"
90+
if echo "$V3_8_COMPATIBLE" | grep -q -w "$version"; then
91+
echo "Version $version is acceptable by 3.8 backup"
92+
file_version=3.8
93+
elif echo "$V3_6_COMPATIBLE" | grep -q -w "$version"; then
94+
echo "Version $version is acceptable by 3.6 backup"
95+
file_version=3.6
96+
else
97+
echo "Version $version is not acceptable"
98+
exit 1
99+
fi
100+
101+
echo "version=$file_version" >> "$GITHUB_OUTPUT"
102+
echo "name=v$file_version-$size.tar.gz" >> "$GITHUB_OUTPUT"
103+
104+
- name: Download from blob storage
105+
run: |
106+
mkdir ghes-data
107+
az storage blob download \
108+
--account-name ghesresults \
109+
--container-name ghes-data \
110+
--name "${{ steps.file.outputs.name }}" \
111+
--file "ghes-data/${{ steps.file.outputs.name }}" \
112+
--connection-string "${{ secrets.CONNECTIONSTRING }}"
113+
114+
- name: Unzip backup and setup symlink
115+
run: |
116+
mkdir "$HOME/ghe-backup-data"
117+
dir_name=$(date +%s)
118+
mkdir "$HOME/ghe-backup-data/$dir_name"
119+
120+
tar -xvf "ghes-data/${{ steps.file.outputs.name }}" -C "$HOME/ghe-backup-data/$dir_name"
121+
122+
ln -s "$dir_name" "$HOME/ghe-backup-data/current"
123+
124+
- name: set up ssh SSH_KEY
125+
run: echo -e "${SSH_KEY}\n" > "$HOME/backup"
126+
127+
- name: set up ssh key permissions
128+
run: chmod 0600 "$HOME/backup"
129+
130+
- name: change version
131+
run: echo "${{ inputs.version }}.0" > "$HOME/version"
132+
133+
- name: Prepare for restore
134+
run: ssh -p122 -i "$HOME/backup" -o StrictHostKeyChecking=no admin@${{ inputs.hostname }} "ghe-maintenance -s"
135+
136+
- name: Restore data to instance
137+
run: |
138+
docker run -e "GHE_HOSTNAME=${{ inputs.hostname }}" \
139+
-e "GHE_DATA_DIR=/data" \
140+
-e "GHE_EXTRA_SSH_OPTS=-p 122 -i /ghe-ssh/id_rsa -o ServerAliveInterval=30 -o ServerAliveCountMax=12000 -o StrictHostKeyChecking=no" \
141+
-e "GHE_NUM_SNAPSHOTS=15" \
142+
-v "$HOME/ghe-backup-data:/data" \
143+
-v "$HOME/backup:/ghe-ssh/id_rsa" \
144+
-v "$HOME/version:/backup-utils/share/github-backup-utils/version" \
145+
--rm \
146+
backup-utils ghe-restore ${{ inputs.hostname }}
147+
148+
- name: Reset maintenance mode after restore
149+
run: ssh -p122 -i "$HOME/backup" -o StrictHostKeyChecking=no admin@${{ inputs.hostname }} "ghe-maintenance -u"
150+
151+
152+

0 commit comments

Comments
 (0)