Skip to content

Commit 18f02ea

Browse files
gamefiendgamefiend
authored
Merge branch 'master' into pluehne/record-total-runtime-of-ghe-backup-and-ghe-restore
2 parents b9e6a58 + 1d924a1 commit 18f02ea

18 files changed

+426
-68
lines changed

‎.github/pull_request_template.md

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
# PR Details
2+
3+
### Description
4+
<!--
5+
[Please fill out a brief description of the change being made]
6+
-->
7+
### Testing
8+
<!--
9+
[Please add testing done as part of this change.]
10+
-->
11+
<!-- Keep in mind that for backup-utils the following applies:
12+
- Backup-util [current version] will support
13+
- GHES [current version]
14+
- GHES [current version -1]
15+
- GHES [current version -2]
16+
- Any changes that are made to backup-utils will also need to be supported on those GHES versions above (n-2)
17+
- Please make sure those versions are tested against for this change
18+
-->
19+
20+
### Ownership
21+
<!-- [Add any relevants owners for this change]
22+
-->
23+
24+
### Related Links
25+
<!-- [Please add any related links/issues to this PR]
26+
-->

‎.github/workflows/build-and-release.yml

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
name: Build and Release
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
version:
7+
description: 'Version - patch version of the release (e.g. x.y.z)'
8+
required: true
9+
type: string
10+
release-notes:
11+
description: 'Release Notes - string of markdown'
12+
required: true
13+
type: string
14+
draft:
15+
description: 'Draft - true if the release should be a draft'
16+
required: true
17+
type: boolean
18+
default: true
19+
20+
jobs:
21+
build:
22+
runs-on: ubuntu-latest
23+
steps:
24+
- name: Install dependencies
25+
run: |
26+
sudo apt-get update
27+
sudo apt-get install -y moreutils debhelper help2man
28+
- name: Create tag
29+
run: |
30+
git tag -a v${{ github.event.inputs.version }} -m "v${{ github.event.inputs.version }}"
31+
- name: Checkout
32+
uses: actions/checkout@v2
33+
- name: Package deb
34+
run: |
35+
bash scripts/package-deb
36+
- name: Package tarball
37+
run: |
38+
bash scripts/package-tarball
39+
release:
40+
needs: build
41+
runs-on: ubuntu-latest
42+
steps:
43+
- name: Create Release
44+
uses: ncipollo/release-action@v1
45+
with:
46+
repo: github/backup-utils
47+
artifacts: |
48+
./dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz
49+
./dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb
50+
# token: ${{ secrets.GITHUB_TOKEN }} may need token, but try without first
51+
name: v${{ github.event.inputs.version }}
52+
draft: ${{ github.event.inputs.draft }}
53+
body: ${{ github.event.inputs.release-notes }}

‎.github/workflows/main.yml

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,6 @@ jobs:
66
build:
77
strategy:
88
matrix:
9-
# macos-latest references are kept here for historical purposes. removed macos-latest from the
10-
#matrix as it is not a typical case for users and causes a lot of friction with other linux-based
11-
# installs. Recommend developing on codespaces or using an ubuntu container.
129
os: ['ubuntu-22.04', 'ubuntu-20.04']
1310
fail-fast: false
1411
runs-on: ${{ matrix.os }}
@@ -20,12 +17,6 @@ jobs:
2017
wget "https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.x86_64.tar.xz"
2118
tar --xz -xvf "shellcheck-stable.linux.x86_64.tar.xz"
2219
sudo cp shellcheck-stable/shellcheck /usr/bin/shellcheck
23-
if: matrix.os != 'macos-latest'
24-
- name: Install Dependencies (macOS)
25-
run: |
26-
brew install gnu-tar shellcheck jq pigz coreutils gnu-sed gnu-getopt wget
27-
brew install moreutils gawk
28-
if: matrix.os == 'macos-latest'
2920
- name: Get Sources
3021
uses: actions/checkout@v3
3122
- name: Test
@@ -35,4 +26,3 @@ jobs:
3526
shell: bash
3627
- name: Build (Linux)
3728
run: DEB_BUILD_OPTIONS=nocheck debuild -us -uc
38-
if: matrix.os != 'macos-latest'

‎bin/ghe-backup

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -165,11 +165,11 @@ echo "$GHE_REMOTE_VERSION" > version
165165
# Setup progress tracking
166166
init-progress
167167
export PROGRESS_TOTAL=14 # Minimum number of steps in backup is 14
168-
echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total
168+
echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress/total
169169
export PROGRESS_TYPE="Backup"
170-
echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress-type
170+
echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress/type
171171
export PROGRESS=0 # Used to track progress of backup
172-
echo "$PROGRESS" > /tmp/backup-utils-progress
172+
echo "$PROGRESS" > /tmp/backup-utils-progress/progress
173173

174174
OPTIONAL_STEPS=0
175175
# Backup actions+mssql
@@ -193,7 +193,7 @@ if [ "$GHE_BACKUP_PAGES" != "no" ]; then
193193
fi
194194

195195
PROGRESS_TOTAL=$((OPTIONAL_STEPS + PROGRESS_TOTAL)) # Minimum number of steps in backup is 14
196-
echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total
196+
echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress/total
197197

198198
# check that incremental settings are valid if set
199199
is_inc=$(is_incremental_backup_feature_on)

‎bin/ghe-backup-progress

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,26 +31,26 @@ while true; do
3131
done
3232

3333
check_for_progress_file() {
34-
if [ ! -f /tmp/backup-utils-progress-info ]; then
34+
if [ ! -f /tmp/backup-utils-progress/info ]; then
3535
echo "No progress file found. Has a backup or restore been started?"
3636
exit 1
3737
fi
3838
}
3939

4040
if [ -n "$ONCE" ]; then
4141
check_for_progress_file
42-
cat /tmp/backup-utils-progress-info
42+
cat /tmp/backup-utils-progress/info
4343
else
4444
check_for_progress_file
4545
clear
46-
cat /tmp/backup-utils-progress-info
46+
cat /tmp/backup-utils-progress/info
4747
while true; do
4848
if read -r -t 1 -n 1; then
4949
clear
5050
exit ;
5151
else
5252
clear
53-
cat /tmp/backup-utils-progress-info
53+
cat /tmp/backup-utils-progress/info
5454
fi
5555
done
5656
fi

‎bin/ghe-restore

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -317,11 +317,11 @@ fi
317317
export PROGRESS_TOTAL=$((OPTIONAL_STEPS + 7))
318318

319319
init-progress
320-
echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total
320+
echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress/total
321321
export PROGRESS_TYPE="Restore"
322-
echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress-type
322+
echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress/type
323323
export PROGRESS=0 # Used to track progress of restore
324-
echo "$PROGRESS" > /tmp/backup-utils-progress
324+
echo "$PROGRESS" > /tmp/backup-utils-progress/progress
325325

326326
# Log restore start message locally and in /var/log/syslog on remote instance
327327
bm_start "$(basename $0)"

‎script/cibuild

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ set -e
55
# Enable verbose logging of ssh commands
66
export GHE_VERBOSE_SSH=true
77

8-
if ! find test -name "test-*.sh" -print0 | xargs -0 -P 4 -n 1 /bin/bash; then
8+
if ! find test -name "test-*.sh" -print0 | xargs -0 -n 1 /bin/bash; then
99
exit 1
1010
fi
1111

‎share/github-backup-utils/ghe-backup-config

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -217,8 +217,10 @@ ghe_parallel_check() {
217217
GHE_PARALLEL_COMMAND="parallel"
218218
local x
219219
for x in \
220+
/usr/bin/parallel-moreutils \
220221
/usr/bin/parallel.moreutils \
221222
/usr/bin/parallel_moreutils \
223+
/usr/bin/moreutils-parallel \
222224
/usr/bin/moreutils.parallel \
223225
/usr/bin/moreutils_parallel \
224226
; do
@@ -650,12 +652,21 @@ restore-secret() {
650652

651653
#initialize progress tracking by clearing out the temp files used to track
652654
init-progress() {
653-
rm -f /tmp/backup-utils-progress*
655+
if [ -d /tmp/backup-utils-progress ]; then
656+
rm -rf /tmp/backup-utils-progress/*
657+
else
658+
mkdir /tmp/backup-utils-progress
659+
fi
660+
touch /tmp/backup-utils-progress/total
661+
touch /tmp/backup-utils-progress/type
662+
touch /tmp/backup-utils-progress/progress
663+
touch /tmp/backup-utils-progress/info
664+
chmod -R 777 /tmp/backup-utils-progress
654665
}
655666

656667

657668
#increase total count of progress
658669
increment-progress-total-count() {
659670
((PROGRESS_TOTAL += $1))
660-
echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total
671+
echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress/total
661672
}

‎share/github-backup-utils/ghe-backup-settings

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,10 @@ if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.8.0)" ]; then
9494
backup-secret "secret scanning encrypted secrets delimited shared transit keys" "secret-scanning-encrypted-secrets-delimited-shared-transit-keys" "secrets.secret-scanning.encrypted-secrets-delimited-shared-transit-keys"
9595
fi
9696

97+
if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.11.0)" ]; then
98+
backup-secret "secret scanning encrypted content keys" "secret-scanning-user-content-delimited-encryption-root-keys" "secrets.secret-scanning.secret-scanning-user-content-delimited-encryption-root-keys"
99+
fi
100+
97101
# Backup argon secrets for multiuser from ghes version 3.8 onwards
98102
if [[ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.8.0)" && "$(version $GHE_REMOTE_VERSION)" -lt "$(version 3.8.2)" ]]; then
99103
backup-secret "management console argon2 secret" "manage-argon-secret" "secrets.manage-auth.argon-secret"

‎share/github-backup-utils/ghe-restore-repositories

Lines changed: 53 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -104,42 +104,60 @@ done > $tmp_list
104104
IFS=$OLDIFS
105105
bm_end "$(basename $0) - Building network list"
106106

107-
# The server returns a list of routes:
108-
#
109-
# a/nw/a8/3f/02/100000855 dgit-node1 dgit-node2 dgit-node3
110-
# a/nw/a8/bc/8d/100000880 dgit-node1 dgit-node2 dgit-node4
111-
# a/nw/a5/06/81/100000659 dgit-node3 dgit-node2 dgit-node4
112-
# ...
113-
#
114-
# One route per line.
115-
#
116-
# NOTE: The route generation is performed on the appliance as it is considerably
117-
# more performant than performing over an SSH pipe.
118-
#
119-
bm_start "$(basename $0) - Transferring network list"
120-
cat $tmp_list | ghe-ssh "$GHE_HOSTNAME" -- sponge $remote_tmp_list
121-
cat $tmp_list | ghe_debug
122-
bm_end "$(basename $0) - Transferring network list"
123-
124-
bm_start "$(basename $0) - Generating routes"
125-
restore_routes_script="github-env ./bin/dgit-cluster-restore-routes"
126-
if ghe-ssh "$GHE_HOSTNAME" test -e /usr/local/share/enterprise/ghe-restore-network-routes; then
127-
restore_routes_script="/usr/local/share/enterprise/ghe-restore-network-routes"
107+
# In cluster environments, we need to ensure that all repository networks are replicated back to the
108+
# same Spokes nodes that they were present on when the backup was taken. For this, the list of
109+
# routes of each repository network is first obtained. Afterward, an rsync file list is created for
110+
# each Spokes node including only those repository networks for which there was a route to the
111+
# respective Spokes node.
112+
if $CLUSTER; then
113+
log_info "* Restoring repository networks to cluster nodes according to Spokes routes" 1>&3
114+
115+
# The server returns a list of routes:
116+
#
117+
# a/nw/a8/3f/02/100000855 dgit-node1 dgit-node2 dgit-node3
118+
# a/nw/a8/bc/8d/100000880 dgit-node1 dgit-node2 dgit-node4
119+
# a/nw/a5/06/81/100000659 dgit-node3 dgit-node2 dgit-node4
120+
# ...
121+
#
122+
# One route per line.
123+
#
124+
# NOTE: The route generation is performed on the appliance as it is considerably
125+
# more performant than performing over an SSH pipe.
126+
#
127+
bm_start "$(basename $0) - Transferring network list"
128+
cat $tmp_list | ghe-ssh "$GHE_HOSTNAME" -- sponge $remote_tmp_list
129+
cat $tmp_list | ghe_debug
130+
bm_end "$(basename $0) - Transferring network list"
131+
132+
bm_start "$(basename $0) - Generating routes"
133+
restore_routes_script="github-env ./bin/dgit-cluster-restore-routes"
134+
if ghe-ssh "$GHE_HOSTNAME" test -e /usr/local/share/enterprise/ghe-restore-network-routes; then
135+
restore_routes_script="/usr/local/share/enterprise/ghe-restore-network-routes"
136+
fi
137+
echo "cat $remote_tmp_list | $restore_routes_script | grep 'git-server-' > $remote_routes_list" | ghe-ssh "$GHE_HOSTNAME" -- /bin/bash
138+
ghe-ssh "$GHE_HOSTNAME" -- cat $remote_routes_list | ghe_debug
139+
bm_end "$(basename $0) - Generating routes"
140+
141+
bm_start "$(basename $0) - Fetching routes"
142+
ghe-ssh "$GHE_HOSTNAME" -- gzip -c $remote_routes_list | gzip -d > $routes_list
143+
cat $routes_list | ghe_debug
144+
bm_end "$(basename $0) - Fetching routes"
145+
146+
bm_start "$(basename $0) - Processing routes"
147+
148+
cat $routes_list | awk -v tempdir="$tempdir" '{ for(i=2;i<=NF;i++){ print $1 > (tempdir"/"$i".rsync") }}'
149+
cat $routes_list | awk '{ n = split($1, p, "/"); printf p[n] " /data/repositories/" $1; $1=""; print $0}' > $to_restore
150+
ghe_debug "\n$(find "$tempdir" -maxdepth 1 -name '*.rsync')"
151+
bm_end "$(basename $0) - Processing routes"
152+
# There is no need to collect routes and split them by Spokes server in noncluster setups because
153+
# we need to transfer all repository networks to the primary instance unconditionally, regardless of
154+
# the Spokes route list captured during the backup. As we already have the list of all repository
155+
# network paths, we can simply use that as the rsync file list in noncluster environments.
156+
else
157+
log_info "* Restoring all repository networks to target host unconditionally" 1>&3
158+
159+
cp "$tmp_list" "$tempdir/git-server-primary.rsync"
128160
fi
129-
echo "cat $remote_tmp_list | $restore_routes_script | grep 'git-server-' > $remote_routes_list" | ghe-ssh "$GHE_HOSTNAME" -- /bin/bash
130-
ghe-ssh "$GHE_HOSTNAME" -- cat $remote_routes_list | ghe_debug
131-
bm_end "$(basename $0) - Generating routes"
132-
133-
bm_start "$(basename $0) - Fetching routes"
134-
ghe-ssh "$GHE_HOSTNAME" -- gzip -c $remote_routes_list | gzip -d > $routes_list
135-
cat $routes_list | ghe_debug
136-
bm_end "$(basename $0) - Fetching routes"
137-
138-
bm_start "$(basename $0) - Processing routes"
139-
cat $routes_list | awk -v tempdir="$tempdir" '{ for(i=2;i<=NF;i++){ print $1 > (tempdir"/"$i".rsync") }}'
140-
cat $routes_list | awk '{ n = split($1, p, "/"); printf p[n] " /data/repositories/" $1; $1=""; print $0}' > $to_restore
141-
ghe_debug "\n$(find "$tempdir" -maxdepth 1 -name '*.rsync')"
142-
bm_end "$(basename $0) - Processing routes"
143161

144162
if [ -z "$(find "$tempdir" -maxdepth 1 -name '*.rsync')" ]; then
145163
log_warn "Warning: no routes found, skipping repositories restore ..."

0 commit comments

Comments
 (0)