Skip to content

Commit 77110e9

Browse files
dooleydevindooleydevin
authored
Merge pull request #1123 from github/3.9.3-patch
3.9.3 patch
2 parents ddcdb33 + 630adfa commit 77110e9

20 files changed

+342
-58
lines changed

‎.github/linters/.yaml-lint.yml

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
---
2+
###########################################
3+
# These are the rules used for #
4+
# linting all the yaml files in the stack #
5+
# NOTE: #
6+
# You can disable line with: #
7+
# # yamllint disable-line #
8+
###########################################
9+
rules:
10+
braces:
11+
level: warning
12+
min-spaces-inside: 0
13+
max-spaces-inside: 0
14+
min-spaces-inside-empty: 1
15+
max-spaces-inside-empty: 5
16+
brackets:
17+
level: warning
18+
min-spaces-inside: 0
19+
max-spaces-inside: 0
20+
min-spaces-inside-empty: 1
21+
max-spaces-inside-empty: 5
22+
colons:
23+
level: warning
24+
max-spaces-before: 0
25+
max-spaces-after: 1
26+
commas:
27+
level: warning
28+
max-spaces-before: 0
29+
min-spaces-after: 1
30+
max-spaces-after: 1
31+
comments: disable
32+
comments-indentation: disable
33+
document-end: disable
34+
document-start: disable
35+
empty-lines:
36+
level: warning
37+
max: 2
38+
max-start: 0
39+
max-end: 0
40+
hyphens:
41+
level: warning
42+
max-spaces-after: 1
43+
indentation:
44+
level: warning
45+
spaces: consistent
46+
indent-sequences: true
47+
check-multi-line-strings: false
48+
key-duplicates: enable
49+
line-length: disable
50+
new-line-at-end-of-file: disable
51+
new-lines:
52+
type: unix
53+
trailing-spaces: disable

‎.github/workflows/build-and-release.yml

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
---
2+
name: Build and Release
3+
4+
on:
5+
workflow_dispatch:
6+
inputs:
7+
gh-token:
8+
description: 'GitHub Token - used to create a commit in the backup-utils repo'
9+
required: true
10+
type: string
11+
version:
12+
description: 'Version - patch version of the release (e.g. x.y.z)'
13+
required: true
14+
type: string
15+
draft:
16+
description: 'Draft - true if the release should be a draft'
17+
required: true
18+
type: boolean
19+
default: true
20+
21+
jobs:
22+
build:
23+
runs-on: ubuntu-latest
24+
steps:
25+
# resulting token still gets denied by the backup-utils repo
26+
# see: https://github.com/actions/create-github-app-token/pull/46
27+
# - uses: timreimherr/create-github-app-token@main
28+
# id: app-token
29+
# with:
30+
# # required
31+
# app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }}
32+
# private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }}
33+
# owner: ${{ github.repository_owner }}
34+
# repositories: backup-utils,backup-utils-private
35+
- name: Checkout backup-utils-private
36+
uses: actions/checkout@v4
37+
with:
38+
token: ${{ github.event.inputs.gh-token }}
39+
repository: github/backup-utils-private
40+
- name: Install dependencies
41+
run: |
42+
sudo apt-get update -y
43+
sudo apt-get install -y moreutils debhelper help2man devscripts gzip
44+
- name: Create tag # this is required for the build scripts
45+
run: |
46+
git config user.name "${{ github.actor }}"
47+
git config user.email "[email protected]"
48+
git tag -a "v${{ github.event.inputs.version }}" -m "v${{ github.event.inputs.version }}"
49+
git push origin "v${{ github.event.inputs.version }}"
50+
- name: Package deb
51+
run: |
52+
./script/package-deb
53+
# many need to remove this once release-notes compilation is automated
54+
- name: Rename deb artifact
55+
run: |
56+
for file in dist/github-backup-utils_*_all.deb; do
57+
if [[ -f "$file" ]]; then
58+
mv "$file" "dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb"
59+
fi
60+
done
61+
- name: Upload deb artifact
62+
uses: actions/upload-artifact@v3
63+
with:
64+
name: github-backup-utils_${{ github.event.inputs.version }}_all.deb
65+
path: |
66+
dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb
67+
- name: Package tarball
68+
run: |
69+
./script/package-tarball
70+
- name: Upload tarball artifact
71+
uses: actions/upload-artifact@v3
72+
with:
73+
name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz
74+
path: |
75+
dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz
76+
release:
77+
needs: build
78+
runs-on: ubuntu-latest
79+
outputs:
80+
commit_hash: ${{ steps.empty-commit.outputs.commit_hash }}
81+
steps:
82+
# resulting token still gets denied by the backup-utils repo
83+
# see: https://github.com/actions/create-github-app-token/pull/46
84+
# - uses: timreimherr/create-github-app-token@main
85+
# id: app-token
86+
# with:
87+
# app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }}
88+
# private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }}
89+
# owner: ${{ github.repository_owner }}
90+
# repositories: backup-utils,backup-utils-private
91+
- name: Checkout backup-utils
92+
uses: actions/checkout@v4
93+
with:
94+
token: ${{ github.event.inputs.gh-token }}
95+
repository: github/backup-utils
96+
ref: master
97+
- name: Create empty commit
98+
uses: stefanzweifel/git-auto-commit-action@v4
99+
id: empty-commit
100+
with:
101+
branch: master
102+
commit_message: "${{ github.event.inputs.version }} release"
103+
commit_user_name: "${{ github.actor }}"
104+
commit_user_email: "[email protected]"
105+
commit_options: "--allow-empty"
106+
skip_dirty_check: true
107+
- name: Checkout backup-utils
108+
uses: actions/checkout@v4
109+
with:
110+
token: ${{ github.event.inputs.gh-token }}
111+
repository: github/backup-utils-private
112+
- name: Download deb artifact
113+
uses: actions/download-artifact@v3
114+
with:
115+
name: github-backup-utils_${{ github.event.inputs.version }}_all.deb
116+
- name: Download tarball artifact
117+
uses: actions/download-artifact@v3
118+
with:
119+
name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz
120+
- name: Create Release
121+
uses: ncipollo/release-action@v1
122+
with:
123+
token: ${{ github.event.inputs.gh-token }}
124+
repo: backup-utils
125+
name: |
126+
GitHub Enterprise Server Backup Utilities v${{ github.event.inputs.version }}
127+
artifacts: |
128+
github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, \
129+
github-backup-utils_${{ github.event.inputs.version }}_all.deb
130+
tag: v${{ github.event.inputs.version }}
131+
commit: ${{ steps.empty-commit.outputs.commit_hash }}
132+
bodyFile: release-notes/${{ github.event.inputs.version }}.md
133+
draft: ${{ github.event.inputs.draft }}
134+
allowUpdates: true
135+
artifactContentType: "raw"
136+
137+
138+
139+

‎.github/workflows/lint.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
---
12
name: Lint Code Base
23

34
on:
@@ -21,3 +22,4 @@ jobs:
2122
env:
2223
VALIDATE_ALL_CODEBASE: false
2324
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
25+
FILTER_REGEX_EXCLUDE: .*release-notes/.*

‎.github/workflows/main.yml

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,12 @@
11
name: Test and build
22

3-
on: [pull_request]
3+
on: [pull_request, workflow_dispatch]
44

55

66
jobs:
77
build:
88
strategy:
99
matrix:
10-
# macos-latest references are kept here for historical purposes. removed macos-latest from the
11-
#matrix as it is not a typical case for users and causes a lot of friction with other linux-based
12-
# installs. Recommend developing on codespaces or using an ubuntu container.
1310
os: ['ubuntu-22.04', 'ubuntu-20.04']
1411
fail-fast: false
1512
runs-on: ${{ matrix.os }}
@@ -21,12 +18,6 @@ jobs:
2118
wget "https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.x86_64.tar.xz"
2219
tar --xz -xvf "shellcheck-stable.linux.x86_64.tar.xz"
2320
sudo cp shellcheck-stable/shellcheck /usr/bin/shellcheck
24-
if: matrix.os != 'macos-latest'
25-
- name: Install Dependencies (macOS)
26-
run: |
27-
brew install gnu-tar shellcheck jq pigz coreutils gnu-sed gnu-getopt wget
28-
brew install moreutils gawk
29-
if: matrix.os == 'macos-latest'
3021
- name: Get Sources
3122
uses: actions/checkout@v3
3223
- name: Test
@@ -36,4 +27,3 @@ jobs:
3627
shell: bash
3728
- name: Build (Linux)
3829
run: DEB_BUILD_OPTIONS=nocheck debuild -us -uc
39-
if: matrix.os != 'macos-latest'

‎.releaseignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
ownership.yaml
2+
.github

‎bin/ghe-backup

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -164,11 +164,11 @@ echo "$GHE_REMOTE_VERSION" > version
164164
# Setup progress tracking
165165
init-progress
166166
export PROGRESS_TOTAL=14 # Minimum number of steps in backup is 14
167-
echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total
167+
echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress/total
168168
export PROGRESS_TYPE="Backup"
169-
echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress-type
169+
echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress/type
170170
export PROGRESS=0 # Used to track progress of backup
171-
echo "$PROGRESS" > /tmp/backup-utils-progress
171+
echo "$PROGRESS" > /tmp/backup-utils-progress/progress
172172

173173
OPTIONAL_STEPS=0
174174
# Backup actions+mssql
@@ -192,7 +192,7 @@ if [ "$GHE_BACKUP_PAGES" != "no" ]; then
192192
fi
193193

194194
PROGRESS_TOTAL=$((OPTIONAL_STEPS + PROGRESS_TOTAL)) # Minimum number of steps in backup is 14
195-
echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total
195+
echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress/total
196196

197197
if [ -n "$GHE_ALLOW_REPLICA_BACKUP" ]; then
198198
echo "Warning: backing up a high availability replica may result in inconsistent or unreliable backups."

‎bin/ghe-backup-progress

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,26 +31,26 @@ while true; do
3131
done
3232

3333
check_for_progress_file() {
34-
if [ ! -f /tmp/backup-utils-progress-info ]; then
34+
if [ ! -f /tmp/backup-utils-progress/info ]; then
3535
echo "No progress file found. Has a backup or restore been started?"
3636
exit 1
3737
fi
3838
}
3939

4040
if [ -n "$ONCE" ]; then
4141
check_for_progress_file
42-
cat /tmp/backup-utils-progress-info
42+
cat /tmp/backup-utils-progress/info
4343
else
4444
check_for_progress_file
4545
clear
46-
cat /tmp/backup-utils-progress-info
46+
cat /tmp/backup-utils-progress/info
4747
while true; do
4848
if read -r -t 1 -n 1; then
4949
clear
5050
exit ;
5151
else
5252
clear
53-
cat /tmp/backup-utils-progress-info
53+
cat /tmp/backup-utils-progress/info
5454
fi
5555
done
5656
fi

‎bin/ghe-host-check

Lines changed: 27 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -91,13 +91,19 @@ if ghe-ssh "$host" -- \
9191
CLUSTER=true
9292
fi
9393

94-
# ensure all nodes in the cluster are running the same version
94+
# ensure all nodes in the cluster are online/reachable and running the same version
9595
if "$CLUSTER"; then
96+
online_status=$(ghe-ssh "$host" ghe-cluster-host-check)
97+
if [ "$online_status" != "Cluster is ready to configure." ]; then
98+
echo "Error: Not all nodes are online! Please ensure cluster is in a healthy state before using backup-utils." 1>&2
99+
exit 1
100+
fi
101+
96102
node_version_list=$(ghe-ssh "$host" ghe-cluster-each -- ghe-version)
97103
distinct_versions=$(echo "$node_version_list" | awk '{split($0, a, ":"); print a[2]}' | awk '{print $4}' | uniq | wc -l)
98104
if [ "$distinct_versions" -ne 1 ]; then
99-
echo "$node_version_list" 1>&2
100-
echo "Error: Not all nodes are running the same version! Please ensure all nodes are running the same version before using backup-utils." 1>&3
105+
echo "Version mismatch: $node_version_list" 1>&2
106+
echo "Error: Not all nodes are running the same version! Please ensure all nodes are running the same version before using backup-utils." 1>&2
101107
exit 1
102108
fi
103109
fi
@@ -177,21 +183,24 @@ SKIP_MSG
177183
mssql_disk_size=$(transfer_size mssql /tmp)
178184

179185
min_disk_req=$((repos_disk_size + pages_disk_size + es_disk_size + stor_disk_size + minio_disk_size + mysql_disk_size + actions_disk_size + mssql_disk_size))
180-
echo "Available space: $((available_space / (1024 ** 2))) MB" 1>&2
181-
echo -e "Min Disk required for this backup is at least $min_disk_req MB\n" 1>&2
182-
183-
cat <<DATA_TRANSFER_SIZE 1>&2
184-
### Data Transfer Sizes
185-
repositories: $repos_disk_size MB
186-
pages: $pages_disk_size MB
187-
elasticsearch: $es_disk_size MB
188-
storage: $stor_disk_size MB
189-
minio: $minio_disk_size MB
190-
mysql: $mysql_disk_size MB
191-
actions: $actions_disk_size MB
192-
mssql: $mssql_disk_size MB
193-
194-
DATA_TRANSFER_SIZE
186+
recommended_disk_req=$((min_disk_req * 5))
187+
echo " - Available space: $((available_space / (1024 ** 2))) MB" 1>&2
188+
echo " - Min Disk required for this backup is at least $min_disk_req MB" 1>&2
189+
echo " - Recommended Disk requirement is $recommended_disk_req MB" 1>&2
190+
echo "" 1>&2
191+
192+
printf '### Estimated Data Transfer Sizes
193+
194+
- repositories: %d MB
195+
- pages: %d MB
196+
- elasticsearch: %d MB
197+
- storage: %d MB
198+
- minio: %d MB
199+
- mysql: %d MB
200+
- actions: %d MB
201+
- mssql: %d MB
202+
\n' \
203+
"$repos_disk_size" "$pages_disk_size" "$es_disk_size" "$stor_disk_size" "$minio_disk_size" "$mysql_disk_size" "$actions_disk_size" "$mssql_disk_size" 1>&2
195204

196205
if [[ $((available_space / (1024 * 1024))) -lt $min_disk_req ]]; then
197206
echo "There is not enough disk space for the backup. Please allocate more space and continue." 1>&2

‎bin/ghe-restore

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -311,11 +311,11 @@ fi
311311
export PROGRESS_TOTAL=$((OPTIONAL_STEPS + 7))
312312

313313
init-progress
314-
echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total
314+
echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress/total
315315
export PROGRESS_TYPE="Restore"
316-
echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress-type
316+
echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress/type
317317
export PROGRESS=0 # Used to track progress of restore
318-
echo "$PROGRESS" > /tmp/backup-utils-progress
318+
echo "$PROGRESS" > /tmp/backup-utils-progress/progress
319319

320320
# Log restore start message locally and in /var/log/syslog on remote instance
321321
START_TIME=$(date +%s)

‎docs/requirements.md

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ storage and must have network connectivity with the GitHub Enterprise Server app
55

66
## Backup host requirements
77

8-
Backup host software requirements are modest: Linux or other modern Unix operating system (Ubuntu is highly recommended) with [bash][1], [git][2], [OpenSSH][3] 5.6 or newer, [rsync][4] v2.6.4 or newer* (see [below](april-2023-update-of-rsync-requirements) for exceptions), and [jq][11] v1.5 or newer. See below for an update on rsync.
8+
Backup host software requirements are modest: Linux or other modern Unix operating system (Ubuntu is highly recommended) with [bash][1], [git][2], [OpenSSH][3] 5.6 or newer, [rsync][4] v2.6.4 or newer* (see [below](#april-2023-update-of-rsync-requirements) for exceptions), [jq][11] v1.5 or newer, and [bc][12] v1.07 or newer.
99

1010
Ubuntu is the operating system we use to test `backup-utils` and it’s what we recommend you use too. You are welcome to use a different operating system, and we'll do our best to help you if you run into issues. But we can't guarantee that we'll be able to resolve issues that are specific to that operating system.
1111

@@ -101,3 +101,4 @@ Due to how some components of Backup Utilities (e.g. MSSQL) take incremental bac
101101
[9]: https://joeyh.name/code/moreutils
102102
[10]: https://www.gnu.org/software/gawk
103103
[11]: https://stedolan.github.io/jq/
104+
[12]: https://www.gnu.org/software/bc/

0 commit comments

Comments
 (0)