Skip to content

Commit 1069ddd

Browse files
gamefiendgamefiend
authored
Merge branch 'master' into krayon-git-ver-req
2 parents 0a4001f + 3bee490 commit 1069ddd

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

54 files changed

+2116
-203
lines changed

‎.github/CODEOWNERS

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
# Backup-Utils owned by lifecycle AOR
2+
* @github/ghes-lifecycle
3+
# Actions related backups and restores
4+
# /share/github-backup-utils/*-actions @github/ghes-lifecycle @github/<TBD>
5+
# Git related backups and restores
6+
# /share/github-backup-utils/*-repositories @github/ghes-lifecycle @github/<TBD>
7+
# /share/github-backup-utils/*-git-hooks @github/ghes-lifecycle @github/<TBD>

‎.github/workflows/backup.yml

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
name: Backup GHES instance and save to Azure
2+
run-name: "${{ github.actor }} - Backup GHES instance and save to Azure"
3+
4+
on:
5+
workflow_call:
6+
inputs:
7+
github-hostname:
8+
description: GitHub Hostname to backup
9+
required: true
10+
type: string
11+
backup-name:
12+
description: The name of the backup to be saved in Azure storage
13+
required: false
14+
default: ""
15+
type: string
16+
secrets:
17+
BACKUP_SSH_KEY:
18+
description: SSH key to access the GitHub Enterprise instance
19+
required: true
20+
INTERNAL_ACTIONS_DX_BOT_ACCOUNT_TOKEN:
21+
description: Token for the internal actions dx bot account
22+
required: true
23+
AZURE_USERNAME:
24+
description: Azure service principal username
25+
required: false
26+
AZURE_PASSWORD:
27+
description: Azure service principal password
28+
required: false
29+
AZURE_TENANT_ID:
30+
description: Azure tenant ID
31+
required: false
32+
AZURE_SUBSCRIPTION_ID:
33+
description: Azure subscription ID
34+
required: false
35+
AZURE_ACCOUNT_NAME:
36+
description: Azure storage account name
37+
required: false
38+
AZURE_CONTAINER_NAME:
39+
description: Azure storage container name
40+
required: false
41+
CONNECTIONSTRING:
42+
description: Azure storage connection string
43+
required: false
44+
45+
46+
jobs:
47+
build:
48+
runs-on: ubuntu-latest
49+
steps:
50+
- uses: actions/checkout@v3
51+
with:
52+
repository: github/backup-utils-private
53+
token: "${{ secrets.INTERNAL_ACTIONS_DX_BOT_ACCOUNT_TOKEN }}"
54+
- run: docker build . --file Dockerfile --tag backup-utils
55+
- run: docker save backup-utils -o backup-utils.tar
56+
- uses: actions/upload-artifact@v3
57+
with:
58+
name: backup-utils
59+
path: backup-utils.tar
60+
61+
backup-utils-backup:
62+
needs: build
63+
runs-on:
64+
group: larger-hosted-public-runners
65+
labels: ubuntu-latest-xl
66+
env:
67+
SSH_KEY: ${{ secrets.BACKUP_SSH_KEY }}
68+
steps:
69+
- uses: actions/download-artifact@v3
70+
with:
71+
name: backup-utils
72+
- name: Load docker container
73+
run: docker load -i backup-utils.tar
74+
- uses: actions/checkout@v3
75+
- name: Create backup directory
76+
run: mkdir "$HOME/ghe-backup-data"
77+
- name: set up ssh SSH_KEY
78+
run: echo -e "${SSH_KEY}\n" > "$HOME/backup"
79+
- name: set up ssh key permissions
80+
run: chmod 0600 "$HOME/backup"
81+
- name: change version
82+
run: echo "3.8.0" > "$HOME/version"
83+
84+
- name: Perform backup
85+
run: |
86+
docker run -e "GHE_HOSTNAME=${{ inputs.github-hostname }}" \
87+
-e "GHE_DATA_DIR=/data" \
88+
-e "GHE_EXTRA_SSH_OPTS=-p 122 -i /ghe-ssh/id_rsa -o ServerAliveInterval=30 -o ServerAliveCountMax=12000 -o StrictHostKeyChecking=no" \
89+
-e "GHE_NUM_SNAPSHOTS=15" \
90+
-v "$HOME/ghe-backup-data:/data" \
91+
-v "$HOME/backup:/ghe-ssh/id_rsa" \
92+
-v "$HOME/version:/backup-utils/share/github-backup-utils/version" \
93+
--rm \
94+
backup-utils ghe-backup
95+
- name: Check the backup file
96+
run: |
97+
current=$(readlink "$HOME/ghe-backup-data/current")
98+
sudo tar -czvf "${{ inputs.backup-name }}.tar.gz" -C "$HOME/ghe-backup-data/$current" .
99+
100+
- name: Login to Azure
101+
if: ${{ inputs.backup-name }} != ""
102+
run: |
103+
az login \
104+
--service-principal \
105+
-u "${{ secrets.AZURE_USERNAME }}" \
106+
-p "${{ secrets.AZURE_PASSWORD }}" \
107+
--tenant "${{ secrets.AZURE_TENANT_ID }}"
108+
az account set --subscription "${{ secrets.AZURE_SUBSCRIPTION_ID }}"
109+
110+
- name: Upload backup to Azure
111+
if: ${{ inputs.backup-name }} != ""
112+
run: |
113+
az storage blob upload \
114+
--account-name "${{ secrets.AZURE_ACCOUNT_NAME }}" \
115+
--container-name "${{ secrets.AZURE_CONTAINER_NAME }}" \
116+
--name "${{ inputs.backup-name }}.tar.gz" \
117+
--file "${{ inputs.backup-name }}.tar.gz" \
118+
--connection-string "${{ secrets.CONNECTIONSTRING }}"

‎.github/workflows/docker-image.yml

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,12 +14,15 @@ jobs:
1414

1515
steps:
1616
- uses: actions/checkout@v3
17-
- name: Build the Debian Docker image
17+
- name: Build the Ubuntu Docker image
1818
run: docker build . --file Dockerfile --tag backup-utils:${GITHUB_RUN_ID}
1919
- name: Build the Alpine Docker image
2020
run: docker build . --file Dockerfile.alpine --tag backup-utils-alpine:${GITHUB_RUN_ID}
21-
- name: Run tests in Debian Docker image
22-
run: docker run backup-utils:${GITHUB_RUN_ID} ghe-backup --version
21+
- name: Run tests in Ubuntu Docker image
22+
run: |
23+
docker run backup-utils:${GITHUB_RUN_ID} ghe-backup --version
24+
docker run backup-utils:${GITHUB_RUN_ID} rsync --version
2325
- name: Run tests in Alpine Docker image
24-
run: docker run backup-utils-alpine:${GITHUB_RUN_ID} ghe-backup --version
25-
26+
run: |
27+
docker run backup-utils-alpine:${GITHUB_RUN_ID} ghe-backup --version
28+
docker run backup-utils-alpine:${GITHUB_RUN_ID} rsync --version

‎.github/workflows/lint.yml

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,6 @@
11
name: Lint Code Base
22

33
on:
4-
push:
5-
branches-ignore: [master]
64
pull_request:
75
branches: [master]
86

@@ -17,7 +15,8 @@ jobs:
1715
# Full git history is needed to get a proper list of changed files within `super-linter`
1816
fetch-depth: 0
1917
- name: Lint Code Base
20-
uses: github/super-linter@v5
18+
uses: super-linter/super-linter@v5
2119
env:
2220
VALIDATE_ALL_CODEBASE: false
21+
BASH_SEVERITY: error
2322
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

‎.github/workflows/main.yml

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
name: Test and build
22

3-
on: [pull_request]
4-
3+
on: [pull_request, workflow_dispatch]
54

65
jobs:
76
build:
@@ -18,9 +17,9 @@ jobs:
1817
run: |
1918
sudo apt-get update -y
2019
sudo apt-get install -y devscripts debhelper moreutils fakeroot jq pigz help2man
21-
wget "https://github.com/koalaman/shellcheck/releases/download/latest/shellcheck-latest.linux.x86_64.tar.xz"
22-
tar --xz -xvf "shellcheck-latest.linux.x86_64.tar.xz"
23-
sudo cp shellcheck-latest/shellcheck /usr/bin/shellcheck
20+
wget "https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.x86_64.tar.xz"
21+
tar --xz -xvf "shellcheck-stable.linux.x86_64.tar.xz"
22+
sudo cp shellcheck-stable/shellcheck /usr/bin/shellcheck
2423
if: matrix.os != 'macos-latest'
2524
- name: Install Dependencies (macOS)
2625
run: |

‎.github/workflows/restore.yml

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -85,19 +85,22 @@ jobs:
8585
run: |
8686
version="${{ inputs.version }}"
8787
size="${{ inputs.size }}"
88-
V3_8_COMPATIBLE="3.6 3.7 3.8 3.9 3.10"
89-
echo "$V3_8_COMPATIBLE" | tr " " '\n' | grep -F -q -x "$version"
90-
exit_code="$?"
91-
if [ "$exit_code" -eq "0" ]; then
92-
echo "Version $version is acceptable"
88+
V3_6_COMPATIBLE="3.6 3.7"
89+
V3_8_COMPATIBLE="3.8 3.9 3.10"
90+
if echo "$V3_8_COMPATIBLE" | grep -q -w "$version"; then
91+
echo "Version $version is acceptable by 3.8 backup"
9392
file_version=3.8
94-
echo "version=3.8" >> "$GITHUB_OUTPUT"
95-
echo "name=v$file_version-$size.tar.gz" >> "$GITHUB_OUTPUT"
93+
elif echo "$V3_6_COMPATIBLE" | grep -q -w "$version"; then
94+
echo "Version $version is acceptable by 3.6 backup"
95+
file_version=3.6
9696
else
9797
echo "Version $version is not acceptable"
9898
exit 1
9999
fi
100100
101+
echo "version=$file_version" >> "$GITHUB_OUTPUT"
102+
echo "name=v$file_version-$size.tar.gz" >> "$GITHUB_OUTPUT"
103+
101104
- name: Download from blob storage
102105
run: |
103106
mkdir ghes-data

‎.github/workflows/rsync-docker-bump.yml

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
name: Update Rsync Tag in Dockerfile
2+
3+
on:
4+
schedule:
5+
- cron: '0 0 * * *' # Runs daily at 00:00
6+
7+
jobs:
8+
update-rsync-tag:
9+
runs-on: ubuntu-latest
10+
11+
steps:
12+
- name: Checkout repository
13+
uses: actions/checkout@v3
14+
15+
- name: Get latest rsync tag
16+
id: latest_tag
17+
run: |
18+
curl --silent "https://api.github.com/repos/WayneD/rsync/tags" | jq -r '.[0].name' | xargs -I {} echo "::set-output name=latest_tag::{}"
19+
20+
- name: Update Dockerfile with latest tag
21+
run: |
22+
sed -i -E "s/RSYNC_TAG=[0-9\.]+/RSYNC_TAG=${{ steps.latest_tag.outputs.latest_tag }}/g" Dockerfile
23+
24+
- name: Create Pull Request for tag update
25+
uses: peter-evans/create-pull-request@v3
26+
with:
27+
token: ${{ secrets.GITHUB_TOKEN }}
28+
commit-message: "Update rsync tag in Dockerfile"
29+
title: "Update rsync tag in Dockerfile"
30+
body: "This PR updates the rsync tag in the Dockerfile to the latest tagged version."
31+
branch: "update-rsync-tag"
32+
base: "master"
33+
path: "."
34+
labels: "automated-update,rsync"

‎.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
11
/backup.config
22
/data
33
/dist
4+
dash
5+
parallel

‎Dockerfile

Lines changed: 61 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,73 @@
1-
FROM ubuntu:focal
1+
# Multi stage build for backup-utils
2+
# Build layer is for compiling rsync from source
3+
# Runtime layer is for running backup-utils
4+
# https://docs.docker.com/develop/develop-images/multistage-build/
5+
# https://docs.docker.com/engine/userguide/eng-image/multistage-build/
26

3-
RUN apt-get -q -y update && \
4-
apt-get install -y --no-install-recommends \
7+
# Build layer
8+
FROM ubuntu:focal AS build
9+
10+
# Install build dependencies
11+
RUN apt-get update && apt-get install --no-install-recommends -y \
12+
gcc \
13+
g++ \
14+
gawk \
15+
autoconf \
16+
make \
17+
automake \
18+
python3-cmarkgfm \
19+
acl \
20+
libacl1-dev \
21+
attr \
22+
libattr1-dev \
23+
libxxhash-dev \
24+
libzstd-dev \
25+
liblz4-dev \
26+
libssl-dev \
27+
git \
28+
jq \
29+
bc \
30+
curl \
531
tar \
6-
rsync \
32+
gzip \
733
ca-certificates \
8-
ssh \
34+
&& rm -rf /var/lib/apt/lists/*
35+
36+
# Download rsync source from https://github.com/WayneD/rsync/archive/refs/tags/[TAG].tar.gz pinned to specified tag
37+
ARG RSYNC_TAG=v3.2.7
38+
RUN curl https://github.com/WayneD/rsync/archive/refs/tags/${RSYNC_TAG}.tar.gz -L -o ${RSYNC_TAG}.tar.gz
39+
RUN mkdir -p /rsync-${RSYNC_TAG}&& tar -xzf ${RSYNC_TAG}.tar.gz -C /rsync-${RSYNC_TAG} --strip-components=1 && ls -la
40+
# Change to the working directory of the rsync source
41+
WORKDIR /rsync-${RSYNC_TAG}
42+
RUN ls -la && ./configure
43+
RUN make
44+
RUN make install
45+
46+
# Reset working directory
47+
WORKDIR /
48+
49+
# Runtime layer
50+
FROM ubuntu:focal AS runtime
51+
52+
# Install runtime dependencies - bash, git, OpenSSH 5.6 or newer, and jq v1.5 or newer.
53+
RUN apt-get update && apt-get install --no-install-recommends -y \
54+
bash \
955
git \
56+
openssh-client \
57+
jq \
58+
bc \
1059
moreutils \
1160
gawk \
61+
ca-certificates \
62+
xxhash \
1263
&& rm -rf /var/lib/apt/lists/*
1364

65+
# Copy rsync from build layer
66+
COPY --from=build /usr/local/bin/rsync /usr/local/bin/rsync
67+
68+
# Copy backup-utils from repository into /backup-utils
1469
COPY ./ /backup-utils/
70+
1571
WORKDIR /backup-utils
1672

1773
RUN chmod +x /backup-utils/share/github-backup-utils/ghe-docker-init

‎Makefile

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
SHELL = /bin/sh
22

33
test: info
4+
@echo Running tests
45
@script/cibuild --no-package
56

67
info:

0 commit comments

Comments
 (0)