Skip to content

Commit 9ec7fea

Browse files
rubiojrrubiojr
authored
Merge pull request #231 from github/git-hooks-fixes
git-hooks backup/restore fixes
2 parents 1deb1da + 3a5c704 commit 9ec7fea

11 files changed

+210
-57
lines changed

‎bin/ghe-backup‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -190,8 +190,10 @@ if [ "$GHE_VERSION_MAJOR" -ge 2 ]; then
190190
failures="$failures hookshot"
191191

192192
echo "Backing up custom Git hooks ..."
193-
ghe-backup-userdata git-hooks ||
194-
failures="$failures git-hooks"
193+
ghe-backup-userdata git-hooks/environments/tarballs ||
194+
failures="$failures git-hooks-environments"
195+
ghe-backup-userdata git-hooks/repos ||
196+
failures="$failures git-hooks-repos"
195197
fi
196198
fi
197199

‎bin/ghe-restore‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -274,7 +274,9 @@ elif [ "$GHE_VERSION_MAJOR" -ge 2 ]; then
274274
ghe-restore-userdata hookshot "$GHE_HOSTNAME" 1>&3
275275

276276
echo "Restoring custom Git hooks ..."
277-
ghe-restore-userdata git-hooks "$GHE_HOSTNAME" 1>&3
277+
ghe-restore-userdata git-hooks/environments/tarballs "$GHE_HOSTNAME" 1>&3
278+
ghe-restore-userdata git-hooks/repos "$GHE_HOSTNAME" 1>&3
279+
ghe-restore-git-hooks-extract "$GHE_HOSTNAME" 1>&3
278280
fi
279281

280282
if $cluster; then

‎share/github-backup-utils/ghe-backup-git-hooks-cluster‎

Lines changed: 35 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -58,12 +58,6 @@ cleanup() {
5858
trap 'cleanup' EXIT
5959
trap 'exit $?' INT # ^C always terminate
6060

61-
# If we have a previous increment, avoid transferring existing files via rsync's
62-
# --link-dest support. This also decreases physical space usage considerably.
63-
if [ -d "$backup_current" ]; then
64-
link_dest="--link-dest=../../current/git-hooks"
65-
fi
66-
6761
# Transfer Git hooks data from a GitHub instance to the current snapshot
6862
# directory, using a previous snapshot to avoid transferring files that have
6963
# already been transferred. A set of rsync filter rules are provided on stdin
@@ -72,16 +66,42 @@ rsync_git_hooks_data () {
7266
port=$(ssh_port_part "$1")
7367
host=$(ssh_host_part "$1")
7468

75-
shift
76-
ghe-rsync -a \
69+
subpath=$2
70+
shift 2
71+
72+
# If we have a previous increment and it is not empty, avoid transferring existing files via rsync's
73+
# --link-dest support. This also decreases physical space usage considerably.
74+
if [ -d "$backup_current/$subpath" ] && [ "$(ls -A $backup_current/$subpath)" ]; then
75+
76+
subdir="git-hooks/$subpath"
77+
link_path=".."
78+
while true; do
79+
if [ $(dirname $subdir) = "." ]; then
80+
break
81+
fi
82+
83+
if [ $(dirname $subdir) = "/" ]; then
84+
break
85+
fi
86+
87+
link_path="../$link_path"
88+
subdir=$(dirname $subdir)
89+
done
90+
91+
local link_dest="--link-dest=../${link_path}/current/git-hooks/$subpath"
92+
fi
93+
94+
# Ensure target directory exists, is needed with subdirectories
95+
mkdir -p "$backup_dir/$subpath"
96+
97+
ghe-rsync -av \
7798
-e "ssh -q $opts -p $port -F $config_file -l $user" $link_dest \
7899
--rsync-path='sudo -u git rsync' \
79-
"$host:$GHE_REMOTE_DATA_USER_DIR/git-hooks/" \
80-
"$backup_dir" 1>&3
100+
"$host:$GHE_REMOTE_DATA_USER_DIR/git-hooks/$subpath/" \
101+
"$backup_dir/$subpath" 1>&3
81102
}
82103

83-
for hostname in $hostnames; do
84-
ghe-ssh -F $config_file "$hostname:122" -- "[ -d '$GHE_REMOTE_DATA_USER_DIR/git-hooks' ]" || exit 0
85-
# We can stop after the first successful sync since hooks are the same everywhere
86-
rsync_git_hooks_data $hostname:122 && break
87-
done
104+
hostname=$(echo $hostnames | awk '{ print $1; }')
105+
ghe-ssh -F $config_file "$hostname:122" -- "sudo -u git [ -d '$GHE_REMOTE_DATA_USER_DIR/git-hooks' ]" || exit 0
106+
rsync_git_hooks_data $hostname:122 environments/tarballs
107+
rsync_git_hooks_data $hostname:122 repos

‎share/github-backup-utils/ghe-backup-userdata‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,14 +23,33 @@ ghe_remote_version_required "$host"
2323

2424
# Verify that the user data directory exists. Bail out if not, which may be due
2525
# to an older version of GHE or no data has been added to this directory yet.
26-
ghe-ssh "$host" -- "[ -d '$GHE_REMOTE_DATA_USER_DIR/$dirname' ]" || exit 0
26+
ghe-ssh "$host" -- "sudo -u git [ -d '$GHE_REMOTE_DATA_USER_DIR/$dirname' ]" || exit 0
2727

2828
# If we have a previous increment and it is not empty, avoid transferring existing files via rsync's
2929
# --link-dest support. This also decreases physical space usage considerably.
3030
if [ -d "$GHE_DATA_DIR/current/$dirname" ] && [ "$(ls -A $GHE_DATA_DIR/current/$dirname)" ]; then
31-
link_dest="--link-dest=../../current/$dirname"
31+
32+
subdir=$dirname
33+
link_path=".."
34+
while true; do
35+
if [ $(dirname $subdir) = "." ]; then
36+
break
37+
fi
38+
39+
if [ $(dirname $subdir) = "/" ]; then
40+
break
41+
fi
42+
43+
link_path="../$link_path"
44+
subdir=$(dirname $subdir)
45+
done
46+
47+
link_dest="--link-dest=../${link_path}/current/$dirname"
3248
fi
3349

50+
# Ensure target directory exists, is needed with subdirectories
51+
mkdir -p "$GHE_SNAPSHOT_DIR/$dirname"
52+
3453
# Transfer all data from the user data directory using rsync.
3554
ghe-rsync -avz \
3655
-e "ghe-ssh -p $(ssh_port_part "$host")" \

‎share/github-backup-utils/ghe-restore-alambic-cluster-ng‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ GHE_HOSTNAME="$1"
2323
: ${GHE_RESTORE_SNAPSHOT:=current}
2424

2525
# Find the objets to restore
26-
storage_paths=$(cd $GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/ && find storage -mindepth 4 -maxdepth 4 -type f -printf '%p %s\n' | cut -d / -f2-)
26+
storage_paths=$(cd $GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/ && find storage -mindepth 4 -maxdepth 4 -type f -exec wc -c {} \;)
2727

2828
# No need to restore anything, early exit
2929
if [ -z "$storage_paths" ]; then
@@ -80,8 +80,8 @@ trap 'cleanup' EXIT
8080
# ...
8181
OLDIFS=$IFS; IFS=$'\n'
8282
for path in $storage_paths; do
83-
oid=$(echo $path | cut -d ' ' -f 1 | awk -F/ '{print $(NF)}')
84-
size=$(echo $path | cut -d ' ' -f 2)
83+
oid=$(echo $path | awk -F/ '{print $(NF)}')
84+
size=$(echo $path | awk '{print $1}')
8585
echo $oid $size
8686
done > $tmp_list
8787
IFS=$OLDIFS

‎share/github-backup-utils/ghe-restore-git-hooks-cluster‎

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -58,13 +58,31 @@ cleanup() {
5858
}
5959
trap 'cleanup' INT TERM EXIT
6060

61-
if [ -d "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/git-hooks" ]; then
61+
if [ -d "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/git-hooks/environments/tarballs" ]; then
62+
tarballs=$(cd $GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/git-hooks/environments/tarballs && find . -type f)
63+
hostname=$(echo $hostnames | awk '{ print $1; }')
64+
65+
if [ -n "$hostname" ]; then
66+
ghe-rsync -avH --delete \
67+
-e "ssh -q $opts -p $port -F $config_file -l $user" \
68+
--rsync-path="sudo -u git rsync" \
69+
"$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/git-hooks/environments/tarballs/" \
70+
"$hostname:$GHE_REMOTE_DATA_USER_DIR/git-hooks/environments/tarballs" 1>&3
71+
72+
for tarball in $tarballs; do
73+
env_id=$(echo $tarball | cut -d '/' -f 2)
74+
ssh -q $opts -p $port -F $config_file -l $user $hostname "/bin/bash -c 'export PATH=\$PATH:/usr/local/share/enterprise && ghe-hook-env-update $env_id $GHE_REMOTE_DATA_USER_DIR/git-hooks/environments/tarballs/$tarball'" 1>&3 2>&3
75+
done
76+
fi
77+
fi
78+
79+
if [ -d "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/git-hooks/repos" ]; then
6280
for hostname in $hostnames; do
63-
ghe-rsync -aH --delete \
81+
ghe-rsync -avH --delete \
6482
-e "ssh -q $opts -p $port -F $config_file -l $user" \
6583
--rsync-path="sudo -u git rsync" \
66-
"$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/git-hooks/" \
67-
"$hostname:$GHE_REMOTE_DATA_USER_DIR/git-hooks/" &
84+
"$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/git-hooks/repos/" \
85+
"$hostname:$GHE_REMOTE_DATA_USER_DIR/git-hooks/repos" 1>&3 &
6886
done
6987

7088
for pid in $(jobs -p); do

‎share/github-backup-utils/ghe-restore-git-hooks-extract‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
#!/usr/bin/env bash
2+
#/ Usage: ghe-restore-git-hooks-extract <host>
3+
#/ Restore custom Git hooks environments from tarballs
4+
#/
5+
#/ Note: This command typically isn't called directly. It's invoked by
6+
#/ ghe-restore
7+
set -e
8+
9+
# Bring in the backup configuration
10+
. $( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config
11+
12+
# Show usage and bail with no arguments
13+
[ -z "$*" ] && print_usage
14+
15+
# Grab host arg
16+
GHE_HOSTNAME="$1"
17+
18+
# Perform a host-check and establish GHE_REMOTE_XXX variables.
19+
ghe_remote_version_required "$GHE_HOSTNAME"
20+
21+
# The snapshot to restore should be set by the ghe-restore command but this lets
22+
# us run this script directly.
23+
: ${GHE_RESTORE_SNAPSHOT:=current}
24+
25+
if [ ! -d $GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/git-hooks/environments/tarballs ]; then
26+
echo "Warning: No pre-receive hook environments. Skipping ..."
27+
exit 0
28+
fi
29+
30+
tarballs=$(cd $GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/git-hooks/environments/tarballs && find . -type f)
31+
32+
for tarball in $tarballs; do
33+
env_id=$(echo $tarball | cut -d '/' -f 2)
34+
ghe-ssh "$GHE_HOSTNAME" "/bin/bash -c 'export PATH=\$PATH:/usr/local/share/enterprise && ghe-hook-env-update $env_id $GHE_REMOTE_DATA_USER_DIR/git-hooks/environments/tarballs/$tarball'"
35+
done

‎share/github-backup-utils/ghe-restore-userdata‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ ghe_remote_version_required "$GHE_HOSTNAME"
3535
if [ -d "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/$dirname" ]; then
3636
ghe-rsync -avz --delete \
3737
-e "ghe-ssh -p $(ssh_port_part "$GHE_HOSTNAME")" \
38-
--rsync-path="sudo -u git rsync" \
38+
--rsync-path="sudo -u git mkdir -p $GHE_REMOTE_DATA_USER_DIR/$dirname && sudo -u git rsync" \
3939
"$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/$dirname/" \
4040
"$(ssh_host_part "$GHE_HOSTNAME"):$GHE_REMOTE_DATA_USER_DIR/$dirname" 1>&3
4141
fi

‎test/bin/ghe-hook-env-update‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
#!/usr/bin/env bash
2+
# Emulates a remote GitHub import command. Each of the ghe-import-* utilities
3+
# that are run on the remote side have corresponding commands in this directory
4+
# that symlink to this file. The command just gobbles up stdin and writes a
5+
# simple success message.
6+
env_id=$1
7+
tarball=$2
8+
9+
path=$(dirname $tarball)/../../
10+
mkdir -p $path/$1
11+
cd $path/$i
12+
tar -zxf $tarball

‎test/test-ghe-backup.sh‎

Lines changed: 43 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -17,21 +17,30 @@ touch alice/index.html bob/index.html
1717
mkdir -p "$GHE_REMOTE_DATA_USER_DIR/common"
1818
echo "fake password hash data" > "$GHE_REMOTE_DATA_USER_DIR/common/manage-password"
1919

20-
# Create some fake hookshot data in the remote data directory
2120
if [ "$GHE_VERSION_MAJOR" -ge 2 ]; then
21+
# Create some fake data in the remote data directory
2222
mkdir -p "$GHE_REMOTE_DATA_USER_DIR/hookshot"
2323
cd "$GHE_REMOTE_DATA_USER_DIR/hookshot"
2424
mkdir -p repository-123 repository-456
2525
touch repository-123/test.bpack repository-456/test.bpack
2626

27-
mkdir -p "$GHE_REMOTE_DATA_USER_DIR/git-hooks"
28-
cd "$GHE_REMOTE_DATA_USER_DIR/git-hooks"
29-
mkdir -p repository-123 repository-456
30-
touch repository-123/script.sh repository-456/foo.sh
31-
fi
27+
# Create some fake hooks in the remote data directory
28+
mkdir -p "$GHE_REMOTE_DATA_USER_DIR/git-hooks/environments/tarballs"
29+
mkdir -p "$GHE_REMOTE_DATA_USER_DIR/git-hooks/repos"
3230

33-
# Create some fake alambic data in the remote data directory
34-
if [ "$GHE_VERSION_MAJOR" -ge 2 ]; then
31+
cd "$GHE_REMOTE_DATA_USER_DIR/git-hooks/environments"
32+
mkdir -p 123/abcdef 456/fed314
33+
touch 123/abcdef/script.sh 456/fed314/foo.sh
34+
35+
cd "$GHE_REMOTE_DATA_USER_DIR/git-hooks/environments/tarballs"
36+
mkdir -p 123/abcdef 456/fed314
37+
touch 123/abcdef/script.tar.gz 456/fed314/foo.tar.gz
38+
39+
cd "$GHE_REMOTE_DATA_USER_DIR/git-hooks/repos"
40+
mkdir -p 321 654
41+
touch 321/script.sh 654/foo.sh
42+
43+
# Create some fake alambic data in the remote data directory
3544
mkdir -p "$GHE_REMOTE_DATA_USER_DIR/alambic_assets/github-enterprise-assets/0000"
3645
touch "$GHE_REMOTE_DATA_USER_DIR/alambic_assets/github-enterprise-assets/0000/test.png"
3746

@@ -127,8 +136,14 @@ begin_test "ghe-backup first snapshot"
127136
# verify all hookshot user data was transferred
128137
diff -ru "$GHE_REMOTE_DATA_USER_DIR/hookshot" "$GHE_DATA_DIR/current/hookshot"
129138

130-
# verify all git hooks data was transferred
131-
diff -ru "$GHE_REMOTE_DATA_USER_DIR/git-hooks" "$GHE_DATA_DIR/current/git-hooks"
139+
# verify all git hooks tarballs were transferred
140+
diff -ru "$GHE_REMOTE_DATA_USER_DIR/git-hooks/environments/tarballs" "$GHE_DATA_DIR/current/git-hooks/environments/tarballs"
141+
142+
# verify the extracted environments were not transferred
143+
! diff -ru "$GHE_REMOTE_DATA_USER_DIR/git-hooks/environments" "$GHE_DATA_DIR/current/git-hooks/environments"
144+
145+
# verify the extracted repositories were transferred
146+
diff -ru "$GHE_REMOTE_DATA_USER_DIR/git-hooks/repos" "$GHE_DATA_DIR/current/git-hooks/repos"
132147

133148
# verify all alambic assets user data was transferred
134149
diff -ru "$GHE_REMOTE_DATA_USER_DIR/alambic_assets" "$GHE_DATA_DIR/current/alambic_assets"
@@ -204,8 +219,14 @@ begin_test "ghe-backup subsequent snapshot"
204219
# verify all hookshot user data was transferred
205220
diff -ru "$GHE_REMOTE_DATA_USER_DIR/hookshot" "$GHE_DATA_DIR/current/hookshot"
206221

207-
# verify all git hooks data was transferred
208-
diff -ru "$GHE_REMOTE_DATA_USER_DIR/git-hooks" "$GHE_DATA_DIR/current/git-hooks"
222+
# verify all git hooks tarballs were transferred
223+
diff -ru "$GHE_REMOTE_DATA_USER_DIR/git-hooks/environments/tarballs" "$GHE_DATA_DIR/current/git-hooks/environments/tarballs"
224+
225+
# verify the extracted environments were not transferred
226+
! diff -ru "$GHE_REMOTE_DATA_USER_DIR/git-hooks/environments" "$GHE_DATA_DIR/current/git-hooks/environments"
227+
228+
# verify the extracted repositories were transferred
229+
diff -ru "$GHE_REMOTE_DATA_USER_DIR/git-hooks/repos" "$GHE_DATA_DIR/current/git-hooks/repos"
209230

210231
# verify all alambic assets user data was transferred
211232
diff -ru "$GHE_REMOTE_DATA_USER_DIR/alambic_assets" "$GHE_DATA_DIR/current/alambic_assets"
@@ -282,8 +303,14 @@ begin_test "ghe-backup with relative data dir path"
282303
# verify all hookshot user data was transferred
283304
diff -ru "$GHE_REMOTE_DATA_USER_DIR/hookshot" "$GHE_DATA_DIR/current/hookshot"
284305

285-
# verify all git hooks data was transferred
286-
diff -ru "$GHE_REMOTE_DATA_USER_DIR/git-hooks" "$GHE_DATA_DIR/current/git-hooks"
306+
# verify all git hooks tarballs were transferred
307+
diff -ru "$GHE_REMOTE_DATA_USER_DIR/git-hooks/environments/tarballs" "$GHE_DATA_DIR/current/git-hooks/environments/tarballs"
308+
309+
# verify the extracted environments were not transferred
310+
! diff -ru "$GHE_REMOTE_DATA_USER_DIR/git-hooks/environments" "$GHE_DATA_DIR/current/git-hooks/environments"
311+
312+
# verify the extracted repositories were transferred
313+
diff -ru "$GHE_REMOTE_DATA_USER_DIR/git-hooks/repos" "$GHE_DATA_DIR/current/git-hooks/repos"
287314

288315
# verify all alambic assets user data was transferred
289316
diff -ru "$GHE_REMOTE_DATA_USER_DIR/alambic_assets" "$GHE_DATA_DIR/current/alambic_assets"
@@ -375,8 +402,8 @@ begin_test "ghe-backup empty git-hooks directory"
375402
(
376403
set -e
377404

378-
rm -rf $GHE_REMOTE_DATA_USER_DIR/git-hooks/repository-*
379-
rm -rf $GHE_DATA_DIR/current/git-hooks/repository-*
405+
rm -rf $GHE_REMOTE_DATA_USER_DIR/git-hooks/*
406+
rm -rf $GHE_DATA_DIR/current/git-hooks/*
380407
ghe-backup
381408

382409
# Check that the "--link-dest arg does not exist" message hasn't occurred.

0 commit comments

Comments
 (0)