Skip to content

Commit 431bc1e

Browse files
committed
Merge pull request #122 from github/rsync-netshard-layout
Support for sharded repository storage layout
2 parents 727767d + 0e61eeb commit 431bc1e

File tree

3 files changed

+218
-2
lines changed

3 files changed

+218
-2
lines changed

script/cibuild

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ set -e
99
REMOTE_VERSIONS="
1010
11.10.344
1111
2.0.0
12+
2.2.0
1213
"
1314

1415
# Enable verbose logging of ssh commands

share/github-backup-utils/ghe-backup-repositories-rsync

Lines changed: 85 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -119,13 +119,35 @@ echo 1>&3
119119
echo "* Transferring auxiliary files ..." 1>&3
120120
rsync_repository_data -z <<RULES
121121
- /__*__/
122+
- /info/
123+
122124
+ /*/
123125
+ /*/*.git
124126
- /*/*.git/objects
125127
- /*/*.git/refs
126128
- /*/*.git/packed-refs
127129
- /*/*.git/logs
128130
+ /*/*.git/**
131+
132+
+ /*/??/
133+
+ /*/??/??/
134+
+ /*/??/??/??/
135+
+ /*/??/??/??/gist/
136+
+ /*/??/??/??/gist/*.git
137+
- /*/??/??/??/gist/*.git/objects
138+
- /*/??/??/??/gist/*.git/refs
139+
- /*/??/??/??/gist/*.git/packed-refs
140+
- /*/??/??/??/gist/*.git/logs
141+
+ /*/??/??/??/gist/*.git/**
142+
143+
+ /*/nw/??/??/??/
144+
+ /*/nw/??/??/??/*/
145+
+ /*/nw/??/??/??/*/*.git
146+
- /*/nw/??/??/??/*/*.git/objects
147+
- /*/nw/??/??/??/*/*.git/refs
148+
- /*/nw/??/??/??/*/*.git/packed-refs
149+
- /*/nw/??/??/??/*/*.git/logs
150+
+ /*/nw/??/??/??/*/*.git/**
129151
RULES
130152

131153
# Sync packed refs files. This is performed before sync'ing loose refs since
@@ -134,9 +156,25 @@ echo 1>&3
134156
echo "* Transferring packed-refs files ..." 1>&3
135157
rsync_repository_data -z <<RULES
136158
- /__*__/
159+
- /info/
160+
137161
+ /*/
138162
+ /*/*.git
139163
+ /*/*.git/packed-refs
164+
165+
+ /*/??/
166+
+ /*/??/??/
167+
+ /*/??/??/??/
168+
+ /*/??/??/??/gist/
169+
+ /*/??/??/??/gist/*.git
170+
+ /*/??/??/??/gist/*.git/packed-refs
171+
172+
+ /*/nw/??/
173+
+ /*/nw/??/??/
174+
+ /*/nw/??/??/??/
175+
+ /*/nw/??/??/??/*/
176+
+ /*/nw/??/??/??/*/*.git
177+
+ /*/nw/??/??/??/*/*.git/packed-refs
140178
RULES
141179

142180
# Sync loose refs and reflogs. This must be performed before object data is
@@ -145,25 +183,67 @@ echo 1>&3
145183
echo "* Transferring refs and reflogs ..." 1>&3
146184
rsync_repository_data -z <<RULES
147185
- /__*__/
186+
- /info/
187+
148188
+ /*/
149189
+ /*/*.git
150190
+ /*/*.git/refs
151191
+ /*/*.git/refs/**
152192
+ /*/*.git/logs
153193
+ /*/*.git/logs/**
194+
195+
+ /*/??/
196+
+ /*/??/??/
197+
+ /*/??/??/??/
198+
+ /*/??/??/??/gist/
199+
+ /*/??/??/??/gist/*.git
200+
+ /*/??/??/??/gist/*.git/refs
201+
+ /*/??/??/??/gist/*.git/refs/**
202+
+ /*/??/??/??/gist/*.git/logs
203+
+ /*/??/??/??/gist/*.git/logs/**
204+
205+
+ /*/nw/??/
206+
+ /*/nw/??/??/
207+
+ /*/nw/??/??/??/
208+
+ /*/nw/??/??/??/*/
209+
+ /*/nw/??/??/??/*/*.git
210+
+ /*/nw/??/??/??/*/*.git/refs
211+
+ /*/nw/??/??/??/*/*.git/refs/**
212+
+ /*/nw/??/??/??/*/*.git/logs
213+
+ /*/nw/??/??/??/*/*.git/logs/**
154214
RULES
155215

156-
# Sync git objects and pack files. Compressing is disabled during this phase
216+
# Sync git objects and pack files. Compression is disabled during this phase
157217
# since these files are already well compressed.
158218
echo 1>&3
159219
echo "* Transferring objects and packs ..." 1>&3
160220
rsync_repository_data -H <<RULES
161221
- /__*__/
222+
- /info/
223+
162224
+ /*/
163225
+ /*/*.git
164226
+ /*/*.git/objects
165-
- tmp_*
227+
- /*/*.git/objects/**/tmp_*
166228
+ /*/*.git/objects/**
229+
230+
+ /*/??/
231+
+ /*/??/??/
232+
+ /*/??/??/??/
233+
+ /*/??/??/??/gist/
234+
+ /*/??/??/??/gist/*.git
235+
+ /*/??/??/??/gist/*.git/objects
236+
- /*/??/??/??/gist/*.git/objects/**/tmp_*
237+
+ /*/??/??/??/gist/*.git/objects/**
238+
239+
+ /*/nw/??/
240+
+ /*/nw/??/??/
241+
+ /*/nw/??/??/??/
242+
+ /*/nw/??/??/??/*/
243+
+ /*/nw/??/??/??/*/*.git
244+
+ /*/nw/??/??/??/*/*.git/objects
245+
- /*/nw/??/??/??/*/*.git/objects/**/tmp_*
246+
+ /*/nw/??/??/??/*/*.git/objects/**
167247
RULES
168248

169249
# Sync __special__ data directories, including the __alambic_assets__,
@@ -182,5 +262,8 @@ rsync_repository_data <<RULES
182262
- /__render__/
183263
+ /__*__/
184264
+ /__*__/**
265+
+ /info/
266+
- /info/lost+found/
267+
+ /info/*
185268
RULES
186269
echo 1>&3
Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
#!/bin/sh
2+
# ghe-backup-repositories-rsync-nw command tests
3+
# uses the net-shard filesystem layout
4+
5+
# Bring in testlib
6+
. $(dirname "$0")/testlib.sh
7+
8+
# Create the backup data dir and fake remote repositories dirs
9+
mkdir -p "$GHE_DATA_DIR" "$GHE_REMOTE_DATA_USER_DIR/repositories"
10+
mkdir -p "$TRASHDIR/hooks"
11+
12+
# Run under the remote repositories directory
13+
cd "$GHE_REMOTE_DATA_USER_DIR/repositories"
14+
15+
# Create some test repositories in the remote repositories dir
16+
17+
repo1="0/nw/01/aa/3f/1234/1234.git"
18+
repo2="0/nw/01/aa/3f/1234/1235.git"
19+
repo3="1/nw/23/bb/4c/2345/2345.git"
20+
mkdir -p "$repo1" "$repo2" "$repo3"
21+
22+
wiki1="0/nw/01/aa/3f/1234/1234.wiki.git"
23+
mkdir -p "$wiki1"
24+
25+
gist1="0/01/aa/3f/gist/93069ad4c391b6203f183e147d52a97a.git"
26+
gist2="1/23/bb/4c/gist/1234.git"
27+
mkdir -p "$gist1" "$gist2"
28+
29+
# Initialize test repositories with a fake commit
30+
for repo in $(find . -type d -name '*.git' -prune); do
31+
git init -q --bare "$repo"
32+
git --git-dir="$repo" --work-tree=. commit -q --allow-empty -m 'test commit'
33+
rm -rf "$repo/hooks"
34+
ln -s "$TRASHDIR/hooks" "$repo/hooks"
35+
done
36+
37+
# Generate a packed-refs file in repo1
38+
git --git-dir="$repo1" pack-refs
39+
40+
# Generate a pack in repo2
41+
git --git-dir="$repo2" repack -q
42+
43+
# Add some fake svn data to repo3
44+
echo "fake svn history data" > "$repo3/svn.history.msgpack"
45+
mkdir "$repo3/svn_data"
46+
echo "fake property history data" > "$repo3/svn_data/property_history.msgpack"
47+
48+
# Touch the info/nw-layout file, marking this as a net-shard fs
49+
mkdir -p "info"
50+
date "+%s" > info/nw-layout
51+
52+
begin_test "ghe-backup-repositories-rsync first snapshot (nw)"
53+
(
54+
set -e
55+
56+
# force snapshot number instead of generating a timestamp
57+
GHE_SNAPSHOT_TIMESTAMP=1
58+
export GHE_SNAPSHOT_TIMESTAMP
59+
60+
# run it
61+
ghe-backup-repositories-rsync
62+
63+
# check that repositories directory was created
64+
[ -d "$GHE_DATA_DIR/1/repositories" ]
65+
66+
# check that packed-refs file was transferred
67+
[ -f "$GHE_DATA_DIR/1/repositories/$repo1/packed-refs" ]
68+
69+
# check that a pack file was transferred
70+
[ -f "$GHE_DATA_DIR"/1/repositories/$repo2/objects/pack/*.pack ]
71+
72+
# check that svn data was transferred
73+
[ -f "$GHE_DATA_DIR"/1/repositories/$repo3/svn.history.msgpack ]
74+
[ -f "$GHE_DATA_DIR"/1/repositories/$repo3/svn_data/property_history.msgpack ]
75+
76+
# verify all repository data was transferred
77+
diff -ru "$GHE_REMOTE_DATA_USER_DIR/repositories" "$GHE_DATA_DIR/1/repositories"
78+
)
79+
end_test
80+
81+
82+
begin_test "ghe-backup-repositories-rsync subsequent snapshot (nw)"
83+
(
84+
set -e
85+
86+
# force snapshot number instead of generating a timestamp
87+
GHE_SNAPSHOT_TIMESTAMP=2
88+
export GHE_SNAPSHOT_TIMESTAMP
89+
90+
# make current symlink point to previous increment
91+
ln -s 1 "$GHE_DATA_DIR/current"
92+
93+
# run it
94+
ghe-backup-repositories-rsync
95+
96+
# check that repositories directory was created
97+
snapshot="$GHE_DATA_DIR/2/repositories"
98+
[ -d "$snapshot" ]
99+
100+
# verify hard links used for existing files
101+
inode1=$(ls -i "$GHE_DATA_DIR/1/repositories/$repo1/packed-refs" | awk '{ print $1; }')
102+
inode2=$(ls -i "$GHE_DATA_DIR/2/repositories/$repo1/packed-refs" | awk '{ print $1; }')
103+
[ "$inode1" = "$inode2" ]
104+
105+
# verify all repository data exists in the increment
106+
diff -ru "$GHE_REMOTE_DATA_USER_DIR/repositories" "$GHE_DATA_DIR/2/repositories"
107+
)
108+
end_test
109+
110+
111+
begin_test "ghe-backup-repositories-rsync temp files (nw)"
112+
(
113+
set -e
114+
115+
# force snapshot number instead of generating a timestamp
116+
GHE_SNAPSHOT_TIMESTAMP=4
117+
export GHE_SNAPSHOT_TIMESTAMP
118+
119+
# create a tmp pack to emulate a pack being written to
120+
touch "$GHE_REMOTE_DATA_USER_DIR/repositories/$repo1/objects/pack/tmp_pack_1234"
121+
122+
# run it
123+
ghe-backup-repositories-rsync
124+
125+
# check that repositories directory was created
126+
snapshot="$GHE_DATA_DIR/4/repositories"
127+
[ -d "$snapshot" ]
128+
129+
# check that tmp pack was not transferred
130+
[ ! -d "$snapshot/$repo1/objects/pack/tmp_pack_1234" ]
131+
)
132+
end_test

0 commit comments

Comments
 (0)