Skip to content

Commit b47d807

Browse files
larsxschneidergitster
larsxschneider
authored and
gitster
committed
git-p4: add Git LFS backend for large file system
Add example implementation including test cases for the large file system using Git LFS. Pushing files to the Git LFS server is not tested. Signed-off-by: Lars Schneider <[email protected]> Signed-off-by: Junio C Hamano <[email protected]>
1 parent a5db4b1 commit b47d807

File tree

2 files changed

+360
-0
lines changed

2 files changed

+360
-0
lines changed

‎git-p4.py

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1038,6 +1038,78 @@ def pushFile(self, localLargeFile):
10381038
os.makedirs(remotePath)
10391039
shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
10401040

1041+
class GitLFS(LargeFileSystem):
1042+
"""Git LFS as backend for the git-p4 large file system.
1043+
See https://git-lfs.github.com/ for details."""
1044+
1045+
def __init__(self, *args):
1046+
LargeFileSystem.__init__(self, *args)
1047+
self.baseGitAttributes = []
1048+
1049+
def generatePointer(self, contentFile):
1050+
"""Generate a Git LFS pointer for the content. Return LFS Pointer file
1051+
mode and content which is stored in the Git repository instead of
1052+
the actual content. Return also the new location of the actual
1053+
content.
1054+
"""
1055+
pointerProcess = subprocess.Popen(
1056+
['git', 'lfs', 'pointer', '--file=' + contentFile],
1057+
stdout=subprocess.PIPE
1058+
)
1059+
pointerFile = pointerProcess.stdout.read()
1060+
if pointerProcess.wait():
1061+
os.remove(contentFile)
1062+
die('git-lfs pointer command failed. Did you install the extension?')
1063+
pointerContents = [i+'\n' for i in pointerFile.split('\n')[2:][:-1]]
1064+
oid = pointerContents[1].split(' ')[1].split(':')[1][:-1]
1065+
localLargeFile = os.path.join(
1066+
os.getcwd(),
1067+
'.git', 'lfs', 'objects', oid[:2], oid[2:4],
1068+
oid,
1069+
)
1070+
# LFS Spec states that pointer files should not have the executable bit set.
1071+
gitMode = '100644'
1072+
return (gitMode, pointerContents, localLargeFile)
1073+
1074+
def pushFile(self, localLargeFile):
1075+
uploadProcess = subprocess.Popen(
1076+
['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1077+
)
1078+
if uploadProcess.wait():
1079+
die('git-lfs push command failed. Did you define a remote?')
1080+
1081+
def generateGitAttributes(self):
1082+
return (
1083+
self.baseGitAttributes +
1084+
[
1085+
'\n',
1086+
'#\n',
1087+
'# Git LFS (see https://git-lfs.github.com/)\n',
1088+
'#\n',
1089+
] +
1090+
['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1091+
for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1092+
] +
1093+
['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1094+
for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1095+
]
1096+
)
1097+
1098+
def addLargeFile(self, relPath):
1099+
LargeFileSystem.addLargeFile(self, relPath)
1100+
self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1101+
1102+
def removeLargeFile(self, relPath):
1103+
LargeFileSystem.removeLargeFile(self, relPath)
1104+
self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1105+
1106+
def processContent(self, git_mode, relPath, contents):
1107+
if relPath == '.gitattributes':
1108+
self.baseGitAttributes = contents
1109+
return (git_mode, self.generateGitAttributes())
1110+
else:
1111+
return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1112+
10411113
class Command:
10421114
def __init__(self):
10431115
self.usage = "usage: %prog [options]"

‎t/t9824-git-p4-git-lfs.sh

Lines changed: 288 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,288 @@
1+
#!/bin/sh
2+
3+
test_description='Clone repositories and store files in Git LFS'
4+
5+
. ./lib-git-p4.sh
6+
7+
git lfs help >/dev/null 2>&1 || {
8+
skip_all='skipping git p4 Git LFS tests; Git LFS not found'
9+
test_done
10+
}
11+
12+
test_file_in_lfs () {
13+
FILE="$1" &&
14+
SIZE="$2" &&
15+
EXPECTED_CONTENT="$3" &&
16+
cat "$FILE" | grep "size $SIZE" &&
17+
HASH=$(cat "$FILE" | grep "oid sha256:" | sed -e "s/oid sha256://g") &&
18+
LFS_FILE=".git/lfs/objects/$(echo "$HASH" | cut -c1-2)/$(echo "$HASH" | cut -c3-4)/$HASH" &&
19+
echo $EXPECTED_CONTENT >expect &&
20+
test_path_is_file "$FILE" &&
21+
test_path_is_file "$LFS_FILE" &&
22+
test_cmp expect "$LFS_FILE"
23+
}
24+
25+
test_file_count_in_dir () {
26+
DIR="$1" &&
27+
EXPECTED_COUNT="$2" &&
28+
find "$DIR" -type f >actual &&
29+
test_line_count = $EXPECTED_COUNT actual
30+
}
31+
32+
test_expect_success 'start p4d' '
33+
start_p4d
34+
'
35+
36+
test_expect_success 'Create repo with binary files' '
37+
client_view "//depot/... //client/..." &&
38+
(
39+
cd "$cli" &&
40+
41+
echo "content 1 txt 23 bytes" >file1.txt &&
42+
p4 add file1.txt &&
43+
echo "content 2-3 bin 25 bytes" >file2.dat &&
44+
p4 add file2.dat &&
45+
p4 submit -d "Add text and binary file" &&
46+
47+
mkdir "path with spaces" &&
48+
echo "content 2-3 bin 25 bytes" >"path with spaces/file3.bin" &&
49+
p4 add "path with spaces/file3.bin" &&
50+
p4 submit -d "Add another binary file with same content and spaces in path" &&
51+
52+
echo "content 4 bin 26 bytes XX" >file4.bin &&
53+
p4 add file4.bin &&
54+
p4 submit -d "Add another binary file with different content"
55+
)
56+
'
57+
58+
test_expect_success 'Store files in LFS based on size (>24 bytes)' '
59+
client_view "//depot/... //client/..." &&
60+
test_when_finished cleanup_git &&
61+
(
62+
cd "$git" &&
63+
git init . &&
64+
git config git-p4.useClientSpec true &&
65+
git config git-p4.largeFileSystem GitLFS &&
66+
git config git-p4.largeFileThreshold 24 &&
67+
git p4 clone --destination="$git" //depot@all &&
68+
69+
test_file_in_lfs file2.dat 25 "content 2-3 bin 25 bytes" &&
70+
test_file_in_lfs "path with spaces/file3.bin" 25 "content 2-3 bin 25 bytes" &&
71+
test_file_in_lfs file4.bin 26 "content 4 bin 26 bytes XX" &&
72+
73+
test_file_count_in_dir ".git/lfs/objects" 2 &&
74+
75+
cat >expect <<-\EOF &&
76+
77+
#
78+
# Git LFS (see https://git-lfs.github.com/)
79+
#
80+
/file2.dat filter=lfs -text
81+
/file4.bin filter=lfs -text
82+
/path[[:space:]]with[[:space:]]spaces/file3.bin filter=lfs -text
83+
EOF
84+
test_path_is_file .gitattributes &&
85+
test_cmp expect .gitattributes
86+
)
87+
'
88+
89+
test_expect_success 'Store files in LFS based on size (>25 bytes)' '
90+
client_view "//depot/... //client/..." &&
91+
test_when_finished cleanup_git &&
92+
(
93+
cd "$git" &&
94+
git init . &&
95+
git config git-p4.useClientSpec true &&
96+
git config git-p4.largeFileSystem GitLFS &&
97+
git config git-p4.largeFileThreshold 25 &&
98+
git p4 clone --destination="$git" //depot@all &&
99+
100+
test_file_in_lfs file4.bin 26 "content 4 bin 26 bytes XX" &&
101+
test_file_count_in_dir ".git/lfs/objects" 1 &&
102+
103+
cat >expect <<-\EOF &&
104+
105+
#
106+
# Git LFS (see https://git-lfs.github.com/)
107+
#
108+
/file4.bin filter=lfs -text
109+
EOF
110+
test_path_is_file .gitattributes &&
111+
test_cmp expect .gitattributes
112+
)
113+
'
114+
115+
test_expect_success 'Store files in LFS based on extension (dat)' '
116+
client_view "//depot/... //client/..." &&
117+
test_when_finished cleanup_git &&
118+
(
119+
cd "$git" &&
120+
git init . &&
121+
git config git-p4.useClientSpec true &&
122+
git config git-p4.largeFileSystem GitLFS &&
123+
git config git-p4.largeFileExtensions dat &&
124+
git p4 clone --destination="$git" //depot@all &&
125+
126+
test_file_in_lfs file2.dat 25 "content 2-3 bin 25 bytes" &&
127+
test_file_count_in_dir ".git/lfs/objects" 1 &&
128+
129+
cat >expect <<-\EOF &&
130+
131+
#
132+
# Git LFS (see https://git-lfs.github.com/)
133+
#
134+
*.dat filter=lfs -text
135+
EOF
136+
test_path_is_file .gitattributes &&
137+
test_cmp expect .gitattributes
138+
)
139+
'
140+
141+
test_expect_success 'Store files in LFS based on size (>25 bytes) and extension (dat)' '
142+
client_view "//depot/... //client/..." &&
143+
test_when_finished cleanup_git &&
144+
(
145+
cd "$git" &&
146+
git init . &&
147+
git config git-p4.useClientSpec true &&
148+
git config git-p4.largeFileSystem GitLFS &&
149+
git config git-p4.largeFileExtensions dat &&
150+
git config git-p4.largeFileThreshold 25 &&
151+
git p4 clone --destination="$git" //depot@all &&
152+
153+
test_file_in_lfs file2.dat 25 "content 2-3 bin 25 bytes" &&
154+
test_file_in_lfs file4.bin 26 "content 4 bin 26 bytes XX" &&
155+
test_file_count_in_dir ".git/lfs/objects" 2 &&
156+
157+
cat >expect <<-\EOF &&
158+
159+
#
160+
# Git LFS (see https://git-lfs.github.com/)
161+
#
162+
*.dat filter=lfs -text
163+
/file4.bin filter=lfs -text
164+
EOF
165+
test_path_is_file .gitattributes &&
166+
test_cmp expect .gitattributes
167+
)
168+
'
169+
170+
test_expect_success 'Remove file from repo and store files in LFS based on size (>24 bytes)' '
171+
client_view "//depot/... //client/..." &&
172+
(
173+
cd "$cli" &&
174+
p4 delete file4.bin &&
175+
p4 submit -d "Remove file"
176+
) &&
177+
178+
client_view "//depot/... //client/..." &&
179+
test_when_finished cleanup_git &&
180+
(
181+
cd "$git" &&
182+
git init . &&
183+
git config git-p4.useClientSpec true &&
184+
git config git-p4.largeFileSystem GitLFS &&
185+
git config git-p4.largeFileThreshold 24 &&
186+
git p4 clone --destination="$git" //depot@all &&
187+
188+
test_file_in_lfs file2.dat 25 "content 2-3 bin 25 bytes" &&
189+
test_file_in_lfs "path with spaces/file3.bin" 25 "content 2-3 bin 25 bytes" &&
190+
test_path_is_missing file4.bin &&
191+
test_file_count_in_dir ".git/lfs/objects" 2 &&
192+
193+
cat >expect <<-\EOF &&
194+
195+
#
196+
# Git LFS (see https://git-lfs.github.com/)
197+
#
198+
/file2.dat filter=lfs -text
199+
/path[[:space:]]with[[:space:]]spaces/file3.bin filter=lfs -text
200+
EOF
201+
test_path_is_file .gitattributes &&
202+
test_cmp expect .gitattributes
203+
)
204+
'
205+
206+
test_expect_success 'Add .gitattributes and store files in LFS based on size (>24 bytes)' '
207+
client_view "//depot/... //client/..." &&
208+
(
209+
cd "$cli" &&
210+
echo "*.txt text" >.gitattributes &&
211+
p4 add .gitattributes &&
212+
p4 submit -d "Add .gitattributes"
213+
) &&
214+
215+
client_view "//depot/... //client/..." &&
216+
test_when_finished cleanup_git &&
217+
(
218+
cd "$git" &&
219+
git init . &&
220+
git config git-p4.useClientSpec true &&
221+
git config git-p4.largeFileSystem GitLFS &&
222+
git config git-p4.largeFileThreshold 24 &&
223+
git p4 clone --destination="$git" //depot@all &&
224+
225+
test_file_in_lfs file2.dat 25 "content 2-3 bin 25 bytes" &&
226+
test_file_in_lfs "path with spaces/file3.bin" 25 "content 2-3 bin 25 bytes" &&
227+
test_path_is_missing file4.bin &&
228+
test_file_count_in_dir ".git/lfs/objects" 2 &&
229+
230+
cat >expect <<-\EOF &&
231+
*.txt text
232+
233+
#
234+
# Git LFS (see https://git-lfs.github.com/)
235+
#
236+
/file2.dat filter=lfs -text
237+
/path[[:space:]]with[[:space:]]spaces/file3.bin filter=lfs -text
238+
EOF
239+
test_path_is_file .gitattributes &&
240+
test_cmp expect .gitattributes
241+
)
242+
'
243+
244+
test_expect_success 'Add big files to repo and store files in LFS based on compressed size (>28 bytes)' '
245+
client_view "//depot/... //client/..." &&
246+
(
247+
cd "$cli" &&
248+
echo "content 5 bin 40 bytes XXXXXXXXXXXXXXXX" >file5.bin &&
249+
p4 add file5.bin &&
250+
p4 submit -d "Add file with small footprint after compression" &&
251+
252+
echo "content 6 bin 39 bytes XXXXXYYYYYZZZZZ" >file6.bin &&
253+
p4 add file6.bin &&
254+
p4 submit -d "Add file with large footprint after compression"
255+
) &&
256+
257+
client_view "//depot/... //client/..." &&
258+
test_when_finished cleanup_git &&
259+
(
260+
cd "$git" &&
261+
git init . &&
262+
git config git-p4.useClientSpec true &&
263+
git config git-p4.largeFileSystem GitLFS &&
264+
git config git-p4.largeFileCompressedThreshold 28 &&
265+
# We only import HEAD here ("@all" is missing!)
266+
git p4 clone --destination="$git" //depot &&
267+
268+
test_file_in_lfs file6.bin 13 "content 6 bin 39 bytes XXXXXYYYYYZZZZZ"
269+
test_file_count_in_dir ".git/lfs/objects" 1 &&
270+
271+
cat >expect <<-\EOF &&
272+
*.txt text
273+
274+
#
275+
# Git LFS (see https://git-lfs.github.com/)
276+
#
277+
/file6.bin filter=lfs -text
278+
EOF
279+
test_path_is_file .gitattributes &&
280+
test_cmp expect .gitattributes
281+
)
282+
'
283+
284+
test_expect_success 'kill p4d' '
285+
kill_p4d
286+
'
287+
288+
test_done

0 commit comments

Comments
 (0)