Skip to content

Commit e09ab7d

Browse files
jrnjrn
committed
Merge branch 'db/text-delta' into svn-fe
* db/text-delta: vcs-svn: avoid hangs from corrupt deltas vcs-svn: guard against overflow when computing preimage length vcs-svn: implement text-delta handling
2 parents a8d3d26 + 3ac10b2 commit e09ab7d

File tree

5 files changed

+280
-14
lines changed

5 files changed

+280
-14
lines changed

‎contrib/svn-fe/svn-fe.txt

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ SYNOPSIS
99
--------
1010
[verse]
1111
mkfifo backchannel &&
12-
svnadmin dump --incremental REPO |
12+
svnadmin dump --deltas REPO |
1313
svn-fe [url] 3<backchannel |
1414
git fast-import --cat-blob-fd=3 3>backchannel
1515

@@ -32,9 +32,6 @@ Subversion's repository dump format is documented in full in
3232
Files in this format can be generated using the 'svnadmin dump' or
3333
'svk admin dump' command.
3434

35-
Dumps produced with 'svnadmin dump --deltas' (dumpfile format v3)
36-
are not supported.
37-
3835
OUTPUT FORMAT
3936
-------------
4037
The fast-import format is documented by the git-fast-import(1)

‎t/t9010-svn-fe.sh

Lines changed: 143 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,13 @@ reinit_git () {
1818

1919
try_dump () {
2020
input=$1 &&
21-
maybe_fail=${2:+test_$2} &&
21+
maybe_fail_svnfe=${2:+test_$2} &&
22+
maybe_fail_fi=${3:+test_$3} &&
2223

2324
{
24-
$maybe_fail test-svn-fe "$input" >stream 3<backflow &
25+
$maybe_fail_svnfe test-svn-fe "$input" >stream 3<backflow &
2526
} &&
26-
git fast-import --cat-blob-fd=3 <stream 3>backflow &&
27+
$maybe_fail_fi git fast-import --cat-blob-fd=3 <stream 3>backflow &&
2728
wait $!
2829
}
2930

@@ -674,7 +675,7 @@ test_expect_success PIPE 'change file mode and reiterate content' '
674675
test_cmp hello actual.target
675676
'
676677

677-
test_expect_success PIPE 'deltas not supported' '
678+
test_expect_success PIPE 'deltas supported' '
678679
reinit_git &&
679680
{
680681
# (old) h + (inline) ello + (old) \n
@@ -735,7 +736,7 @@ test_expect_success PIPE 'deltas not supported' '
735736
echo PROPS-END &&
736737
cat delta
737738
} >delta.dump &&
738-
test_must_fail try_dump delta.dump
739+
try_dump delta.dump
739740
'
740741

741742
test_expect_success PIPE 'property deltas supported' '
@@ -942,6 +943,143 @@ test_expect_success PIPE 'deltas for typechange' '
942943
test_cmp expect actual
943944
'
944945

946+
test_expect_success PIPE 'deltas need not consume the whole preimage' '
947+
reinit_git &&
948+
cat >expect <<-\EOF &&
949+
OBJID
950+
:120000 100644 OBJID OBJID T postimage
951+
OBJID
952+
:100644 120000 OBJID OBJID T postimage
953+
OBJID
954+
:000000 100644 OBJID OBJID A postimage
955+
EOF
956+
echo "first preimage" >expect.1 &&
957+
printf target >expect.2 &&
958+
printf lnk >expect.3 &&
959+
{
960+
printf "SVNQ%b%b%b" "QQ\017\001\017" "\0217" "first preimage\n" |
961+
q_to_nul
962+
} >delta.1 &&
963+
{
964+
properties svn:special "*" &&
965+
echo PROPS-END
966+
} >symlink.props &&
967+
{
968+
printf "SVNQ%b%b%b" "Q\002\013\004\012" "\0201\001\001\0211" "lnk target" |
969+
q_to_nul
970+
} >delta.2 &&
971+
{
972+
printf "SVNQ%b%b" "Q\004\003\004Q" "\001Q\002\002" |
973+
q_to_nul
974+
} >delta.3 &&
975+
{
976+
cat <<-\EOF &&
977+
SVN-fs-dump-format-version: 3
978+
979+
Revision-number: 1
980+
Prop-content-length: 10
981+
Content-length: 10
982+
983+
PROPS-END
984+
985+
Node-path: postimage
986+
Node-kind: file
987+
Node-action: add
988+
Text-delta: true
989+
Prop-content-length: 10
990+
EOF
991+
echo Text-content-length: $(wc -c <delta.1) &&
992+
echo Content-length: $((10 + $(wc -c <delta.1))) &&
993+
echo &&
994+
echo PROPS-END &&
995+
cat delta.1 &&
996+
cat <<-\EOF &&
997+
998+
Revision-number: 2
999+
Prop-content-length: 10
1000+
Content-length: 10
1001+
1002+
PROPS-END
1003+
1004+
Node-path: postimage
1005+
Node-kind: file
1006+
Node-action: change
1007+
Text-delta: true
1008+
EOF
1009+
echo Prop-content-length: $(wc -c <symlink.props) &&
1010+
echo Text-content-length: $(wc -c <delta.2) &&
1011+
echo Content-length: $(($(wc -c <symlink.props) + $(wc -c <delta.2))) &&
1012+
echo &&
1013+
cat symlink.props &&
1014+
cat delta.2 &&
1015+
cat <<-\EOF &&
1016+
1017+
Revision-number: 3
1018+
Prop-content-length: 10
1019+
Content-length: 10
1020+
1021+
PROPS-END
1022+
1023+
Node-path: postimage
1024+
Node-kind: file
1025+
Node-action: change
1026+
Text-delta: true
1027+
Prop-content-length: 10
1028+
EOF
1029+
echo Text-content-length: $(wc -c <delta.3) &&
1030+
echo Content-length: $((10 + $(wc -c <delta.3))) &&
1031+
echo &&
1032+
echo PROPS-END &&
1033+
cat delta.3 &&
1034+
echo
1035+
} >deltapartial.dump &&
1036+
try_dump deltapartial.dump &&
1037+
{
1038+
git rev-list HEAD |
1039+
git diff-tree --root --stdin |
1040+
sed "s/$_x40/OBJID/g"
1041+
} >actual &&
1042+
test_cmp expect actual &&
1043+
git show HEAD:postimage >actual.3 &&
1044+
git show HEAD^:postimage >actual.2 &&
1045+
git show HEAD^^:postimage >actual.1 &&
1046+
test_cmp expect.1 actual.1 &&
1047+
test_cmp expect.2 actual.2 &&
1048+
test_cmp expect.3 actual.3
1049+
'
1050+
1051+
test_expect_success PIPE 'no hang for delta trying to read past end of preimage' '
1052+
reinit_git &&
1053+
{
1054+
# COPY 1
1055+
printf "SVNQ%b%b" "Q\001\001\002Q" "\001Q" |
1056+
q_to_nul
1057+
} >greedy.delta &&
1058+
{
1059+
cat <<-\EOF &&
1060+
SVN-fs-dump-format-version: 3
1061+
1062+
Revision-number: 1
1063+
Prop-content-length: 10
1064+
Content-length: 10
1065+
1066+
PROPS-END
1067+
1068+
Node-path: bootstrap
1069+
Node-kind: file
1070+
Node-action: add
1071+
Text-delta: true
1072+
Prop-content-length: 10
1073+
EOF
1074+
echo Text-content-length: $(wc -c <greedy.delta) &&
1075+
echo Content-length: $((10 + $(wc -c <greedy.delta))) &&
1076+
echo &&
1077+
echo PROPS-END &&
1078+
cat greedy.delta &&
1079+
echo
1080+
} >greedydelta.dump &&
1081+
try_dump greedydelta.dump must_fail might_fail
1082+
'
9451083

9461084
test_expect_success 'set up svn repo' '
9471085
svnconf=$PWD/svnconf &&

‎vcs-svn/fast_export.c

Lines changed: 124 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,15 +7,38 @@
77
#include "strbuf.h"
88
#include "quote.h"
99
#include "fast_export.h"
10-
#include "line_buffer.h"
1110
#include "repo_tree.h"
1211
#include "strbuf.h"
12+
#include "svndiff.h"
13+
#include "sliding_window.h"
14+
#include "line_buffer.h"
1315

1416
#define MAX_GITSVN_LINE_LEN 4096
17+
#define REPORT_FILENO 3
1518

1619
static uint32_t first_commit_done;
20+
static struct line_buffer postimage = LINE_BUFFER_INIT;
1721
static struct line_buffer report_buffer = LINE_BUFFER_INIT;
1822

23+
/* NEEDSWORK: move to fast_export_init() */
24+
static int init_postimage(void)
25+
{
26+
static int postimage_initialized;
27+
if (postimage_initialized)
28+
return 0;
29+
postimage_initialized = 1;
30+
return buffer_tmpfile_init(&postimage);
31+
}
32+
33+
static int init_report_buffer(int fd)
34+
{
35+
static int report_buffer_initialized;
36+
if (report_buffer_initialized)
37+
return 0;
38+
report_buffer_initialized = 1;
39+
return buffer_fdinit(&report_buffer, fd);
40+
}
41+
1942
void fast_export_init(int fd)
2043
{
2144
if (buffer_fdinit(&report_buffer, fd))
@@ -132,6 +155,89 @@ static void die_short_read(struct line_buffer *input)
132155
die("invalid dump: unexpected end of file");
133156
}
134157

158+
static int ends_with(const char *s, size_t len, const char *suffix)
159+
{
160+
const size_t suffixlen = strlen(suffix);
161+
if (len < suffixlen)
162+
return 0;
163+
return !memcmp(s + len - suffixlen, suffix, suffixlen);
164+
}
165+
166+
static int parse_cat_response_line(const char *header, off_t *len)
167+
{
168+
size_t headerlen = strlen(header);
169+
uintmax_t n;
170+
const char *type;
171+
const char *end;
172+
173+
if (ends_with(header, headerlen, " missing"))
174+
return error("cat-blob reports missing blob: %s", header);
175+
type = memmem(header, headerlen, " blob ", strlen(" blob "));
176+
if (!type)
177+
return error("cat-blob header has wrong object type: %s", header);
178+
n = strtoumax(type + strlen(" blob "), (char **) &end, 10);
179+
if (end == type + strlen(" blob "))
180+
return error("cat-blob header does not contain length: %s", header);
181+
if (memchr(type + strlen(" blob "), '-', end - type - strlen(" blob ")))
182+
return error("cat-blob header contains negative length: %s", header);
183+
if (n == UINTMAX_MAX || n > maximum_signed_value_of_type(off_t))
184+
return error("blob too large for current definition of off_t");
185+
*len = n;
186+
if (*end)
187+
return error("cat-blob header contains garbage after length: %s", header);
188+
return 0;
189+
}
190+
191+
static void check_preimage_overflow(off_t a, off_t b)
192+
{
193+
if (signed_add_overflows(a, b))
194+
die("blob too large for current definition of off_t");
195+
}
196+
197+
static long apply_delta(off_t len, struct line_buffer *input,
198+
const char *old_data, uint32_t old_mode)
199+
{
200+
long ret;
201+
struct sliding_view preimage = SLIDING_VIEW_INIT(&report_buffer, 0);
202+
FILE *out;
203+
204+
if (init_postimage() || !(out = buffer_tmpfile_rewind(&postimage)))
205+
die("cannot open temporary file for blob retrieval");
206+
if (init_report_buffer(REPORT_FILENO))
207+
die("cannot open fd 3 for feedback from fast-import");
208+
if (old_data) {
209+
const char *response;
210+
printf("cat-blob %s\n", old_data);
211+
fflush(stdout);
212+
response = get_response_line();
213+
if (parse_cat_response_line(response, &preimage.max_off))
214+
die("invalid cat-blob response: %s", response);
215+
check_preimage_overflow(preimage.max_off, 1);
216+
}
217+
if (old_mode == REPO_MODE_LNK) {
218+
strbuf_addstr(&preimage.buf, "link ");
219+
check_preimage_overflow(preimage.max_off, strlen("link "));
220+
preimage.max_off += strlen("link ");
221+
check_preimage_overflow(preimage.max_off, 1);
222+
}
223+
if (svndiff0_apply(input, len, &preimage, out))
224+
die("cannot apply delta");
225+
if (old_data) {
226+
/* Read the remainder of preimage and trailing newline. */
227+
assert(!signed_add_overflows(preimage.max_off, 1));
228+
preimage.max_off++; /* room for newline */
229+
if (move_window(&preimage, preimage.max_off - 1, 1))
230+
die("cannot seek to end of input");
231+
if (preimage.buf.buf[0] != '\n')
232+
die("missing newline after cat-blob response");
233+
}
234+
ret = buffer_tmpfile_prepare_to_read(&postimage);
235+
if (ret < 0)
236+
die("cannot read temporary file for blob retrieval");
237+
strbuf_release(&preimage.buf);
238+
return ret;
239+
}
240+
135241
void fast_export_data(uint32_t mode, uint32_t len, struct line_buffer *input)
136242
{
137243
if (mode == REPO_MODE_LNK) {
@@ -199,3 +305,20 @@ int fast_export_ls(const char *path, uint32_t *mode, struct strbuf *dataref)
199305
ls_from_active_commit(path);
200306
return parse_ls_response(get_response_line(), mode, dataref);
201307
}
308+
309+
void fast_export_blob_delta(uint32_t mode,
310+
uint32_t old_mode, const char *old_data,
311+
uint32_t len, struct line_buffer *input)
312+
{
313+
long postimage_len;
314+
if (len > maximum_signed_value_of_type(off_t))
315+
die("enormous delta");
316+
postimage_len = apply_delta((off_t) len, input, old_data, old_mode);
317+
if (mode == REPO_MODE_LNK) {
318+
buffer_skip_bytes(&postimage, strlen("link "));
319+
postimage_len -= strlen("link ");
320+
}
321+
printf("data %ld\n", postimage_len);
322+
buffer_copy_bytes(&postimage, postimage_len);
323+
fputc('\n', stdout);
324+
}

‎vcs-svn/fast_export.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,9 @@ void fast_export_begin_commit(uint32_t revision, const char *author,
1515
const char *url, unsigned long timestamp);
1616
void fast_export_end_commit(uint32_t revision);
1717
void fast_export_data(uint32_t mode, uint32_t len, struct line_buffer *input);
18+
void fast_export_blob_delta(uint32_t mode,
19+
uint32_t old_mode, const char *old_data,
20+
uint32_t len, struct line_buffer *input);
1821

1922
/* If there is no such file at that rev, returns -1, errno == ENOENT. */
2023
int fast_export_ls_rev(uint32_t rev, const char *path,

‎vcs-svn/svndump.c

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -217,9 +217,7 @@ static void handle_node(void)
217217
*/
218218
static const char *const empty_blob = "::empty::";
219219
const char *old_data = NULL;
220-
221-
if (node_ctx.text_delta)
222-
die("text deltas not supported");
220+
uint32_t old_mode = REPO_MODE_BLB;
223221

224222
if (node_ctx.action == NODEACT_DELETE) {
225223
if (have_text || have_props || node_ctx.srcRev)
@@ -255,6 +253,7 @@ static void handle_node(void)
255253
if (mode != REPO_MODE_DIR && type == REPO_MODE_DIR)
256254
die("invalid dump: cannot modify a file into a directory");
257255
node_ctx.type = mode;
256+
old_mode = mode;
258257
} else if (node_ctx.action == NODEACT_ADD) {
259258
if (type == REPO_MODE_DIR)
260259
old_data = NULL;
@@ -289,8 +288,14 @@ static void handle_node(void)
289288
fast_export_modify(node_ctx.dst.buf, node_ctx.type, old_data);
290289
return;
291290
}
291+
if (!node_ctx.text_delta) {
292+
fast_export_modify(node_ctx.dst.buf, node_ctx.type, "inline");
293+
fast_export_data(node_ctx.type, node_ctx.textLength, &input);
294+
return;
295+
}
292296
fast_export_modify(node_ctx.dst.buf, node_ctx.type, "inline");
293-
fast_export_data(node_ctx.type, node_ctx.textLength, &input);
297+
fast_export_blob_delta(node_ctx.type, old_mode, old_data,
298+
node_ctx.textLength, &input);
294299
}
295300

296301
static void begin_revision(void)

0 commit comments

Comments
 (0)