Merge branch 'fa/remote-svn'

A GSoC project.

* fa/remote-svn:
  Add a test script for remote-svn
  remote-svn: add marks-file regeneration
  Add a svnrdump-simulator replaying a dump file for testing
  remote-svn: add incremental import
  remote-svn: Activate import/export-marks for fast-import
  Create a note for every imported commit containing svn metadata
  vcs-svn: add fast_export_note to create notes
  Allow reading svn dumps from files via file:// urls
  remote-svn, vcs-svn: Enable fetching to private refs
  When debug==1, start fast-import with "--stats" instead of "--quiet"
  Add documentation for the 'bidi-import' capability of remote-helpers
  Connect fast-import to the remote-helper via pipe, adding 'bidi-import' capability
  Add argv_array_detach and argv_array_free_detached
  Add svndump_init_fd to allow reading dumps from arbitrary FDs
  Add git-remote-testsvn to Makefile
  Implement a remote helper for svn in C

Author: peff

.gitignore

@@ -125,6 +125,7 @@
 /git-remote-fd
 /git-remote-ext
 /git-remote-testgit
+/git-remote-testsvn
 /git-repack
 /git-replace
 /git-repo-config

Documentation/git-remote-helpers.txt

@@ -98,6 +98,20 @@ advertised with this capability must cover all refs reported by
 the list command.  If no 'refspec' capability is advertised,
 there is an implied `refspec *:*`.
 
+'bidi-import'::
+	The fast-import commands 'cat-blob' and 'ls' can be used by remote-helpers
+	to retrieve information about blobs and trees that already exist in
+	fast-import's memory. This requires a channel from fast-import to the
+	remote-helper.
+	If it is advertised in addition to "import", git establishes a pipe from
+	fast-import to the remote-helper's stdin.
+	It follows that git and fast-import are both connected to the
+	remote-helper's stdin. Because git can send multiple commands to
+	the remote-helper it is required that helpers that use 'bidi-import'
+	buffer all 'import' commands of a batch before sending data to fast-import.
+	This is to prevent mixing commands and fast-import responses on the
+	helper's stdin.
+
 Capabilities for Pushing
 ~~~~~~~~~~~~~~~~~~~~~~~~
 'connect'::
@@ -286,7 +300,12 @@ terminated with a blank line. For each batch of 'import', the remote
 helper should produce a fast-import stream terminated by a 'done'
 command.
 +
-Supported if the helper has the "import" capability.
+Note that if the 'bidi-import' capability is used the complete batch
+sequence has to be buffered before starting to send data to fast-import
+to prevent mixing of commands and fast-import responses on the helper's
+stdin.
++
+Supported if the helper has the 'import' capability.
 
 'connect' <service>::
 	Connects to given service. Standard input and standard output

Documentation/technical/api-argv-array.txt

@@ -53,3 +53,11 @@ Functions
 `argv_array_clear`::
 	Free all memory associated with the array and return it to the
 	initial, empty state.
+
+`argv_array_detach`::
+	Detach the argv array from the `struct argv_array`, transfering
+	ownership of the allocated array and strings.
+
+`argv_array_free_detached`::
+	Free the memory allocated by a `struct argv_array` that was later
+	detached and is now no longer needed.

Makefile

@@ -495,6 +495,7 @@ PROGRAM_OBJS += sh-i18n--envsubst.o
 PROGRAM_OBJS += shell.o
 PROGRAM_OBJS += show-index.o
 PROGRAM_OBJS += upload-pack.o
+PROGRAM_OBJS += remote-testsvn.o
 
 # Binary suffix, set to .exe for Windows builds
 X =
@@ -2449,6 +2450,10 @@ git-http-push$X: revision.o http.o http-push.o GIT-LDFLAGS $(GITLIBS)
 	$(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) \
 		$(LIBS) $(CURL_LIBCURL) $(EXPAT_LIBEXPAT)
 
+git-remote-testsvn$X: remote-testsvn.o GIT-LDFLAGS $(GITLIBS) $(VCSSVN_LIB)
+	$(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) $(LIBS) \
+	$(VCSSVN_LIB)
+
 $(REMOTE_CURL_ALIASES): $(REMOTE_CURL_PRIMARY)
 	$(QUIET_LNCP)$(RM) $@ && \
 	ln $< $@ 2>/dev/null || \

argv-array.c

@@ -68,3 +68,23 @@ void argv_array_clear(struct argv_array *array)
 	}
 	argv_array_init(array);
 }
+
+const char **argv_array_detach(struct argv_array *array, int *argc)
+{
+	const char **argv =
+		array->argv == empty_argv || array->argc == 0 ? NULL : array->argv;
+	if (argc)
+		*argc = array->argc;
+	argv_array_init(array);
+	return argv;
+}
+
+void argv_array_free_detached(const char **argv)
+{
+	if (argv) {
+		int i;
+		for (i = 0; argv[i]; i++)
+			free((char **)argv[i]);
+		free(argv);
+	}
+}

argv-array.h

@@ -18,5 +18,7 @@ void argv_array_pushf(struct argv_array *, const char *fmt, ...);
 void argv_array_pushl(struct argv_array *, ...);
 void argv_array_pop(struct argv_array *);
 void argv_array_clear(struct argv_array *);
+const char **argv_array_detach(struct argv_array *array, int *argc);
+void argv_array_free_detached(const char **argv);
 
 #endif /* ARGV_ARRAY_H */

contrib/svn-fe/svn-fe.c

@@ -10,7 +10,8 @@ int main(int argc, char **argv)
 {
 	if (svndump_init(NULL))
 		return 1;
-	svndump_read((argc > 1) ? argv[1] : NULL);
+	svndump_read((argc > 1) ? argv[1] : NULL, "refs/heads/master",
+			"refs/notes/svn/revs");
 	svndump_deinit();
 	svndump_reset();
 	return 0;

contrib/svn-fe/svnrdump_sim.py

@@ -0,0 +1,53 @@
+#!/usr/bin/python
+"""
+Simulates svnrdump by replaying an existing dump from a file, taking care
+of the specified revision range.
+To simulate incremental imports the environment variable SVNRMAX can be set
+to the highest revision that should be available.
+"""
+import sys, os
+
+
+def getrevlimit():
+        var = 'SVNRMAX'
+        if os.environ.has_key(var):
+                return os.environ[var]
+        return None
+
+def writedump(url, lower, upper):
+        if url.startswith('sim://'):
+                filename = url[6:]
+                if filename[-1] == '/': filename = filename[:-1] #remove terminating slash
+        else:
+                raise ValueError('sim:// url required')
+        f = open(filename, 'r');
+        state = 'header'
+        wroterev = False
+        while(True):
+                l = f.readline()
+                if l == '': break
+                if state == 'header' and l.startswith('Revision-number: '):
+                        state = 'prefix'
+                if state == 'prefix' and l == 'Revision-number: %s\n' % lower:
+                        state = 'selection'
+                if not upper == 'HEAD' and state == 'selection' and l == 'Revision-number: %s\n' % upper:
+                        break;
+
+                if state == 'header' or state == 'selection':
+                        if state == 'selection': wroterev = True
+                        sys.stdout.write(l)
+        return wroterev
+
+if __name__ == "__main__":
+        if not (len(sys.argv) in (3, 4, 5)):
+                print "usage: %s dump URL -rLOWER:UPPER"
+                sys.exit(1)
+        if not sys.argv[1] == 'dump': raise NotImplementedError('only "dump" is suppported.')
+        url = sys.argv[2]
+        r = ('0', 'HEAD')
+        if len(sys.argv) == 4 and sys.argv[3][0:2] == '-r':
+                r = sys.argv[3][2:].lstrip().split(':')
+        if not getrevlimit() is None: r[1] = getrevlimit()
+        if writedump(url, r[0], r[1]): ret = 0
+        else: ret = 1
+        sys.exit(ret)