From 2441ccfe3a294013260db9965dce996ea231dc3a Mon Sep 17 00:00:00 2001 From: Wade Berrier Date: Thu, 24 May 2012 20:47:09 -0600 Subject: [PATCH 01/18] First step at creating 'git-svn-ext' The aim is to take all of these scripts and combine into a single python script with sub commands. It should be easier to implement more advanced features in python --- git-svn-check-unpushed => git-svn-ext | 35 ++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) rename git-svn-check-unpushed => git-svn-ext (80%) diff --git a/git-svn-check-unpushed b/git-svn-ext similarity index 80% rename from git-svn-check-unpushed rename to git-svn-ext index 8d6ddb8..8ff91cc 100755 --- a/git-svn-check-unpushed +++ b/git-svn-ext @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python from commands import getoutput import sys @@ -79,8 +79,8 @@ def find_uncommitted(svn_branch): else: print 'No unpushed commits found.' - -if __name__ == '__main__': +def check_unpushed_handler(): + """Check if any local externals have any unpushed changes""" status = getoutput('git status') if status.startswith('fatal'): print status @@ -91,3 +91,32 @@ if __name__ == '__main__': sys.exit(1) logger.debug('Found branch: %s', svn_branch) find_uncommitted(svn_branch) + +def usage(): + """Descriptive text of all options""" + print "" + print "Usage: " + sys.argv[0] + " [check-unpushed]" + print "" + print " check-unpushed: " + check_unpushed_handler.__doc__ + print "" + +handler_map = {} +handler_map['check-unpushed'] = check_unpushed_handler + +if __name__ == '__main__': + + handler = "" + + try: + sub_command = sys.argv[1] + + if handler_map.has_key(sub_command): + handler = handler_map[sub_command] + else: + raise "Invalid sub command" + + except: + usage() + sys.exit(1) + + handler() From ae1c64df57d0e5a1f6827c7f72100e432fd91ac6 Mon Sep 17 00:00:00 2001 From: Wade Berrier Date: Thu, 24 May 2012 21:20:53 -0600 Subject: [PATCH 02/18] incorporate git-svn-externals-check as "git-svn-ext check" --- git-svn-externals-check | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100755 git-svn-externals-check diff --git a/git-svn-externals-check b/git-svn-externals-check deleted file mode 100755 index fddc313..0000000 --- a/git-svn-externals-check +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -for dir in *; do - if [ -d $dir ]; then - cd $dir - STATUS=$(git status) - UNPUSHED=$(git-svn-check-unpushed) - if [ $(echo $STATUS|grep -c "clean") -lt 1 -o \ - $(echo $UNPUSHED|grep -c "No unpushed") -lt 1 ]; then - echo '>>>>>>>>>>>>>>>>' $dir '<<<<<<<<<<<<<<<<' - git status - git-svn-check-unpushed - echo '----------------------------------------' - else - echo $dir 'is clean' - fi - cd .. - fi -done From 4809fd9eb85a8333ffa872fa671e5d57978226f2 Mon Sep 17 00:00:00 2001 From: Wade Berrier Date: Thu, 24 May 2012 21:43:39 -0600 Subject: [PATCH 03/18] collapse git-svn-externals-update into: git-svn-ext update --- git-svn-ext | 67 ++++++++++++++++++++++++++++++++++++---- git-svn-externals-update | 15 --------- 2 files changed, 61 insertions(+), 21 deletions(-) delete mode 100755 git-svn-externals-update diff --git a/git-svn-ext b/git-svn-ext index 8ff91cc..f80ac09 100755 --- a/git-svn-ext +++ b/git-svn-ext @@ -1,6 +1,8 @@ #!/usr/bin/env python from commands import getoutput +import os +import re import sys import logging @@ -11,6 +13,10 @@ console.setFormatter(formatter) logger.addHandler(console) logger.level = logging.INFO +# This is so we can change directories but still have a reference +# to ourselves +git_svn_ext_fullpath = os.path.abspath(sys.argv[0]) + def list_references(): """List the references in the local repo. @@ -79,8 +85,33 @@ def find_uncommitted(svn_branch): else: print 'No unpushed commits found.' + +def update_handler(): + """Update all svn externals""" + + toplevel_directory = getoutput("git rev-parse --show-cdup") + + if toplevel_directory != "": + print "please run from the toplevel directory" + sys.exit(1) + + git_ext_dir = ".git_externals" + if os.path.isdir(git_ext_dir): + for external in os.listdir(git_ext_dir): + gitdir = os.path.join(git_ext_dir, external, ".git") + if os.path.isdir(gitdir): + dir = os.path.dirname(gitdir) + if os.path.isdir(dir): + current_dir = os.getcwd() + os.chdir(dir) + print dir + os.system("git svn fetch") + os.system("git svn rebase") + os.chdir(current_dir) + + def check_unpushed_handler(): - """Check if any local externals have any unpushed changes""" + """Check if a local external has any unpushed changes""" status = getoutput('git status') if status.startswith('fatal'): print status @@ -92,17 +123,41 @@ def check_unpushed_handler(): logger.debug('Found branch: %s', svn_branch) find_uncommitted(svn_branch) + +def check_handler(): + """Run check-unpushed for all externals""" + for dir in os.listdir("."): + if os.path.isdir(dir) and dir[0] != ".": + os.chdir(dir) + status = getoutput("git status") + unpushed = getoutput(git_svn_ext_fullpath + " check-unpushed") + if not re.search("clean", status) or not re.search("No unpushed", unpushed): + print ">>>>>>>>>>>>>>>> %s <<<<<<<<<<<<<<<<" % dir + os.system("git status") + os.system(git_svn_ext_fullpath + " check-unpushed") + print "----------------------------------------" + else: + print dir + " is clean" + + os.chdir("..") + + +# TODO: use a structure that preserves order (for usage) +handler_map = {} +handler_map['update'] = update_handler +handler_map['check-unpushed'] = check_unpushed_handler +handler_map['check'] = check_handler + def usage(): """Descriptive text of all options""" print "" - print "Usage: " + sys.argv[0] + " [check-unpushed]" + print "Usage: " + sys.argv[0] + " [" + "|".join(handler_map.keys()) + "]" print "" - print " check-unpushed: " + check_unpushed_handler.__doc__ + # TODO: would be nice if the docs were aligned + for k,v in handler_map.items(): + print " %s: %s" % (k, v.__doc__) print "" -handler_map = {} -handler_map['check-unpushed'] = check_unpushed_handler - if __name__ == '__main__': handler = "" diff --git a/git-svn-externals-update b/git-svn-externals-update deleted file mode 100755 index 4313b29..0000000 --- a/git-svn-externals-update +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -toplevel_directory="$(git rev-parse --show-cdup)" -[ -n "$toplevel_directory" ] && { echo "please run from the toplevel directory"; exit 1; } - -find .git_externals -type d -name .git | while read gitdir; do - dir=$(dirname "$gitdir") - if [ -d $dir ]; then - pushd $dir - echo $dir - git svn fetch - git svn rebase - popd - fi -done From 1a22161c46fb7f12d8a7113b3336b4a3e277df6d Mon Sep 17 00:00:00 2001 From: Wade Berrier Date: Tue, 12 Jun 2012 23:36:24 -0600 Subject: [PATCH 04/18] fold git-svn-clone-externals into git-svn-ext also add support for relative externals. Works, but the normal case probably still needs some testing --- git-svn-clone-externals | 141 --------------------- git-svn-ext | 267 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 263 insertions(+), 145 deletions(-) delete mode 100755 git-svn-clone-externals diff --git a/git-svn-clone-externals b/git-svn-clone-externals deleted file mode 100755 index 091cab3..0000000 --- a/git-svn-clone-externals +++ /dev/null @@ -1,141 +0,0 @@ -#!/bin/bash - -set -e - -toplevel_directory="$(git rev-parse --show-cdup)" -[ -n "$toplevel_directory" ] && { echo "please run from the toplevel directory"; exit 1; } - - -function call() -{ - cmd="$@" - echo "$cmd" - eval "$cmd" - return "$?" -} - -function do_clone() -{ - test -d .git_externals || return 1 - module=`echo $remote_url|sed 's,\(.*\)\(/trunk\|/branch.*\|/tag.*\),\1,'` - branch=`echo $remote_url|sed 's,\(.*\)\(/trunk\|/branch.*\|/tag.*\),\2,'|sed 's,^/,,'` - if [[ $branch = $remote_url ]]; then - branch="" - fi - ( - cd .git_externals - if [ -d "$local_directory" ]; then - ( - cd "$local_directory" - call git svn fetch --all - ) - else - tags="tags" - brch="branches" - branchpath=$(echo $branch|cut -f1 -d/) - echo $tags|grep $branchpath >/dev/null 2>&1 && tags=$branchpath - echo $brch|grep $branchpath >/dev/null 2>&1 && brch=$branchpath - - if [ "$module" = "$remote_url" ]; then - # URL does not contains any trunk, branches or tags part, so we dont need - # additional options for git-svn - call git svn clone "$revision" "$module" "$local_directory" - else - call git svn clone "$revision" "$module" -T trunk -b $brch -t $tags "$local_directory" - fi - - fi - ( - branch="$(echo ${branch}|sed 's,/$,,')" - if [ -n "$branch" ]; then - cd "$local_directory" - call git reset --hard $branch - fi - ) - ) -} - -function do_link() -{ - dir="$1" - base="$(dirname $dir)" - ( - mkdir -p "$base" - cd $base - rel=$(git rev-parse --show-cdup) - ln -sf ${rel}.git_externals/"$dir" - ) -} - -function do_excludes() -{ - dir="$1" - git_excludes_path=.git/info/exclude - if ! grep -q '^.git_externals$' "$git_excludes_path" - then - echo .git_externals >> "$git_excludes_path" - fi - - if ! grep -q '^'"$dir"'$' "$git_excludes_path" - then - echo "$dir" >> "$git_excludes_path" - fi -} - -function is_excluded() -{ - local result=0 - if [ -f .git_externals_exclude ] ; then - matches=`grep -v "^#" .git_externals_exclude|grep "^/$1$"|wc -l` - if [ $matches -gt 0 ] ; then - local result=1 - fi - fi - echo $result - return -} - - -git svn show-externals | grep -vE '#|^$' | \ - sed 's/\(-r\)[ ]*\([0-9]\{1,\}\)/\1\2/' | \ - while read svn_externals -do - - number_fields="$(echo ${svn_externals}|awk '{print NF}')" - case $number_fields in - 2) - local_directory="$(echo ${svn_externals} | awk '{print $1}' | sed 's,^/,,')" - revision="" - remote_url="$(echo ${svn_externals} | awk '{print $2}')" - ;; - 3) - local_directory="$(echo ${svn_externals} | awk '{print $1}' | sed 's,^/,,')" - revision=""$(echo ${svn_externals} | awk '{print $2}') - remote_url="$(echo ${svn_externals} | awk '{print $3}')" - ;; - *) continue ;; - esac - - check_excluded=$(is_excluded $local_directory) - if [ $check_excluded -eq 0 ] ; then - if [ -n "$USE_SSH" ]; then - echo "Rewriting url to use SVN+SSH." - shopt -s extglob - remote_url="${remote_url/+(http|https)/svn+ssh}" - fi - - [ -z "${remote_url}" ] && continue - - export local_directory revision remote_url - - echo "$local_directory -> $remote_url" - - dir=`dirname $local_directory` - [ -d ".git_externals/$dir" ] || mkdir -p ".git_externals/$dir" - - do_clone "$revision" "$remote_url" "$local_directory" || exit - do_link "$local_directory" - do_excludes "$local_directory" - fi - -done \ No newline at end of file diff --git a/git-svn-ext b/git-svn-ext index f80ac09..0c3cd33 100755 --- a/git-svn-ext +++ b/git-svn-ext @@ -16,8 +16,11 @@ logger.level = logging.INFO # This is so we can change directories but still have a reference # to ourselves git_svn_ext_fullpath = os.path.abspath(sys.argv[0]) +excludes_file = ".git_external_excludes" +##################### Helper Methods ##################### + def list_references(): """List the references in the local repo. @@ -85,16 +88,270 @@ def find_uncommitted(svn_branch): else: print 'No unpushed commits found.' - -def update_handler(): - """Update all svn externals""" - +def exit_if_not_toplevel(): + """exit if not run from the top level git directory""" toplevel_directory = getoutput("git rev-parse --show-cdup") if toplevel_directory != "": print "please run from the toplevel directory" sys.exit(1) +def is_excluded(external): + """return True if excluding this external""" + + if os.path.exists(excludes_file): + for line in open(excludes_file).readlines(): + if re.search("^/%s$" % external, line): + return True + + return False + + +def create_dir_if_not_exist(dir): + """helper to create a directory if it doesn't exist""" + if not os.path.exists(dir): os.mkdir(dir) + + +def do_link(external_dir, link_dir): + """create the link to the external""" + + current_dir = os.getcwd() + + os.chdir(external_dir) + + # get relative back to git-svn root + rel = getoutput("git rev-parse --show-cdup") + + link_name = os.path.basename(link_dir) + + # NOTE: the original used -f, so remove and recreate + if os.path.exists(link_name): os.unlink(link_name) + + os.symlink(rel + ".git_externals/" + link_name, link_name) + + os.chdir(current_dir) + +def append_line_if_not_included(filename, line): + + found = False + for l in open(filename).readlines(): + if line == l.rstrip(): + found = True + break + + if not found: + fd = open(filename, "a") + # TODO: do we need to have a newline to the end of the file + # before appending? + fd.write(line + "\n") + fd.close() + +def do_excludes(external_link): + """add symlink to the git excludes path""" + + git_excludes_path=".git/info/exclude" + append_line_if_not_included(git_excludes_path, ".git_externals") + append_line_if_not_included(git_excludes_path, external_link) + +def run_command(command): + """print out and run a command""" + print command + return os.system(command) + +def do_clone(local_directory, remote_url, revision): + """do the actual cloning""" + + # make sure the dir exists + if not os.path.exists(".git_externals"): + return False + + (module, branch) = re.search("(.*)/(trunk|branch.*|tag.*)", remote_url).groups() + + if branch == remote_url: + branch = "" + + current_dir1 = os.getcwd() + + os.chdir(".git_externals") + + # Default + has_branches_tags = False + + # if we found references to branches and tags, clone them + if module != remote_url: + has_branches_tags = True + + # if we are cloning a subdirectory of a branch path, use the remote_url, and don't get tags/branch/trunk + # ie: if it contains some of these, but does not end with it + if re.search("trunk|tags|branch.*", remote_url): + if not re.search("trunk$|tags$|branch.*$", remote_url): + has_branches_tags = False + module = remote_url + + # Try to figure out what the tags and branches portions are + tags = "tags" + brch = "branches" + branchpath = branch.split("/")[0] + if tags.count(branchpath): tags = branchpath + if brch.count(branchpath): brch = branchpath + + # If the directory is alread there, update it + if os.path.isdir(local_directory): + current_dir2 = os.getcwd() + os.chdir(local_directory) + run_command("git svn fetch --all") + os.chdir(current_dir2) + # Otherwise, clone it + else: + + # Format the revision argument + rev_arg = "" + if revision != "": + rev_arg = "-r " + revision + + if has_branches_tags: + run_command("git svn clone %s %s -T trunk -b %s -t %s %s" % (rev_arg, module, brch, tags, local_directory)) + else: + # URL does not contains any trunk, branches or tags part, so we dont need + # additional options for git-svn + run_command("git svn clone %s %s %s" % (rev_arg, module, local_directory)) + + # If we have branches and tags, checkout that directory + # careful, can blow aways changes! + # remove trailing slash + branch = re.sub("/$", "", branch) + + if branch != "" and has_branches_tags: + current_dir3 = os.getcwd() + os.chdir(local_directory) + run_command("git reset --hard " + branch) + os.chdir(current_dir3) + + os.chdir(current_dir1) + + return True + +def get_svn_root_url(): + """Parse out the svn root to use with relative urls""" + + root = "" + for line in getoutput("git svn info").split("\n"): + matches = re.search("Repository Root: (.*)", line) + + if matches: + root = matches.group(1) + + + if root == "": + print "Unable to determine repository root" + sys.exit(1) + + return root + +class SvnExternal: + """Class to hold and manipulate data about an svn external""" + + def __init__(self, external_dir, local_dir, remote_url, revision): + self.external_dir = external_dir + self.local_dir = local_dir + self.remote_url = remote_url + self.revision = revision + + @staticmethod + def GetExternals(): + ret = [] + + current_dir = "" + for line in getoutput("git svn show-externals").split("\n"): + + current_dir_matches = re.search(r"# /(.*)", line) + # format: current_dir/dir[@rev] [-r 123] dir + # the ?P syntax is to get the stuff by name + re_current_dir = current_dir.replace("/", "\\/") # regular expression safe string + external_matches = re.search(r"/%s(?P.+)(@(?P\d+))? (-r(?P\d+) )?(?P.+)" % re_current_dir , line) + + if current_dir_matches: + # save off the current dir + current_dir = current_dir_matches.group(1) + #print "Setting current_dir to: " + current_dir + continue + + elif external_matches: + + external_info = external_matches.groupdict() + + # Check for relative url + remote_url = external_info["remote_url"] + # TODO: there are probably other external url types that we don't handle + if remote_url.count("^"): + svn_root_url = get_svn_root_url() + svn_base = os.path.dirname(svn_root_url) + svn_path = os.path.basename(svn_root_url) + remote_url = remote_url.replace("^", svn_path) + + # Normalize the path + remote_url = os.path.normpath(remote_url) + + # Stick them back together + # had to split apart because normpath turns "//" into "/" + # maybe a better url parsing method would be better? + remote_url = svn_base + os.sep + remote_url + + local_dir = external_info["local_dir"] + + # Determine the revision + revision = "" + if external_info["rev1"] != None: + revision = external_info["rev1"] + elif external_info["rev2"] != None: + revision = external_info["rev2"] + + # Allow exclusion of externals + if not is_excluded(local_dir): + + # Honor env var to override url + if os.environ.has_key("USE_SSH"): + remote_url = re.sub("(http|https)", "svn+ssh", remote_url) + + # Add the info + if remote_url != "": + ret.append(SvnExternal(current_dir, local_dir, remote_url, revision)) + + return ret + + +########### sub-command handlers ############# + + +def clone_handler(): + """git-svn clone of all svn externals into .git_externals (can blow aways changes when rerunning!)""" + + exit_if_not_toplevel() + + externals = SvnExternal.GetExternals() + + for external in externals: + + print "%s -> %s" % (external.local_dir, external.remote_url) + + dir = os.path.dirname(external.local_dir) + + # just create two dirs instead of pulling in mkpath + create_dir_if_not_exist(".git_externals") + create_dir_if_not_exist(os.path.join(".git_externals", dir)) + + if not do_clone(external.local_dir, external.remote_url, external.revision): + sys.exit(1) + + do_link(external.external_dir, external.local_dir) + do_excludes(external.external_dir + external.local_dir) + + +def update_handler(): + """Update all svn externals""" + + exit_if_not_toplevel() + git_ext_dir = ".git_externals" if os.path.isdir(git_ext_dir): for external in os.listdir(git_ext_dir): @@ -142,8 +399,10 @@ def check_handler(): os.chdir("..") + # TODO: use a structure that preserves order (for usage) handler_map = {} +handler_map['clone'] = clone_handler handler_map['update'] = update_handler handler_map['check-unpushed'] = check_unpushed_handler handler_map['check'] = check_handler From fe383fe722f68dbeb4c99f9b1fa119bc4d27d172 Mon Sep 17 00:00:00 2001 From: Wade Berrier Date: Fri, 25 May 2012 15:50:35 -0600 Subject: [PATCH 05/18] force "check" to be run from the root of a repo and only run against externals and not all directories --- git-svn-ext | 76 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 49 insertions(+), 27 deletions(-) diff --git a/git-svn-ext b/git-svn-ext index 0c3cd33..4340feb 100755 --- a/git-svn-ext +++ b/git-svn-ext @@ -18,7 +18,6 @@ logger.level = logging.INFO git_svn_ext_fullpath = os.path.abspath(sys.argv[0]) excludes_file = ".git_external_excludes" - ##################### Helper Methods ##################### def list_references(): @@ -31,10 +30,12 @@ def list_references(): HEAD_ref = None refs = {} for item in references.split('\n'): - sha1, name = item.split() - if name == 'HEAD': - HEAD_ref = sha1 - refs[sha1] = name + stuff = item.split() + if len(stuff) == 2: + (sha1, name) = stuff + if name == 'HEAD': + HEAD_ref = sha1 + refs[sha1] = name return HEAD_ref, refs @@ -80,13 +81,15 @@ def find_svn_branch_name(): def find_uncommitted(svn_branch): """Given the name of the remote branch, show log of the commits.""" + + # One line commit of uncommitted commits + results = [] output = getoutput('git log --pretty="format:%%h %%s" %s..HEAD' % svn_branch) if output: - print 'Possible unpushed commits (against %s):' % svn_branch - print output - else: - print 'No unpushed commits found.' + results += output.split('\n') + + return results def exit_if_not_toplevel(): """exit if not run from the top level git directory""" @@ -367,37 +370,56 @@ def update_handler(): os.chdir(current_dir) +# TODO: would it be nice to have a recursive option? +# otherwise, you have to run it from the root dir def check_unpushed_handler(): - """Check if a local external has any unpushed changes""" + """Check if a local external has any unpushed commits""" + status = getoutput('git status') if status.startswith('fatal'): print status sys.exit(1) svn_branch = find_svn_branch_name() if svn_branch is None: - print "No svn branch found" + print "Warning: no svn branch found for: " + os.getcwd() sys.exit(1) logger.debug('Found branch: %s', svn_branch) - find_uncommitted(svn_branch) + commits = find_uncommitted(svn_branch) + + if len(commits) > 0: + print 'Possible unpushed commits (against %s):' % svn_branch + print "\n".join(commits) + else: + print 'No unpushed commits found.' def check_handler(): - """Run check-unpushed for all externals""" - for dir in os.listdir("."): - if os.path.isdir(dir) and dir[0] != ".": - os.chdir(dir) - status = getoutput("git status") - unpushed = getoutput(git_svn_ext_fullpath + " check-unpushed") - if not re.search("clean", status) or not re.search("No unpushed", unpushed): - print ">>>>>>>>>>>>>>>> %s <<<<<<<<<<<<<<<<" % dir - os.system("git status") - os.system(git_svn_ext_fullpath + " check-unpushed") - print "----------------------------------------" - else: - print dir + " is clean" - - os.chdir("..") + """Check for local externals that have unpushed commits""" + + exit_if_not_toplevel() + git_ext_dir = ".git_externals" + if os.path.isdir(git_ext_dir): + for external in os.listdir(git_ext_dir): + gitdir = os.path.join(git_ext_dir, external, ".git") + if os.path.isdir(gitdir): + dir = os.path.dirname(gitdir) + if os.path.isdir(dir): + current_dir = os.getcwd() + os.chdir(dir) + + status = getoutput("git status") + unpushed = getoutput(git_svn_ext_fullpath + " check-unpushed") + if not re.search("clean", status) or not re.search("No unpushed", unpushed): + print ">>>>>>>>>>>>>>>> %s <<<<<<<<<<<<<<<<" % dir + # Run these again to get color output + os.system("git status") + os.system(git_svn_ext_fullpath + " check-unpushed") + print "----------------------------------------" + else: + print dir + " is clean" + + os.chdir(current_dir) # TODO: use a structure that preserves order (for usage) From 9476322e036719731734b454692639b7af49c232 Mon Sep 17 00:00:00 2001 From: Wade Berrier Date: Sat, 26 May 2012 09:55:09 -0600 Subject: [PATCH 06/18] consolidate some code to iterate through the externals --- git-svn-ext | 73 ++++++++++++++++++++++++++++------------------------- 1 file changed, 38 insertions(+), 35 deletions(-) diff --git a/git-svn-ext b/git-svn-ext index 4340feb..2ee2986 100755 --- a/git-svn-ext +++ b/git-svn-ext @@ -322,6 +322,20 @@ class SvnExternal: return ret +# maybe have this go to the repo root automatically? +def get_externals_paths(): + """Get a list of paths to externals for the current dir""" + + results = [] + + git_ext_dir = ".git_externals" + if os.path.isdir(git_ext_dir): + for external in os.listdir(git_ext_dir): + gitdir = os.path.join(git_ext_dir, external, ".git") + if os.path.isdir(gitdir): + results.append(os.path.dirname(gitdir)) + + return results ########### sub-command handlers ############# @@ -355,19 +369,13 @@ def update_handler(): exit_if_not_toplevel() - git_ext_dir = ".git_externals" - if os.path.isdir(git_ext_dir): - for external in os.listdir(git_ext_dir): - gitdir = os.path.join(git_ext_dir, external, ".git") - if os.path.isdir(gitdir): - dir = os.path.dirname(gitdir) - if os.path.isdir(dir): - current_dir = os.getcwd() - os.chdir(dir) - print dir - os.system("git svn fetch") - os.system("git svn rebase") - os.chdir(current_dir) + for dir in get_externals_paths(): + current_dir = os.getcwd() + os.chdir(dir) + print dir + os.system("git svn fetch") + os.system("git svn rebase") + os.chdir(current_dir) # TODO: would it be nice to have a recursive option? @@ -398,28 +406,23 @@ def check_handler(): exit_if_not_toplevel() - git_ext_dir = ".git_externals" - if os.path.isdir(git_ext_dir): - for external in os.listdir(git_ext_dir): - gitdir = os.path.join(git_ext_dir, external, ".git") - if os.path.isdir(gitdir): - dir = os.path.dirname(gitdir) - if os.path.isdir(dir): - current_dir = os.getcwd() - os.chdir(dir) - - status = getoutput("git status") - unpushed = getoutput(git_svn_ext_fullpath + " check-unpushed") - if not re.search("clean", status) or not re.search("No unpushed", unpushed): - print ">>>>>>>>>>>>>>>> %s <<<<<<<<<<<<<<<<" % dir - # Run these again to get color output - os.system("git status") - os.system(git_svn_ext_fullpath + " check-unpushed") - print "----------------------------------------" - else: - print dir + " is clean" - - os.chdir(current_dir) + for dir in get_externals_paths(): + + current_dir = os.getcwd() + os.chdir(dir) + + status = getoutput("git status") + unpushed = getoutput(git_svn_ext_fullpath + " check-unpushed") + if not re.search("clean", status) or not re.search("No unpushed", unpushed): + print ">>>>>>>>>>>>>>>> %s <<<<<<<<<<<<<<<<" % dir + # Run these again to get color output + os.system("git status") + os.system(git_svn_ext_fullpath + " check-unpushed") + print "----------------------------------------" + else: + print dir + " is clean" + + os.chdir(current_dir) # TODO: use a structure that preserves order (for usage) From 34d932a4330cc0e49fcb8b9f1c369b38489c45d5 Mon Sep 17 00:00:00 2001 From: Wade Berrier Date: Sat, 26 May 2012 10:00:54 -0600 Subject: [PATCH 07/18] abstract out .git_externals string --- git-svn-ext | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/git-svn-ext b/git-svn-ext index 2ee2986..76bdb8d 100755 --- a/git-svn-ext +++ b/git-svn-ext @@ -16,6 +16,7 @@ logger.level = logging.INFO # This is so we can change directories but still have a reference # to ourselves git_svn_ext_fullpath = os.path.abspath(sys.argv[0]) +externals_dir = ".git_externals" excludes_file = ".git_external_excludes" ##################### Helper Methods ##################### @@ -130,7 +131,7 @@ def do_link(external_dir, link_dir): # NOTE: the original used -f, so remove and recreate if os.path.exists(link_name): os.unlink(link_name) - os.symlink(rel + ".git_externals/" + link_name, link_name) + os.symlink(rel + externals_dir + os.sep + link_name, link_name) os.chdir(current_dir) @@ -153,7 +154,7 @@ def do_excludes(external_link): """add symlink to the git excludes path""" git_excludes_path=".git/info/exclude" - append_line_if_not_included(git_excludes_path, ".git_externals") + append_line_if_not_included(git_excludes_path, externals_dir) append_line_if_not_included(git_excludes_path, external_link) def run_command(command): @@ -165,7 +166,7 @@ def do_clone(local_directory, remote_url, revision): """do the actual cloning""" # make sure the dir exists - if not os.path.exists(".git_externals"): + if not os.path.exists(externals_dir): return False (module, branch) = re.search("(.*)/(trunk|branch.*|tag.*)", remote_url).groups() @@ -175,7 +176,7 @@ def do_clone(local_directory, remote_url, revision): current_dir1 = os.getcwd() - os.chdir(".git_externals") + os.chdir(externals_dir) # Default has_branches_tags = False @@ -328,10 +329,9 @@ def get_externals_paths(): results = [] - git_ext_dir = ".git_externals" - if os.path.isdir(git_ext_dir): - for external in os.listdir(git_ext_dir): - gitdir = os.path.join(git_ext_dir, external, ".git") + if os.path.isdir(externals_dir): + for external in os.listdir(externals_dir): + gitdir = os.path.join(externals_dir, external, ".git") if os.path.isdir(gitdir): results.append(os.path.dirname(gitdir)) @@ -354,8 +354,8 @@ def clone_handler(): dir = os.path.dirname(external.local_dir) # just create two dirs instead of pulling in mkpath - create_dir_if_not_exist(".git_externals") - create_dir_if_not_exist(os.path.join(".git_externals", dir)) + create_dir_if_not_exist(externals_dir) + create_dir_if_not_exist(os.path.join(externals_dir, dir)) if not do_clone(external.local_dir, external.remote_url, external.revision): sys.exit(1) From 60b1ab49d49ed2dc022a4aa1999279bcc17f08e4 Mon Sep 17 00:00:00 2001 From: Wade Berrier Date: Sat, 26 May 2012 10:25:37 -0600 Subject: [PATCH 08/18] add for_all handler: runs a command against all externals reimplement "update" using this handler --- git-svn-ext | 40 ++++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/git-svn-ext b/git-svn-ext index 76bdb8d..9658a6b 100755 --- a/git-svn-ext +++ b/git-svn-ext @@ -340,7 +340,7 @@ def get_externals_paths(): ########### sub-command handlers ############# -def clone_handler(): +def clone_handler(args): """git-svn clone of all svn externals into .git_externals (can blow aways changes when rerunning!)""" exit_if_not_toplevel() @@ -364,23 +364,18 @@ def clone_handler(): do_excludes(external.external_dir + external.local_dir) -def update_handler(): +def update_handler(args): """Update all svn externals""" exit_if_not_toplevel() - for dir in get_externals_paths(): - current_dir = os.getcwd() - os.chdir(dir) - print dir - os.system("git svn fetch") - os.system("git svn rebase") - os.chdir(current_dir) + # Must pass in as keywords to fit the interface + for_all_handler("git svn rebase".split()) # TODO: would it be nice to have a recursive option? # otherwise, you have to run it from the root dir -def check_unpushed_handler(): +def check_unpushed_handler(args): """Check if a local external has any unpushed commits""" status = getoutput('git status') @@ -401,7 +396,7 @@ def check_unpushed_handler(): else: print 'No unpushed commits found.' -def check_handler(): +def check_handler(args): """Check for local externals that have unpushed commits""" exit_if_not_toplevel() @@ -424,6 +419,21 @@ def check_handler(): os.chdir(current_dir) +def for_all_handler(args): + """run a command against all externals""" + + command = " ".join(args) + + for dir in get_externals_paths(): + + current_dir = os.getcwd() + os.chdir(dir) + + print ">>> %s: %s" % (dir, command) + + os.system(command) + + os.chdir(current_dir) # TODO: use a structure that preserves order (for usage) handler_map = {} @@ -431,6 +441,7 @@ handler_map['clone'] = clone_handler handler_map['update'] = update_handler handler_map['check-unpushed'] = check_unpushed_handler handler_map['check'] = check_handler +handler_map['for-all'] = for_all_handler def usage(): """Descriptive text of all options""" @@ -458,4 +469,9 @@ if __name__ == '__main__': usage() sys.exit(1) - handler() + args = [] + if len(sys.argv) > 2: + args += sys.argv[2:] + + handler(args) + From 013d56b4ed62b64e40cb161b04e242713421a9de Mon Sep 17 00:00:00 2001 From: Wade Berrier Date: Sat, 26 May 2012 11:14:57 -0600 Subject: [PATCH 09/18] for-all: require a command --- git-svn-ext | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/git-svn-ext b/git-svn-ext index 9658a6b..7291d38 100755 --- a/git-svn-ext +++ b/git-svn-ext @@ -424,16 +424,22 @@ def for_all_handler(args): command = " ".join(args) - for dir in get_externals_paths(): + if command: + for dir in get_externals_paths(): - current_dir = os.getcwd() - os.chdir(dir) + current_dir = os.getcwd() + os.chdir(dir) - print ">>> %s: %s" % (dir, command) + print ">>> %s: %s" % (dir, command) - os.system(command) + os.system(command) + + os.chdir(current_dir) + + else: + usage() + sys.exit(1) - os.chdir(current_dir) # TODO: use a structure that preserves order (for usage) handler_map = {} From 99c3a474dbe774d7f63a55e41b30d8aa359f05e7 Mon Sep 17 00:00:00 2001 From: Wade Berrier Date: Sat, 26 May 2012 11:15:19 -0600 Subject: [PATCH 10/18] fix up help system by using a handler class to keep handler ordering and by aligning handler names in usage() --- git-svn-ext | 83 +++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 61 insertions(+), 22 deletions(-) diff --git a/git-svn-ext b/git-svn-ext index 7291d38..56334b2 100755 --- a/git-svn-ext +++ b/git-svn-ext @@ -97,7 +97,7 @@ def exit_if_not_toplevel(): toplevel_directory = getoutput("git rev-parse --show-cdup") if toplevel_directory != "": - print "please run from the toplevel directory" + print "Error: not in toplevel directory" sys.exit(1) def is_excluded(external): @@ -341,7 +341,8 @@ def get_externals_paths(): def clone_handler(args): - """git-svn clone of all svn externals into .git_externals (can blow aways changes when rerunning!)""" + """clone all svn externals into .git_externals + (warning: removes local changes and commits on subsequent runs)""" exit_if_not_toplevel() @@ -365,7 +366,7 @@ def clone_handler(args): def update_handler(args): - """Update all svn externals""" + """Updates all svn externals (git svn rebase)""" exit_if_not_toplevel() @@ -376,7 +377,7 @@ def update_handler(args): # TODO: would it be nice to have a recursive option? # otherwise, you have to run it from the root dir def check_unpushed_handler(args): - """Check if a local external has any unpushed commits""" + """Check if local git-svn checkout has unpushed commits""" status = getoutput('git status') if status.startswith('fatal'): @@ -397,7 +398,7 @@ def check_unpushed_handler(args): print 'No unpushed commits found.' def check_handler(args): - """Check for local externals that have unpushed commits""" + """run 'git status' and 'check-unpushed' for all externals""" exit_if_not_toplevel() @@ -420,7 +421,8 @@ def check_handler(args): os.chdir(current_dir) def for_all_handler(args): - """run a command against all externals""" + """run a command against all externals + (ie: git svn-ext for-all git grep 'whatever')""" command = " ".join(args) @@ -441,35 +443,72 @@ def for_all_handler(args): sys.exit(1) -# TODO: use a structure that preserves order (for usage) -handler_map = {} -handler_map['clone'] = clone_handler -handler_map['update'] = update_handler -handler_map['check-unpushed'] = check_unpushed_handler -handler_map['check'] = check_handler -handler_map['for-all'] = for_all_handler +class Handlers: + """Register and dispatch handlers, as well as provide documentation""" + def __init__(self): + # keep ordering + self.names = [] + self.handlers = {} + + def add(self, name, handler): + self.names.append(name) + self.handlers[name] = handler + + def get(self, name): + if self.handlers.has_key(name): + return self.handlers[name] + else: + return None + + def _aligned_name(self, name): + """string of spaces to pad command to align text""" + + # find out how long the lines are so they can line up + max_len = 0 + for n in self.names: + if len(n) > max_len: + max_len = len(n) + + return name + " " * (1 + max_len - len(name)) + + def helptext(self): + """get description of all the handlers""" + + ret = "" + + for n in self.names: + ret += " %s: %s\n" % (self._aligned_name(n), self.handlers[n].__doc__) + + return ret + + +# NOTE: this order is preserved for usage help text +handlers = Handlers() +handlers.add('clone', clone_handler) +handlers.add('update', update_handler) +handlers.add('check-unpushed', check_unpushed_handler) +handlers.add('check', check_handler) +handlers.add('for-all', for_all_handler) def usage(): """Descriptive text of all options""" print "" - print "Usage: " + sys.argv[0] + " [" + "|".join(handler_map.keys()) + "]" + print "Usage: " + sys.argv[0] + " [sub command args]" print "" - # TODO: would be nice if the docs were aligned - for k,v in handler_map.items(): - print " %s: %s" % (k, v.__doc__) + print " sub commmands:" + print handlers.helptext() print "" if __name__ == '__main__': - handler = "" + handler = None try: sub_command = sys.argv[1] + handler = handlers.get(sub_command) - if handler_map.has_key(sub_command): - handler = handler_map[sub_command] - else: - raise "Invalid sub command" + if handler == None: + raise "invalid subcommand" except: usage() From 60e40c2219475a531bdb5cba873873e838e4460a Mon Sep 17 00:00:00 2001 From: Wade Berrier Date: Sat, 26 May 2012 14:54:44 -0600 Subject: [PATCH 11/18] fix when externals are pinned to a revision for clone and update --- git-svn-ext | 60 ++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 48 insertions(+), 12 deletions(-) diff --git a/git-svn-ext b/git-svn-ext index 56334b2..86b9b12 100755 --- a/git-svn-ext +++ b/git-svn-ext @@ -199,20 +199,22 @@ def do_clone(local_directory, remote_url, revision): if tags.count(branchpath): tags = branchpath if brch.count(branchpath): brch = branchpath + # Format the revision argument + rev_arg = "" + if revision != "": + rev_arg = "--revision BASE:" + revision + # If the directory is alread there, update it if os.path.isdir(local_directory): current_dir2 = os.getcwd() os.chdir(local_directory) - run_command("git svn fetch --all") + run_command("git svn fetch " + rev_arg) + # -l skips fetching + run_command("git svn rebase --local") os.chdir(current_dir2) # Otherwise, clone it else: - # Format the revision argument - rev_arg = "" - if revision != "": - rev_arg = "-r " + revision - if has_branches_tags: run_command("git svn clone %s %s -T trunk -b %s -t %s %s" % (rev_arg, module, brch, tags, local_directory)) else: @@ -272,7 +274,8 @@ class SvnExternal: # format: current_dir/dir[@rev] [-r 123] dir # the ?P syntax is to get the stuff by name re_current_dir = current_dir.replace("/", "\\/") # regular expression safe string - external_matches = re.search(r"/%s(?P.+)(@(?P\d+))? (-r(?P\d+) )?(?P.+)" % re_current_dir , line) + # NOTE: the remote_url section is non-greedy so that @rev is matched correctly + external_matches = re.search(r"/%s(?P.+?)(@(?P\d+))? (-r\s*(?P\d+) )?(?P.+)" % re_current_dir , line) if current_dir_matches: # save off the current dir @@ -303,7 +306,7 @@ class SvnExternal: local_dir = external_info["local_dir"] - # Determine the revision + # Determine the revision (hopefully only 1 is specified) revision = "" if external_info["rev1"] != None: revision = external_info["rev1"] @@ -323,6 +326,22 @@ class SvnExternal: return ret + def local_storage(self): + """Get the directory that this external will be cloned to""" + + # TODO/NOTE: it's possible to have 2 externals of the same name + # in different directories + return externals_dir + os.sep + self.local_dir + + def revision_arguments(self): + """Format argumntes if there's a revision""" + + # Format the revision argument + rev_arg = "" + if self.revision != "": + rev_arg = "--revision BASE:" + self.revision + return rev_arg + # maybe have this go to the repo root automatically? def get_externals_paths(): """Get a list of paths to externals for the current dir""" @@ -364,14 +383,31 @@ def clone_handler(args): do_link(external.external_dir, external.local_dir) do_excludes(external.external_dir + external.local_dir) - +# TODO: maybe clone should just do this instead? +# but clone can blow stuff away... (reset --hard) def update_handler(args): - """Updates all svn externals (git svn rebase)""" + """Updates all svn externals (git svn fetch[ --revision]/rebase --local)""" exit_if_not_toplevel() - # Must pass in as keywords to fit the interface - for_all_handler("git svn rebase".split()) + # Do show externals to get possible revisions + # looks at svn server + externals = SvnExternal.GetExternals() + + # get the externals dirs + # looks at disk + ext_paths = get_externals_paths() + + for ext in externals: + ext_path = ext.local_storage() + if ext_paths.count(ext_path): + print ">>> " + ext_path + current_dir = os.getcwd() + os.chdir(ext_path) + run_command("git svn fetch %s" % ext.revision_arguments()) + run_command("git svn rebase --local") + + os.chdir(current_dir) # TODO: would it be nice to have a recursive option? From ab0d35a106a72675e63a95ea12709de552e462dd Mon Sep 17 00:00:00 2001 From: Wade Berrier Date: Sat, 26 May 2012 19:20:09 -0600 Subject: [PATCH 12/18] refactor, modularize, and reorganize. also support more external url formats. --- GitSvnExt/__init__.py | 1 + git-svn-ext | 546 ++++++++++++++++++++++++++---------------- tests | 104 ++++++++ 3 files changed, 446 insertions(+), 205 deletions(-) create mode 120000 GitSvnExt/__init__.py create mode 100755 tests diff --git a/GitSvnExt/__init__.py b/GitSvnExt/__init__.py new file mode 120000 index 0000000..624cc90 --- /dev/null +++ b/GitSvnExt/__init__.py @@ -0,0 +1 @@ +../git-svn-ext \ No newline at end of file diff --git a/git-svn-ext b/git-svn-ext index 86b9b12..f566fd2 100755 --- a/git-svn-ext +++ b/git-svn-ext @@ -12,14 +12,13 @@ formatter = logging.Formatter('%(levelname)-8s %(message)s') console.setFormatter(formatter) logger.addHandler(console) logger.level = logging.INFO +#logger.level = logging.DEBUG # This is so we can change directories but still have a reference # to ourselves git_svn_ext_fullpath = os.path.abspath(sys.argv[0]) -externals_dir = ".git_externals" -excludes_file = ".git_external_excludes" -##################### Helper Methods ##################### +################## git/svn Helper Methods ################## def list_references(): """List the references in the local repo. @@ -92,237 +91,272 @@ def find_uncommitted(svn_branch): return results -def exit_if_not_toplevel(): - """exit if not run from the top level git directory""" - toplevel_directory = getoutput("git rev-parse --show-cdup") - if toplevel_directory != "": - print "Error: not in toplevel directory" - sys.exit(1) +def get_svn_info(): + """Parse 'git svn info'""" -def is_excluded(external): - """return True if excluding this external""" + results = {} + found = False + for line in getoutput("git svn info").split("\n"): + match = re.search("(.+): (.+)", line) + if match: + found = True + results[match.group(1)] = match.group(2) - if os.path.exists(excludes_file): - for line in open(excludes_file).readlines(): - if re.search("^/%s$" % external, line): - return True + if not found: + logger.error("'git svn info' failed") + sys.exit(1) - return False + return results +def get_relative_base(): + """Get relative directory to the base of the checkout""" + return getoutput("git rev-parse --show-cdup") -def create_dir_if_not_exist(dir): - """helper to create a directory if it doesn't exist""" - if not os.path.exists(dir): os.mkdir(dir) +def get_git_svn_externals(): + """returns a hash of string->array where the key is the location + in the source of the external property, and the array is the list + of externals""" -def do_link(external_dir, link_dir): - """create the link to the external""" + results = {} - current_dir = os.getcwd() + external_dir = "" + for line in getoutput("git svn show-externals").split("\n"): - os.chdir(external_dir) + external_dir_match = re.search(r"# (.*)", line) + if external_dir_match: + # the external dir is a relative dir, but starts with a slash + # remove the leading / + tmp = re.sub("^/", "", external_dir_match.group(1)) + if tmp != "": + external_dir = tmp + elif line != "": + # start array + if not results.has_key(external_dir): results[external_dir] = [] - # get relative back to git-svn root - rel = getoutput("git rev-parse --show-cdup") + # NOTE: git-svn prepends the external with external_dir, + # undo that + line = re.sub("^/" + external_dir, "", line) + results[external_dir].append(line.rstrip()) - link_name = os.path.basename(link_dir) + logger.debug("Externals: " + str(results)) + return results - # NOTE: the original used -f, so remove and recreate - if os.path.exists(link_name): os.unlink(link_name) +def exit_if_not_toplevel(): + """exit if not run from the top level git directory""" + toplevel_directory = get_relative_base() - os.symlink(rel + externals_dir + os.sep + link_name, link_name) + if toplevel_directory != "": + logger.error("not in toplevel directory") + sys.exit(1) - os.chdir(current_dir) +################## Misc Helper Methods ################## -def append_line_if_not_included(filename, line): +def create_dir_if_not_exist(dir): + """helper to create a directory if it doesn't exist""" + if not os.path.exists(dir): os.makedirs(dir) +def file_contains_line(filename, line): found = False for l in open(filename).readlines(): if line == l.rstrip(): found = True break - if not found: + return found + + +def append_line_if_not_included(filename, line): + + if not file_contains_line(filename, line): fd = open(filename, "a") # TODO: do we need to have a newline to the end of the file # before appending? fd.write(line + "\n") fd.close() -def do_excludes(external_link): - """add symlink to the git excludes path""" - - git_excludes_path=".git/info/exclude" - append_line_if_not_included(git_excludes_path, externals_dir) - append_line_if_not_included(git_excludes_path, external_link) def run_command(command): """print out and run a command""" print command return os.system(command) -def do_clone(local_directory, remote_url, revision): - """do the actual cloning""" - - # make sure the dir exists - if not os.path.exists(externals_dir): - return False - - (module, branch) = re.search("(.*)/(trunk|branch.*|tag.*)", remote_url).groups() - - if branch == remote_url: - branch = "" - - current_dir1 = os.getcwd() - - os.chdir(externals_dir) - - # Default - has_branches_tags = False - - # if we found references to branches and tags, clone them - if module != remote_url: - has_branches_tags = True - - # if we are cloning a subdirectory of a branch path, use the remote_url, and don't get tags/branch/trunk - # ie: if it contains some of these, but does not end with it - if re.search("trunk|tags|branch.*", remote_url): - if not re.search("trunk$|tags$|branch.*$", remote_url): - has_branches_tags = False - module = remote_url - - # Try to figure out what the tags and branches portions are - tags = "tags" - brch = "branches" - branchpath = branch.split("/")[0] - if tags.count(branchpath): tags = branchpath - if brch.count(branchpath): brch = branchpath - - # Format the revision argument - rev_arg = "" - if revision != "": - rev_arg = "--revision BASE:" + revision - - # If the directory is alread there, update it - if os.path.isdir(local_directory): - current_dir2 = os.getcwd() - os.chdir(local_directory) - run_command("git svn fetch " + rev_arg) - # -l skips fetching - run_command("git svn rebase --local") - os.chdir(current_dir2) - # Otherwise, clone it - else: - if has_branches_tags: - run_command("git svn clone %s %s -T trunk -b %s -t %s %s" % (rev_arg, module, brch, tags, local_directory)) +######## Class to parse and represent svn External ########## + +class SvnExternal: + """Represent and parse svn externals from a string + NOTE: must be in the directory where the external is! + because 'git svn info' is queried as part of this + + svn_info is passed in for ease of testing + """ + + def __init__(self, external_string, svn_info): + self.dir = "" + self.url = "" + self.rev = "" + + # 1.6 allows you to reference directly to a file (ignore for now) + self.is_file = False + + # There are a lot of different formats + # http://svnbook.red-bean.com/en/1.7/svn.advanced.externals.html + # + # <1.5 format: + # third-party/skins/toolkit -r21 http://svn.example.com/skin-maker + # 1.5 formats: + # -r21 http://svn.example.com/skin-maker third-party/skins/toolkit + # http://svn.example.com/skin-maker@21 third-party/skins/toolkit + # >=1.6 formats: + # ^/sounds third-party/sounds + # /skinproj@148 third-party/skins + # //svn.example.com/skin-maker@21 third-party/skins/toolkit + # ../skinproj@148 third-party/skins + # file format: + # ^/trunk/bikeshed/blue.html@40 green.html + # + # There's also a quoting and url encode mechanism which isn't implemented yet + + svn_access_part = r"(http|https|svn|svn\+ssh|file)" + scheme_part = r"(%s://)" % svn_access_part + dash_rev_part = r"(-r\s*(?P\d+))?\s*" + + re_pre_1_5 = re.compile(r"(?P.+?)\s+%s(?P%s.+)" % ( dash_rev_part, scheme_part)) + re_1_5_plus = re.compile(r"%s(?P.+?)(@(?P\d+))?\s+(?P.+)" % (dash_rev_part)) + + match_dict = {} + postprocess_1_5 = False + match_pre_1_5 = re_pre_1_5.search(external_string) + match_1_5_plus = re_1_5_plus.search(external_string) + if match_pre_1_5 and not re.search("^\s*-r", external_string): # make sure it doesn't start with -r + logger.debug("Matched as pre 1.5 external: " + external_string) + match_dict = match_pre_1_5.groupdict() + elif match_1_5_plus: + logger.debug("Matched as 1.5 or later external: " + external_string) + postprocess_1_5 = True + match_dict = match_1_5_plus.groupdict() else: - # URL does not contains any trunk, branches or tags part, so we dont need - # additional options for git-svn - run_command("git svn clone %s %s %s" % (rev_arg, module, local_directory)) + logger.error("Unable to parse external: " + external_string) + sys.exit(1) - # If we have branches and tags, checkout that directory - # careful, can blow aways changes! - # remove trailing slash - branch = re.sub("/$", "", branch) + logger.debug("Matches: " + str(match_dict)) - if branch != "" and has_branches_tags: - current_dir3 = os.getcwd() - os.chdir(local_directory) - run_command("git reset --hard " + branch) - os.chdir(current_dir3) + self.dir = match_dict["dir"].lstrip().rstrip() + self.url = match_dict["url"].lstrip().rstrip() - os.chdir(current_dir1) + if match_dict["rev"] != None: + self.rev = match_dict["rev"].lstrip().rstrip() + # this key may not always be in the dict + elif match_dict.has_key("rev2") and match_dict["rev2"] != None: + self.rev = match_dict["rev2"].lstrip().rstrip() - return True + # Handle the several additional options available in 1.5 and beyond + if postprocess_1_5: + url = self.url + logger.debug("Url before post process: " + url) + if url.startswith("^/"): + # Relative to the repository root + url = url[2:] # remove the prefix + url = svn_info["Repository Root"] + os.sep + url + elif url.startswith("//"): + # Relative to the repository URL scheme + url = url[2:] # remove the prefix + scheme = re.search(scheme_part, svn_info["URL"]).group(1) + url = scheme + url + elif url.startswith("/"): + # Relative to the repository server + scheme_server = re.search("(%s.+?)/" % scheme_part, svn_info["URL"]).group(1) + url = scheme_server + url + elif url.startswith("../"): + # Relative to the repository URL + url = svn_info["URL"] + os.sep + url -def get_svn_root_url(): - """Parse out the svn root to use with relative urls""" + logger.debug("Url after post process: " + url) - root = "" - for line in getoutput("git svn info").split("\n"): - matches = re.search("Repository Root: (.*)", line) + # Normalize the path (since .. can also appear in the middle) - if matches: - root = matches.group(1) + # split apart because normpath turns "//" into "/" + matches = re.search("(%s.+?)/(.+)" % scheme_part, url) + #logger.debug("Groups: " + str(matches.groups())) - if root == "": - print "Unable to determine repository root" - sys.exit(1) + scheme_server = matches.group(1) + # group 4 because of all the nested parens + path = os.path.normpath(matches.group(4)) - return root + #logger.debug("Scheme and server: " + scheme_server) + #logger.debug("Path: " + path) -class SvnExternal: - """Class to hold and manipulate data about an svn external""" + # Stick them back together + # maybe a better url parsing method would be better? + url = scheme_server + os.sep + path - def __init__(self, external_dir, local_dir, remote_url, revision): - self.external_dir = external_dir - self.local_dir = local_dir - self.remote_url = remote_url - self.revision = revision + # Legacy: Honor env var to override scheme + # NOTE: projects should use the "//" notation instead + if os.environ.has_key("USE_SSH"): + url = re.sub(svn_access_part, "svn+ssh", url) - @staticmethod - def GetExternals(): - ret = [] + logger.debug("Url after normalization and optional scheme change: " + url) - current_dir = "" - for line in getoutput("git svn show-externals").split("\n"): + self.url = url - current_dir_matches = re.search(r"# /(.*)", line) - # format: current_dir/dir[@rev] [-r 123] dir - # the ?P syntax is to get the stuff by name - re_current_dir = current_dir.replace("/", "\\/") # regular expression safe string - # NOTE: the remote_url section is non-greedy so that @rev is matched correctly - external_matches = re.search(r"/%s(?P.+?)(@(?P\d+))? (-r\s*(?P\d+) )?(?P.+)" % re_current_dir , line) - if current_dir_matches: - # save off the current dir - current_dir = current_dir_matches.group(1) - #print "Setting current_dir to: " + current_dir - continue +######## Class to represent git-svn External ########## + +class GitSvnExternal: + """Class to hold and manipulate data about an svn external + + external_dir: directory in the source that contains the svn:external property + local_dir : the location relative to external_dir where the external source will be + remote_url : svn external + revision : optional pinned revision + """ - elif external_matches: + ExternalsDir = ".git_externals" + LocalExclude = ".git/info/exclude" + ExcludesFile = ".git_external_excludes" - external_info = external_matches.groupdict() + def __init__(self, external_dir, svn_external): + self.external_dir = external_dir + self.local_dir = svn_external.dir + self.remote_url = svn_external.url + self.revision = svn_external.rev - # Check for relative url - remote_url = external_info["remote_url"] - # TODO: there are probably other external url types that we don't handle - if remote_url.count("^"): - svn_root_url = get_svn_root_url() - svn_base = os.path.dirname(svn_root_url) - svn_path = os.path.basename(svn_root_url) - remote_url = remote_url.replace("^", svn_path) + # maybe have this go to the repo root automatically? + @staticmethod + def GetClonedPaths(): + """Get a list of paths to externals for the current dir""" - # Normalize the path - remote_url = os.path.normpath(remote_url) + results = [] - # Stick them back together - # had to split apart because normpath turns "//" into "/" - # maybe a better url parsing method would be better? - remote_url = svn_base + os.sep + remote_url + if os.path.isdir(GitSvnExternal.ExternalsDir): + for external in os.listdir(GitSvnExternal.ExternalsDir): + gitdir = os.path.join(GitSvnExternal.ExternalsDir, external, ".git") + if os.path.isdir(gitdir): + results.append(os.path.dirname(gitdir)) - local_dir = external_info["local_dir"] + return results - # Determine the revision (hopefully only 1 is specified) - revision = "" - if external_info["rev1"] != None: - revision = external_info["rev1"] - elif external_info["rev2"] != None: - revision = external_info["rev2"] + @staticmethod + def GetExternals(): + ret = [] - # Allow exclusion of externals - if not is_excluded(local_dir): + for src_dir, externals in get_git_svn_externals().items(): + for ext in externals: - # Honor env var to override url - if os.environ.has_key("USE_SSH"): - remote_url = re.sub("(http|https)", "svn+ssh", remote_url) + logger.debug("Source dir: %s, External: %s" % (src_dir, ext)) - # Add the info - if remote_url != "": - ret.append(SvnExternal(current_dir, local_dir, remote_url, revision)) + svn_ext = SvnExternal(ext, get_svn_info()) + + git_svn_ext = GitSvnExternal(src_dir, svn_ext) + + # Allow exclusion of externals + if not git_svn_ext.is_excluded_(): + ret.append(git_svn_ext) + else: + logger.info("Excluding svn external: " + git_svn_ext.symlink_()) return ret @@ -331,7 +365,11 @@ class SvnExternal: # TODO/NOTE: it's possible to have 2 externals of the same name # in different directories - return externals_dir + os.sep + self.local_dir + return GitSvnExternal.ExternalsDir + os.sep + self.local_dir + + def symlink_(self): + """Get the path of the symlink that points to local_storage()""" + return self.external_dir + self.local_dir def revision_arguments(self): """Format argumntes if there's a revision""" @@ -342,46 +380,138 @@ class SvnExternal: rev_arg = "--revision BASE:" + self.revision return rev_arg -# maybe have this go to the repo root automatically? -def get_externals_paths(): - """Get a list of paths to externals for the current dir""" + def is_excluded_(self): + """return True if excluding this external""" - results = [] + excluded = False + if os.path.exists(GitSvnExternal.ExcludesFile): + excluded = file_contains_line(GitSvnExternal.ExcludesFile, self.symlink_()) + + return excluded + + def create_link(self): + """create the link to the external""" + + # NOTE: the original used -f, so remove and recreate + # BUG: it looks like os.remove follows links? + # this fails when an invalid link exists... + if os.path.exists(self.symlink_()): + logger.debug("Removing symlink: " + self.symlink_()) + os.remove(self.symlink_()) + + # construct relative link + current_dir = os.getcwd() + os.chdir(os.path.dirname(self.symlink_())) + rel = get_relative_base() + os.chdir(current_dir) + + source = rel + self.local_storage() + link_name = self.symlink_() + logger.debug("Creating link from %s to %s" % (source, link_name)) + os.symlink(source, link_name) + + def update_excludes(self): + """add symlink to the git excludes path""" + + append_line_if_not_included(GitSvnExternal.LocalExclude, GitSvnExternal.ExternalsDir) + append_line_if_not_included(GitSvnExternal.LocalExclude, self.symlink_()) + + + def clone(self): + """do the actual cloning""" + + (module, branch) = re.search("(.*)/(trunk|branch.*|tag.*)", self.remote_url).groups() + + if branch == self.remote_url: + branch = "" + + # Default + has_branches_tags = False + + # if we found references to branches and tags, clone them + if module != self.remote_url: + has_branches_tags = True + + # if we are cloning a subdirectory of a branch path, use the remote_url, and don't get tags/branch/trunk + # ie: if it contains some of these, but does not end with it + if re.search("trunk|tags|branch.*", self.remote_url): + if not re.search("trunk$|tags$|branch.*$", self.remote_url): + has_branches_tags = False + module = self.remote_url + + # Try to figure out what the tags and branches portions are + tags = "tags" + brch = "branches" + branchpath = branch.split("/")[0] + if tags.count(branchpath): tags = branchpath + if brch.count(branchpath): brch = branchpath + + # Format the revision argument + rev_arg = self.revision_arguments() + + # If the directory is alread there, update it + if os.path.isdir(self.local_storage()): + current_dir = os.getcwd() + os.chdir(self.local_storage()) + run_command("git svn fetch " + rev_arg) + # --local skips fetching (for rev pinning) + run_command("git svn rebase --local") + os.chdir(current_dir) + # Otherwise, clone it + else: + clone_dir = os.path.dirname(self.local_storage()) + rel_local_storage = os.path.basename(self.local_storage()) + + # Create dir to run clone from + create_dir_if_not_exist(clone_dir) + + current_dir = os.getcwd() + os.chdir(clone_dir) + + if has_branches_tags: + run_command("git svn clone %s %s --trunk trunk --branches %s --tags %s %s" % (rev_arg, module, brch, tags, rel_local_storage)) + else: + # URL does not contains any trunk, branches or tags part, so we dont need + # additional options for git-svn + run_command("git svn clone %s %s %s" % (rev_arg, module, rel_local_storage)) + + os.chdir(current_dir) + + # If we have branches and tags, checkout that directory + # careful, can blow aways changes! + # remove trailing slash + branch = re.sub("/$", "", branch) + + if branch != "" and has_branches_tags: + current_dir = os.getcwd() + os.chdir(self.local_storage()) + run_command("git reset --hard " + branch) + os.chdir(current_dir) - if os.path.isdir(externals_dir): - for external in os.listdir(externals_dir): - gitdir = os.path.join(externals_dir, external, ".git") - if os.path.isdir(gitdir): - results.append(os.path.dirname(gitdir)) - return results ########### sub-command handlers ############# def clone_handler(args): """clone all svn externals into .git_externals - (warning: removes local changes and commits on subsequent runs)""" + (warning: removes local changes and commits on subsequent runs)""" exit_if_not_toplevel() - externals = SvnExternal.GetExternals() + externals = GitSvnExternal.GetExternals() + + if not len(externals): + logger.error("No svn externals found") + sys.exit(1) for external in externals: print "%s -> %s" % (external.local_dir, external.remote_url) - dir = os.path.dirname(external.local_dir) - - # just create two dirs instead of pulling in mkpath - create_dir_if_not_exist(externals_dir) - create_dir_if_not_exist(os.path.join(externals_dir, dir)) - - if not do_clone(external.local_dir, external.remote_url, external.revision): - sys.exit(1) - - do_link(external.external_dir, external.local_dir) - do_excludes(external.external_dir + external.local_dir) + external.clone() + external.create_link() + external.update_excludes() # TODO: maybe clone should just do this instead? # but clone can blow stuff away... (reset --hard) @@ -392,11 +522,15 @@ def update_handler(args): # Do show externals to get possible revisions # looks at svn server - externals = SvnExternal.GetExternals() + externals = GitSvnExternal.GetExternals() + + if not len(externals): + logger.error("No svn externals found") + sys.exit(1) # get the externals dirs # looks at disk - ext_paths = get_externals_paths() + ext_paths = GitSvnExternal.GetClonedPaths() for ext in externals: ext_path = ext.local_storage() @@ -421,7 +555,7 @@ def check_unpushed_handler(args): sys.exit(1) svn_branch = find_svn_branch_name() if svn_branch is None: - print "Warning: no svn branch found for: " + os.getcwd() + logger.error("no svn branch found for: " + os.getcwd()) sys.exit(1) logger.debug('Found branch: %s', svn_branch) @@ -438,7 +572,7 @@ def check_handler(args): exit_if_not_toplevel() - for dir in get_externals_paths(): + for dir in GitSvnExternal.GetClonedPaths(): current_dir = os.getcwd() os.chdir(dir) @@ -458,12 +592,12 @@ def check_handler(args): def for_all_handler(args): """run a command against all externals - (ie: git svn-ext for-all git grep 'whatever')""" + (ie: git svn-ext for-all git grep 'whatever')""" command = " ".join(args) if command: - for dir in get_externals_paths(): + for dir in GitSvnExternal.GetClonedPaths(): current_dir = os.getcwd() os.chdir(dir) @@ -496,7 +630,7 @@ class Handlers: else: return None - def _aligned_name(self, name): + def aligned_name_(self, name): """string of spaces to pad command to align text""" # find out how long the lines are so they can line up @@ -513,7 +647,7 @@ class Handlers: ret = "" for n in self.names: - ret += " %s: %s\n" % (self._aligned_name(n), self.handlers[n].__doc__) + ret += " %s: %s\n" % (self.aligned_name_(n), self.handlers[n].__doc__) return ret @@ -531,8 +665,10 @@ def usage(): print "" print "Usage: " + sys.argv[0] + " [sub command args]" print "" - print " sub commmands:" + print " sub commmands:" print handlers.helptext() + print " Notes:" + print " externals may be ignored if listed in " + GitSvnExternal.ExcludesFile print "" if __name__ == '__main__': @@ -544,7 +680,7 @@ if __name__ == '__main__': handler = handlers.get(sub_command) if handler == None: - raise "invalid subcommand" + raise BaseException except: usage() diff --git a/tests b/tests new file mode 100755 index 0000000..c69ed68 --- /dev/null +++ b/tests @@ -0,0 +1,104 @@ +#!/usr/bin/env python + +# There are a lot of formats... test them out with some examples + +git_svn_info = { + "Path": ".", + "URL": "svn+ssh://svn.example.com/skin-maker/mydir", + "Repository Root": "svn+ssh://svn.example.com/skin-maker", + "Repository UUID": "a4bac7da-eecc-44ef-9201-0e2b325a63d8", + "Revision": "1000", + "Node Kind": "directory", + "Schedule": "normal", + "Last Changed Author": "wberrier", + "Last Changed Rev": "1000", + "Last Changed Date": "2012-05-25 15:49:11 -0600 (Fri, 25 May 2012)" + } + + +import GitSvnExt + +import unittest + +class TestSvnExternalParsing(unittest.TestCase): + + def setUp(self): + pass + + def testPRE1_5_a(self): + ext = GitSvnExt.SvnExternal("third-party/skins/toolkit -r21 http://svn.example.com/skin-maker", git_svn_info) + + self.assertEqual(ext.dir, "third-party/skins/toolkit") + self.assertEqual(ext.rev, "21") + self.assertEqual(ext.url, "http://svn.example.com/skin-maker") + + def testPRE1_5_b(self): + """Surrounding space, no revision, and file scheme""" + ext = GitSvnExt.SvnExternal(" third-party/skins/toolkit file:///svn.example.com/skin-maker ", git_svn_info) + + self.assertEqual(ext.dir, "third-party/skins/toolkit") + self.assertEqual(ext.rev, "") + self.assertEqual(ext.url, "file:///svn.example.com/skin-maker") + + def test1_5_a(self): + ext = GitSvnExt.SvnExternal("http://svn.example.com/skin-maker third-party/skins/toolkit", git_svn_info) + + self.assertEqual(ext.dir, "third-party/skins/toolkit") + self.assertEqual(ext.rev, "") + self.assertEqual(ext.url, "http://svn.example.com/skin-maker") + + def test1_5_b(self): + ext = GitSvnExt.SvnExternal("http://svn.example.com/skin-maker@21 third-party/skins/toolkit", git_svn_info) + + self.assertEqual(ext.dir, "third-party/skins/toolkit") + self.assertEqual(ext.rev, "21") + self.assertEqual(ext.url, "http://svn.example.com/skin-maker") + + def test1_5_c(self): + ext = GitSvnExt.SvnExternal("-r21 http://svn.example.com/skin-maker third-party/skins/toolkit", git_svn_info) + + self.assertEqual(ext.dir, "third-party/skins/toolkit") + self.assertEqual(ext.rev, "21") + self.assertEqual(ext.url, "http://svn.example.com/skin-maker") + + def test1_6_a(self): + ext = GitSvnExt.SvnExternal("-r21 /skin-maker third-party/skins/toolkit", git_svn_info) + + self.assertEqual(ext.dir, "third-party/skins/toolkit") + self.assertEqual(ext.rev, "21") + self.assertEqual(ext.url, "svn+ssh://svn.example.com/skin-maker") + + def test1_6_b(self): + ext = GitSvnExt.SvnExternal("//svn.example.com/skin-maker@21 third-party/skins/toolkit", git_svn_info) + + self.assertEqual(ext.dir, "third-party/skins/toolkit") + self.assertEqual(ext.rev, "21") + self.assertEqual(ext.url, "svn+ssh://svn.example.com/skin-maker") + + def test1_6_c(self): + ext = GitSvnExt.SvnExternal("^/subdir@21 third-party/skins/toolkit", git_svn_info) + + self.assertEqual(ext.dir, "third-party/skins/toolkit") + self.assertEqual(ext.rev, "21") + self.assertEqual(ext.url, "svn+ssh://svn.example.com/skin-maker/subdir") + + def test1_6_d(self): + ext = GitSvnExt.SvnExternal("../subdir@21 third-party/skins/toolkit", git_svn_info) + + self.assertEqual(ext.dir, "third-party/skins/toolkit") + self.assertEqual(ext.rev, "21") + self.assertEqual(ext.url, "svn+ssh://svn.example.com/skin-maker/subdir") + + def test1_6_e(self): + ext = GitSvnExt.SvnExternal("^/../super-skin-maker@21 third-party/skins/toolkit", git_svn_info) + + self.assertEqual(ext.dir, "third-party/skins/toolkit") + self.assertEqual(ext.rev, "21") + self.assertEqual(ext.url, "svn+ssh://svn.example.com/super-skin-maker") + + def tearDown(self): + pass + + +if __name__ == '__main__': + unittest.main() From b6aa6b4540cd69009cd227f9ae5febfa47db1c15 Mon Sep 17 00:00:00 2001 From: Wade Berrier Date: Tue, 12 Jun 2012 23:47:53 -0600 Subject: [PATCH 13/18] Update README.txt --- README.txt | 96 ++++++++++++++++-------------------------------------- 1 file changed, 28 insertions(+), 68 deletions(-) diff --git a/README.txt b/README.txt index 25ef868..3e8bfa1 100644 --- a/README.txt +++ b/README.txt @@ -1,9 +1,13 @@ -git-svn-clone-externals -======================= +git-svn-ext +=========== -This is a very simple shell script to make git-svn clone your svn:externals -definitions. Place the script in a directory where you have one or more -svn:externals definitions, run it, and it will: +This work is based on git-svn-clone-externals and the associated +scripts written by Andre Pang. + +The methodology remains the same. Put git-svn-ext in your path and +run it from the top-level tree checked out with git-svn. git-svn-ext +will detect and git-svn clone the externals using the following +method: * git svn clone each external into a .git_externals/ directory. * symlink the cloned repository in .git_externals/ to the proper directory @@ -11,74 +15,30 @@ svn:externals definitions, run it, and it will: * add the symlink and .git_externals/ to the .git/info/excludes/ file, so that you're not pestered about it when performing a git status. -That's pretty much about it. Low-tech and cheap and cheery, but I couldn't -find anything else like it after extensive Googling, so hopefully some other -people out there with low-tech minds like mine will find this useful. - -You could certainly make the script a lot more complex and do things such as -share svn:externals repositories between different git repositories, traverse -through the entire git repository to detect svn:externals definitions instead -of having to place the script in the correct directory, etc... but this works, -it's simple, and it does just the one thing, unlike a lot of other git/svn -integration scripts that I've found. I absolutely do welcome those features, -but I figured I'd push this out since it works for me and is probably useful -for others. - -NB: This assumes you have passwordless svn. - -Enjoy, - -- Andre Pang - - -Tools -===== - -The git-svn-clone-externals script does a great job of cloning the -svn:externals. However, in day-to-day work I want to check whether I -need to push stuff and update a buch of 'external' repositories in one -go. Therefore I creates some additional scripts. - -* ``git-svn-check-unpushed`` tries to determine whether there are - commits which are not yet pushed back to the subversion - repository. Originally I took this idea from Magit (an interface to - the version control system Git, implemented as an extension to - Emacs) and implemented it in Python instead of Lisp. +but the code has been converted to python as one script with +sub-commands, supports newer subversion urls, and adds a few new +features. - This script can be run in every location of a git repository. +From the usage output: -* git-svn-externals-check is a script that displays whether there are - uncommitted changes or commits that are not pushed to the subversion - repository yet. Basically it executes ``git status`` and the - ``git-svn-check-unpushed`` scripts for each directory in the current - directory. +Usage: ./git-svn-ext [sub command args] - This script must be run in the directory where the original - svn:externals property was set. It does not walk the complete tree - to search for these kind of repositories. + sub commmands: + clone : clone all svn externals into .git_externals + (warning: removes local changes and commits on subsequent runs) + update : Updates all svn externals (git svn fetch[ --revision]/rebase --local) + check-unpushed : Check if local git-svn checkout has unpushed commits + check : run 'git status' and 'check-unpushed' for all externals + for-all : run a command against all externals + (ie: git svn-ext for-all git grep 'whatever') -* git-svn-externals-update complements the git-svn-externals-check - script. The update script does an ``git svn fetch`` and ``git svn - rebase`` for every directory in the location where the script is - executed. + Notes: + externals may be ignored if listed in .git_external_excludes - This script must also be run in the directory where the original - svn:externals property was set. - - -Feel free to use and improve these scripts. - -- Mark van Lent - -Options +Authors ======= - -* External repository's url can be rewritten, to use SVN+SSH instead of - plain HTTP or HTTPS. To do so, do `export USE_SSH=yes` in your environment. - This can be useful if you use ssh authentication, but other developers don't. -* If you don't want to pull all external repositories, you can create a - .git_externals_exclude file which contains the local paths to be excluded, - one per line, the same way they show up on the first field of git svn show-externals - +- Andre Pang +- Mark van Lent - Alexander Artemenko +- Wade Berrier From 46a8a567d01291904d6df2ad9ea1d5e5dd02c567 Mon Sep 17 00:00:00 2001 From: Wade Berrier Date: Fri, 15 Jun 2012 14:08:52 -0600 Subject: [PATCH 14/18] Use subprocess to reimplement getoutput instead of commands.getoutput That way stderr can be ignored, which needs to be the case with git svn show-externals --- README.txt | 4 ++++ git-svn-ext | 27 +++++++++++++++++++++++++-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/README.txt b/README.txt index 3e8bfa1..5cceffb 100644 --- a/README.txt +++ b/README.txt @@ -42,3 +42,7 @@ Authors - Alexander Artemenko - Wade Berrier +TODO +==== +- make clone not blow stuff away +- handle the case when externals are removed or altered diff --git a/git-svn-ext b/git-svn-ext index f566fd2..0245530 100755 --- a/git-svn-ext +++ b/git-svn-ext @@ -1,6 +1,6 @@ #!/usr/bin/env python -from commands import getoutput +import subprocess import os import re import sys @@ -18,6 +18,29 @@ logger.level = logging.INFO # to ourselves git_svn_ext_fullpath = os.path.abspath(sys.argv[0]) +################## shell helpers ########################### +def getoutput(command, include_stderr=True): + """Get the output of a command, optionally ignoring stderr""" + + if include_stderr: + stderr=subprocess.STDOUT + else: + # Pipe stderr, but we ignore it + stderr=subprocess.PIPE + + proc = subprocess.Popen(command, shell=True, stderr=stderr, stdout=subprocess.PIPE) + + # Wait for the process to finish + proc.wait() + + # Get stdout (which may optionally include stderr) + output = proc.communicate()[0] + + # Keep compatibility with commands.getoutput() by removing trailing newline + output = output.rstrip('\n') + + return output + ################## git/svn Helper Methods ################## def list_references(): @@ -122,7 +145,7 @@ def get_git_svn_externals(): results = {} external_dir = "" - for line in getoutput("git svn show-externals").split("\n"): + for line in getoutput("git svn show-externals", include_stderr=False).split("\n"): external_dir_match = re.search(r"# (.*)", line) if external_dir_match: From 9f030b4a5b760aa24d8c1cc914ae3ed16bcf41bc Mon Sep 17 00:00:00 2001 From: rockeyca Date: Mon, 3 Apr 2017 12:38:22 -0700 Subject: [PATCH 15/18] Updates regular expressions for additional svn-external formats --- git-svn-ext | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/git-svn-ext b/git-svn-ext index 0245530..c3d8507 100755 --- a/git-svn-ext +++ b/git-svn-ext @@ -151,9 +151,10 @@ def get_git_svn_externals(): if external_dir_match: # the external dir is a relative dir, but starts with a slash # remove the leading / - tmp = re.sub("^/", "", external_dir_match.group(1)) - if tmp != "": - external_dir = tmp + if external_dir_match.group(1) and external_dir_match.group(1)[0] == "/": + tmp = re.sub("^/", "", external_dir_match.group(1)) + if tmp != "": + external_dir = tmp elif line != "": # start array if not results.has_key(external_dir): results[external_dir] = [] @@ -247,7 +248,7 @@ class SvnExternal: dash_rev_part = r"(-r\s*(?P\d+))?\s*" re_pre_1_5 = re.compile(r"(?P.+?)\s+%s(?P%s.+)" % ( dash_rev_part, scheme_part)) - re_1_5_plus = re.compile(r"%s(?P.+?)(@(?P\d+))?\s+(?P.+)" % (dash_rev_part)) + re_1_5_plus = re.compile(r"/?%s'?(?P.+?)(@(?P\d+))?'?\s+(?P.+)" % (dash_rev_part)) match_dict = {} postprocess_1_5 = False From 1d6d6a58a1c83276e2ee24c5349b4f1d14c0bc3c Mon Sep 17 00:00:00 2001 From: rockeyca Date: Mon, 3 Apr 2017 12:46:21 -0700 Subject: [PATCH 16/18] Handle parent directories for output symlink directories --- git-svn-ext | 3 +++ 1 file changed, 3 insertions(+) diff --git a/git-svn-ext b/git-svn-ext index 0245530..ca77b7a 100755 --- a/git-svn-ext +++ b/git-svn-ext @@ -424,6 +424,9 @@ class GitSvnExternal: # construct relative link current_dir = os.getcwd() + # Create parent directories (eg: foo/bar/baz) if required + if not os.path.exists(os.path.dirname(self.symlink_())): + os.makedirs(os.path.dirname(self.symlink_())) os.chdir(os.path.dirname(self.symlink_())) rel = get_relative_base() os.chdir(current_dir) From 4143f60f0372248e1fa5ab1072e76c428913e4b2 Mon Sep 17 00:00:00 2001 From: lepommosaure <56764164+lepommosaure@users.noreply.github.com> Date: Thu, 23 Jan 2025 14:53:58 +0100 Subject: [PATCH 17/18] chore: converting to python3 --- git-svn-ext | 72 ++++++++++++++++++++++++++--------------------------- 1 file changed, 35 insertions(+), 37 deletions(-) diff --git a/git-svn-ext b/git-svn-ext index 299598e..83551d5 100755 --- a/git-svn-ext +++ b/git-svn-ext @@ -37,7 +37,7 @@ def getoutput(command, include_stderr=True): output = proc.communicate()[0] # Keep compatibility with commands.getoutput() by removing trailing newline - output = output.rstrip('\n') + output = output.rstrip(b'\n') return output @@ -52,7 +52,7 @@ def list_references(): references = getoutput('git show-ref --head') HEAD_ref = None refs = {} - for item in references.split('\n'): + for item in references.decode('utf-8').split('\n'): stuff = item.split() if len(stuff) == 2: (sha1, name) = stuff @@ -70,7 +70,7 @@ def commit_objects(): """ commit_list = getoutput('git rev-list --all --parents') commits = {} - for item in commit_list.split('\n'): + for item in commit_list.decode('utf-8').split('\n'): splitted_item = item.split() if len(splitted_item) != 2: commit = splitted_item[0] @@ -85,7 +85,6 @@ def find_svn_branch_name(): """Return the reference name of the current remote branch.""" head, references = list_references() commits = commit_objects() - current_commit = head while current_commit: if current_commit in references: @@ -110,7 +109,7 @@ def find_uncommitted(svn_branch): output = getoutput('git log --pretty="format:%%h %%s" %s..HEAD' % svn_branch) if output: - results += output.split('\n') + results += output.decode('utf-8').split('\n') return results @@ -120,7 +119,7 @@ def get_svn_info(): results = {} found = False - for line in getoutput("git svn info").split("\n"): + for line in getoutput("git svn info").decode('utf-8').split("\n"): match = re.search("(.+): (.+)", line) if match: found = True @@ -145,7 +144,7 @@ def get_git_svn_externals(): results = {} external_dir = "" - for line in getoutput("git svn show-externals", include_stderr=False).split("\n"): + for line in getoutput("git svn show-externals", include_stderr=False).decode('utf-8').split("\n"): external_dir_match = re.search(r"# (.*)", line) if external_dir_match: @@ -157,7 +156,7 @@ def get_git_svn_externals(): external_dir = tmp elif line != "": # start array - if not results.has_key(external_dir): results[external_dir] = [] + if external_dir not in results: results[external_dir] = [] # NOTE: git-svn prepends the external with external_dir, # undo that @@ -170,8 +169,7 @@ def get_git_svn_externals(): def exit_if_not_toplevel(): """exit if not run from the top level git directory""" toplevel_directory = get_relative_base() - - if toplevel_directory != "": + if toplevel_directory.decode("utf-8") != "": logger.error("not in toplevel directory") sys.exit(1) @@ -203,7 +201,7 @@ def append_line_if_not_included(filename, line): def run_command(command): """print out and run a command""" - print command + print (command) return os.system(command) @@ -269,11 +267,11 @@ class SvnExternal: self.dir = match_dict["dir"].lstrip().rstrip() self.url = match_dict["url"].lstrip().rstrip() - + print(f"{match_dict=}") if match_dict["rev"] != None: self.rev = match_dict["rev"].lstrip().rstrip() # this key may not always be in the dict - elif match_dict.has_key("rev2") and match_dict["rev2"] != None: + elif "rev2" in match_dict and match_dict["rev2"] != None: self.rev = match_dict["rev2"].lstrip().rstrip() # Handle the several additional options available in 1.5 and beyond @@ -319,7 +317,7 @@ class SvnExternal: # Legacy: Honor env var to override scheme # NOTE: projects should use the "//" notation instead - if os.environ.has_key("USE_SSH"): + if "USE_SSH" in os.environ: url = re.sub(svn_access_part, "svn+ssh", url) logger.debug("Url after normalization and optional scheme change: " + url) @@ -367,7 +365,7 @@ class GitSvnExternal: def GetExternals(): ret = [] - for src_dir, externals in get_git_svn_externals().items(): + for src_dir, externals in list(get_git_svn_externals().items()): for ext in externals: logger.debug("Source dir: %s, External: %s" % (src_dir, ext)) @@ -469,7 +467,7 @@ class GitSvnExternal: # Try to figure out what the tags and branches portions are tags = "tags" brch = "branches" - branchpath = branch.split("/")[0] + branchpath = branch.decode('utf-8').split("/")[0] if tags.count(branchpath): tags = branchpath if brch.count(branchpath): brch = branchpath @@ -534,7 +532,7 @@ def clone_handler(args): for external in externals: - print "%s -> %s" % (external.local_dir, external.remote_url) + print("%s -> %s" % (external.local_dir, external.remote_url)) external.clone() external.create_link() @@ -562,7 +560,7 @@ def update_handler(args): for ext in externals: ext_path = ext.local_storage() if ext_paths.count(ext_path): - print ">>> " + ext_path + print(">>> " + ext_path) current_dir = os.getcwd() os.chdir(ext_path) run_command("git svn fetch %s" % ext.revision_arguments()) @@ -577,8 +575,8 @@ def check_unpushed_handler(args): """Check if local git-svn checkout has unpushed commits""" status = getoutput('git status') - if status.startswith('fatal'): - print status + if status.decode('utf-8').startswith('fatal'): + print(status) sys.exit(1) svn_branch = find_svn_branch_name() if svn_branch is None: @@ -589,10 +587,10 @@ def check_unpushed_handler(args): commits = find_uncommitted(svn_branch) if len(commits) > 0: - print 'Possible unpushed commits (against %s):' % svn_branch - print "\n".join(commits) + print('Possible unpushed commits (against %s):' % svn_branch) + print("\n".join(commits)) else: - print 'No unpushed commits found.' + print('No unpushed commits found.') def check_handler(args): """run 'git status' and 'check-unpushed' for all externals""" @@ -606,14 +604,14 @@ def check_handler(args): status = getoutput("git status") unpushed = getoutput(git_svn_ext_fullpath + " check-unpushed") - if not re.search("clean", status) or not re.search("No unpushed", unpushed): - print ">>>>>>>>>>>>>>>> %s <<<<<<<<<<<<<<<<" % dir + if not re.search("clean", status.decode('utf-8')) or not re.search("No unpushed", unpushed.decode('utf-8')): + print(">>>>>>>>>>>>>>>> %s <<<<<<<<<<<<<<<<" % dir) # Run these again to get color output os.system("git status") os.system(git_svn_ext_fullpath + " check-unpushed") - print "----------------------------------------" + print("----------------------------------------") else: - print dir + " is clean" + print(dir + " is clean") os.chdir(current_dir) @@ -629,7 +627,7 @@ def for_all_handler(args): current_dir = os.getcwd() os.chdir(dir) - print ">>> %s: %s" % (dir, command) + print(">>> %s: %s" % (dir, command)) os.system(command) @@ -652,7 +650,7 @@ class Handlers: self.handlers[name] = handler def get(self, name): - if self.handlers.has_key(name): + if name in self.handlers: return self.handlers[name] else: return None @@ -689,14 +687,14 @@ handlers.add('for-all', for_all_handler) def usage(): """Descriptive text of all options""" - print "" - print "Usage: " + sys.argv[0] + " [sub command args]" - print "" - print " sub commmands:" - print handlers.helptext() - print " Notes:" - print " externals may be ignored if listed in " + GitSvnExternal.ExcludesFile - print "" + print("") + print("Usage: " + sys.argv[0] + " [sub command args]") + print("") + print(" sub commmands:") + print(handlers.helptext()) + print(" Notes:") + print(" externals may be ignored if listed in " + GitSvnExternal.ExcludesFile) + print("") if __name__ == '__main__': From 9ced4f70564acd1eea0b4ad182858b1f33d244a9 Mon Sep 17 00:00:00 2001 From: lepommosaure <56764164+lepommosaure@users.noreply.github.com> Date: Thu, 23 Jan 2025 14:58:14 +0100 Subject: [PATCH 18/18] fix: removing debug print --- git-svn-ext | 1 - 1 file changed, 1 deletion(-) diff --git a/git-svn-ext b/git-svn-ext index 83551d5..260f294 100755 --- a/git-svn-ext +++ b/git-svn-ext @@ -267,7 +267,6 @@ class SvnExternal: self.dir = match_dict["dir"].lstrip().rstrip() self.url = match_dict["url"].lstrip().rstrip() - print(f"{match_dict=}") if match_dict["rev"] != None: self.rev = match_dict["rev"].lstrip().rstrip() # this key may not always be in the dict