Merge pull request #7 from JuliaPackaging/tk/bindeps

RFC: download a binary of git if it is not already installed

Author: tkelman

.gitignore

@@ -1,3 +1,6 @@
 *.jl.cov
 *.jl.*.cov
 *.jl.mem
+deps/downloads/
+deps/usr/
+deps/deps.jl

REQUIRE

@@ -1,2 +1,4 @@
 julia 0.4
 Compat 0.8.4
+BinDeps
+@osx Homebrew

appveyor.yml

@@ -20,6 +20,11 @@ notifications:
     on_build_status_changed: false
 
 install:
+# If there's a newer build queued for the same PR, cancel this one
+  - ps: if ($env:APPVEYOR_PULL_REQUEST_NUMBER -and $env:APPVEYOR_BUILD_NUMBER -ne ((Invoke-RestMethod `
+        https://ci.appveyor.com/api/projects/$env:APPVEYOR_ACCOUNT_NAME/$env:APPVEYOR_PROJECT_SLUG/history?recordsNumber=50).builds | `
+        Where-Object pullRequestId -eq $env:APPVEYOR_PULL_REQUEST_NUMBER)[0].buildNumber) { `
+        throw "There are newer queued builds for this pull request, failing early." }
 # Download most recent Julia Windows binary
   - ps: (new-object net.webclient).DownloadFile(
         $("http://s3.amazonaws.com/"+$env:JULIAVERSION),

deps/build.jl

@@ -0,0 +1,76 @@
+using Compat
+import BinDeps: download_cmd, unpack_cmd, splittarpath
+if is_apple()
+    using Homebrew
+end
+
+gitcmd = `git`
+gitver = "notfound"
+try
+    gitver = readchomp(`$gitcmd --version`)
+end
+if gitver == "notfound"
+    if is_apple()
+        # we could allow other options, but lots of other packages already
+        # depend on Homebrew.jl on mac and it needs a working git to function
+        error("Working git not found on path, try running\nPkg.build(\"Homebrew\")")
+    end
+    baseurl = ""
+    filename = ""
+    if is_linux() && Sys.ARCH === :x86_64
+        # use conda for a non-root option
+        gitver = "2.8.2"
+        baseurl = "https://anaconda.org/conda-forge/git/$gitver/download/linux-64/"
+        filename = "git-$gitver-2.tar.bz2"
+    elseif is_linux() && (Sys.ARCH in (:i686, :i586, :i486, :i386))
+        # conda-forge doesn't build for 32 bit linux
+        gitver = "2.6.4"
+        baseurl = "https://anaconda.org/anaconda/git/$gitver/download/linux-32/"
+        filename = "git-$gitver-0.tar.bz2"
+    elseif is_windows()
+        # download and extract portablegit
+        gitver = "2.9.0"
+        baseurl = "https://github.com/git-for-windows/git/releases/" *
+            "download/v$gitver.windows.1/"
+        filename = "PortableGit-$gitver-$(Sys.WORD_SIZE)-bit.7z.exe"
+    end
+    downloadurl = baseurl * filename
+    if !isempty(downloadurl)
+        info("Downloading git version $gitver from $downloadurl\nTo " *
+            "avoid this download, install git manually and add it to your " *
+            "path before\nrunning Pkg.add(\"Git\") or Pkg.build(\"Git\")")
+        dest = joinpath("usr", string(Sys.MACHINE))
+        for dir in ("downloads", "usr", dest)
+            isdir(dir) || mkdir(dir)
+        end
+        filename = joinpath("downloads", filename)
+        isfile(filename) || run(download_cmd(downloadurl, filename))
+        # TODO: checksum validation
+        (b, ext, sec_ext) = splittarpath(filename)
+        run(unpack_cmd(filename, dest, is_windows() ? ".7z" : ext, sec_ext))
+        # TODO: make this less noisy on windows, see how WinRPM does it
+        gitcmd = `$(joinpath(dirname(@__FILE__), dest, "bin", "git"))`
+    end
+    try
+        gitver = readchomp(`$gitcmd --version`)
+        info("Successfully installed $gitver to $gitcmd")
+        # TODO: fix a warning about missing /templates here on linux
+        # by setting an environment variable in deps.jl
+    catch err
+        error("Could not automatically install git, error was: $err\n" *
+            (isempty(downloadurl) ? "Try installing git via your system " *
+            "package manager then running\nPkg.build(\"Git\")" : ""))
+    end
+else
+    try
+        # this is in a try because some environments like centos 7
+        # docker containers don't have `which` installed by default
+        gitpath = chomp(readlines(is_windows() ? `where git` : `which git`)[1])
+        gitcmd = `$gitpath`
+    end
+    info("Using $gitver found on path" * (gitcmd == `git` ?
+        "" : " at $gitcmd"))
+end
+open("deps.jl", "w") do f
+    println(f, "gitcmd = $gitcmd")
+end

src/Git.jl

@@ -6,19 +6,25 @@ module Git
 #
 using Compat
 import Base: shell_escape
+export gitcmd # determined by deps/build.jl and saved in deps/deps.jl
+
+depsjl = joinpath(dirname(@__FILE__), "..", "deps", "deps.jl")
+isfile(depsjl) ? include(depsjl) : error("Git.jl not properly installed. " *
+    "Please run\nPkg.build(\"Git\")")
 
 function dir(d)
     g = joinpath(d,".git")
     isdir(g) && return g
-    normpath(d, Base.readchomp(setenv(`git rev-parse --git-dir`, dir=d)))
+    normpath(d, Base.readchomp(setenv(`$gitcmd rev-parse --git-dir`, dir=d)))
 end
 
+git() = gitcmd
 function git(d)
-    isempty(d) && return `git`
+    isempty(d) && return gitcmd
     work_tree = abspath(d)
     git_dir = joinpath(work_tree, dir(work_tree))
     normpath(work_tree, ".") == normpath(git_dir, ".") ? # is it a bare repo?
-    `git --git-dir=$work_tree` : `git --work-tree=$work_tree --git-dir=$git_dir`
+    `$gitcmd --git-dir=$work_tree` : `$gitcmd --work-tree=$work_tree --git-dir=$git_dir`
 end
 
 cmd(args::Cmd; dir="") = `$(git(dir)) $args`

test/gitutils.jl

@@ -14,7 +14,7 @@ function mktree(d::Dict)
     lstree = ""
     for (name, data) in d
         if isa(data, AbstractString)
-            sha1 = write_and_readchomp(data, `git hash-object -w --stdin`)
+            sha1 = write_and_readchomp(data, `$gitcmd hash-object -w --stdin`)
             lstree *= "100644 blob $sha1\t$name\n"
         elseif isa(data, Dict)
             sha1 = mktree(data)
@@ -25,21 +25,21 @@ function mktree(d::Dict)
             error("mktree: don't know what to do with $name => $data")
         end
     end
-    write_and_readchomp(lstree, `git mktree`)
+    write_and_readchomp(lstree, `$gitcmd mktree`)
 end
 
 function verify_tree(d::Dict, tree::AbstractString)
     # check that tree matches d
     seen = Set()
-    for line in eachline(`git ls-tree $tree`)
+    for line in eachline(`$gitcmd ls-tree $tree`)
         m = match(r"^(\d{6}) (\w+) ([0-9a-f]{40})\t(.*)$", line)
         @test m != nothing
         perm, kind, sha1, name = m.captures
         @test haskey(d,name)
         data = d[name]
         if isa(data, AbstractString)
             @test kind == "blob"
-            @test data == readstring(`git cat-file blob $sha1`)
+            @test data == readstring(`$gitcmd cat-file blob $sha1`)
         elseif isa(data, Dict)
             @test kind == "tree"
             verify_tree(data, sha1)
@@ -74,15 +74,15 @@ function verify_work(d::Dict)
         end
     end
     # check for anything that's not in d
-    for line in eachline(`ls -A`)
+    for line in readdir()
         name = chomp(line)
         @test name == ".git" || haskey(d,name)
     end
 end
 
 function git_verify(h::Dict, i::Dict, w::Dict)
     verify_tree(h, "HEAD")
-    verify_tree(i, readchomp(`git write-tree`))
+    verify_tree(i, readchomp(`$gitcmd write-tree`))
     verify_work(w)
 end
 
@@ -93,24 +93,24 @@ function git_setup(h::Dict, i::Dict, w::Dict, parents::AbstractString...)
     work  = mktree(w)
 
     # clear the repo
-    for line in eachline(`ls -A`)
+    for line in readdir()
         name = chomp(line)
         name == ".git" || rm(name, recursive=true)
     end
 
     # create the head commit
-    commit_tree = `git commit-tree $headt`
+    commit_tree = `$gitcmd commit-tree $headt`
     for parent in parents
         commit_tree = `$commit_tree -p $parent`
     end
     head = write_and_readchomp(headt, commit_tree)
-    run(`git reset -q --soft $head`)
+    run(`$gitcmd reset -q --soft $head`)
 
-    run(`git read-tree $work`)      # read work into the index
-    run(`git checkout-index -fa`)   # check the index out
-    run(`git read-tree $index`)     # setup the index
+    run(`$gitcmd read-tree $work`)      # read work into the index
+    run(`$gitcmd checkout-index -fa`)   # check the index out
+    run(`$gitcmd read-tree $index`)     # setup the index
 
     # verify that everything is as expected
     git_verify(h, i, w)
 end
-git_setup(h::Dict, i::Dict, w::Dict) = git_setup(h, i, w, readchomp(`git rev-parse HEAD`))
+git_setup(h::Dict, i::Dict, w::Dict) = git_setup(h, i, w, readchomp(`$gitcmd rev-parse HEAD`))

test/runtests.jl

@@ -12,10 +12,10 @@ dir = string("tmp.",randstring())
 mkdir(dir)
 @test isdir(dir)
 try cd(dir) do
-    run(`git init -q`)
-    run(`git config user.name "Julia Tester"`)
-    run(`git config user.email [email protected]`)
-    run(`git commit -q --allow-empty -m "initial empty commit"`)
+    run(`$gitcmd init -q`)
+    run(`$gitcmd config user.name "Julia Tester"`)
+    run(`$gitcmd config user.email [email protected]`)
+    run(`$gitcmd commit -q --allow-empty -m "initial empty commit"`)
     git_verify(Dict(), Dict(), Dict())
 
     # each path can have one of these content in each of head, index, work
@@ -26,7 +26,7 @@ try cd(dir) do
 
     contents = [nothing, "foo", "bar", Dict{Any,Any}("baz"=>"qux")]
     b = length(contents)
-    states = [ [ base(b,k,6) => contents[rem(div(k,b^p),b)+1] for k=0:(b^3)^2-1 ] for p=0:5 ]
+    states = [Dict([(base(b,k,6), contents[rem(div(k,b^p),b)+1]) for k=0:(b^3)^2-1]) for p=0:5]
 
     git_setup(states[1:3]...)
     try Git.transact() do