feat(in): use a custom in script to clone the required commit only

cut release 2.0.0-beta1

Author: JohannesRudolph

CHANGELOG.md

@@ -1,5 +1,17 @@
 # Release Notes
 
+## 2.0.0-beta1
+
+- Use a custom `in` script instead of the script provided by the embedded [git-resource](https://github.com/concourse/git-resource).
+  This allows us to efficiently fetch shallow clones at exactly a specified
+  revision without incurring needless `git fetch --deepen` roundtrips.
+
+Note: this release can break configuration for the `get` step in your pipelines as
+it removes configuration options inherited from `git-resource` that don't make
+sense in the context of gate-resource anymore. This helps to achieve better `get` performance and is critical for large gate repository.
+
+Please review the [README](./README.md) for updated configuration options.
+
 ## 1.1.1
 
 - Patch an issue with shallow-clones not fetching at the correct depth in git-resource.

Dockerfile

@@ -2,9 +2,6 @@ FROM concourse/git-resource:1.7.0 as resource
 
 RUN mv /opt/resource /opt/git-resource
 
-# patch the git-resource deepen script with our fixes, see https://github.com/concourse/git-resource/pull/316
-COPY git-resource/deepen_shallow_clone_until_ref_is_found_then_check_out /opt/git-resource/deepen_shallow_clone_until_ref_is_found_then_check_out
-
 ADD assets/ /opt/resource/
 RUN chmod +x /opt/resource/*
 

Makefile

@@ -1,7 +1,7 @@
 user=meshcloud
 name=gate-resource
 image=$(user)/$(name)
-tag=1.1.1
+tag=2.0.0-beta1
 
 docker=docker
 dockerfile = Dockerfile

README.md

@@ -56,9 +56,9 @@ Since concourse detects that the `none` version already exists after the first t
 
 ## Source Configuration
 
-* `git`: *Required.* Configuration of the repository. See [git-resource](https://github.com/concourse/git-resource) for options.
-  * **note:** the `paths` and `ignore_path` parameters are not supported
-
+* `git`: *Required.* Configuration of the repository. Supports the following subset of [git-resource](https://github.com/concourse/git-resource) configuration options (review the link for descriptions)
+  * *Required*: `uri`, `branch`, `private_key`
+  * *Optional*: `git_config`, `short_ref_format`,
 * `gate`: *Optional.* The gate to track.
 
 ## Behavior
@@ -77,6 +77,9 @@ Outputs 2 files:
 * `metadata`: Contains the contents of whatever was in your gate item. This is
   useful for environment configuration settings or documenting approval workflows.
 
+> note: the git repository cloned during `in` is a shallow clone and does not track an upstream branch.
+> This is not a problem when using the `out` step to create or update gates.
+
 ### `out`: Pass an item through a gate
 
 Performs one of the following actions to change the state of a gate.

assets/git-in

@@ -0,0 +1,95 @@
+#!/bin/bash
+# vim: set ft=sh
+
+set -e
+
+exec 3>&1 # make stdout available as fd 3 for the result
+exec 1>&2 # redirect all output to stderr for logging
+
+source /opt/git-resource/common.sh
+
+destination=$1
+
+if [ -z "$destination" ]; then
+  echo "usage: $0 <path/to/destination>" >&2
+  exit 1
+fi
+
+# for jq
+PATH=/usr/local/bin:$PATH
+
+bin_dir="${0%/*}"
+if [ "${bin_dir#/}" == "$bin_dir" ]; then
+  bin_dir="$PWD/$bin_dir"
+fi
+
+payload=$(mktemp $TMPDIR/git-resource-request.XXXXXX)
+
+cat > $payload <&0
+
+load_pubkey $payload
+configure_https_tunnel $payload
+configure_git_ssl_verification $payload
+configure_credentials $payload
+
+uri=$(jq -r '.source.uri // ""' < $payload)
+git_config_payload=$(jq -r '.source.git_config // []' < $payload)
+ref=$(jq -r '.version.ref // "HEAD"' < $payload)
+depth=$(jq -r '(.params.depth // 0)' < $payload)
+short_ref_format=$(jq -r '(.params.short_ref_format // "%s")' < $payload)
+
+configure_git_global "${git_config_payload}"
+
+if [ -z "$uri" ]; then
+  echo "invalid payload (missing uri):" >&2
+  cat $payload >&2
+  exit 1
+fi
+
+depthflag=""
+if test "$depth" -gt 0 2> /dev/null; then
+  depthflag="--depth $depth"
+fi
+
+git init $destination
+cd $destination
+
+git remote add origin $uri
+git fetch origin "$ref" $depthflag
+
+# this will set  master to the current commit 
+git reset --hard FETCH_HEAD
+
+git log -1 --oneline
+git clean --force --force -d
+
+if [ "$ref" == "HEAD" ]; then
+  return_ref=$(git rev-parse HEAD)
+else
+  return_ref=$ref
+fi
+
+# Store committer email in .git/committer. Can be used to send email to last committer on failed build
+# Using https://github.com/mdomke/concourse-email-resource for example
+git --no-pager log -1 --pretty=format:"%ae" > .git/committer
+
+# Store git-resource returned version ref .git/ref. Useful to know concourse
+# pulled ref in following tasks and resources.
+echo "${return_ref}" > .git/ref
+
+# Store short ref with templating. Useful to build Docker images with
+# a custom tag
+echo "${return_ref}" | cut -c1-7 | awk "{ printf \"${short_ref_format}\", \$1 }" > .git/short_ref
+
+# Store commit message in .git/commit_message. Can be used to inform about
+# the content of a successfull build.
+# Using https://github.com/cloudfoundry-community/slack-notification-resource
+# for example
+git log -1 --format=format:%B > .git/commit_message
+
+metadata=$(git_metadata)
+
+jq -n "{
+  version: {ref: $(echo $return_ref | jq -R .)},
+  metadata: $metadata
+}" >&3

assets/in

@@ -35,14 +35,12 @@ else
       "source": '$git_source',
       "version": '$version',
       "params": {
-        "disable_git_lfs": true,
-        "submodules": "none",
         "depth": 2
       }
     }' | jq -r)
 
   # forward to git-resource to let it fetch the repository
-  echo "$git_payload" | /opt/git-resource/in $destination
+  echo "$git_payload" | /opt/resource/git-in $destination
 
   cd $destination
 

assets/out

@@ -30,7 +30,6 @@ jq -n "{
 }" > $git_source_payload
 
 load_pubkey $git_source_payload
-load_git_crypt_key $git_source_payload
 configure_https_tunnel $git_source_payload
 configure_git_ssl_verification $git_source_payload
 configure_credentials $git_source_payload
@@ -116,7 +115,8 @@ try_autoclose() {
   echo "$dashed_line"
 }
 
-# clone the repository. by default we only need a single branch and a single 
+# clone the repository at its current latest commit - we want to make more commits on top.
+# by default we only need to fetch a single branch and the latest commit
 repository=$(mktemp -d "$TMPDIR/gate-resource-repo.XXXXXX")
 git clone $uri --branch $branch $repository --depth=1
 

git-resource/deepen_shallow_clone_until_ref_is_found_then_check_out

@@ -148,7 +148,8 @@ get_gate_at_ref() {
   jq -n "{
     source: {
       git: {
-        uri: $(echo file://$uriPath | jq -R .)
+        uri: $(echo file://$uriPath | jq -R .),
+        branch: \"master\"
       },
       gate: $(echo $gate | jq -R .)
     },

test/helpers.sh

@@ -12,6 +12,10 @@ it_can_get_from_regular_version() {
 
   local gate_ref=$(make_commit_to_file $repo "$gate/$item")
 
+  echo "xxxx"
+  git -C "$repo" branch -a 
+  echo "xxxx"
+  
   result=$(get_gate_at_ref "$repo" "$gate_ref" "$gate" "$dest")
   echo "$result" | jq -e '
     .version == { "ref": "'$gate_ref'" }