From 679927cb735399bf116302d2fdd4d08fdf7c2d5e Mon Sep 17 00:00:00 2001 From: Christopher Mancini Date: Thu, 19 Sep 2019 12:10:28 -0400 Subject: [PATCH 1/6] context improvements - added base_context - added syntax highlighting to README - removed obsolete groundcrew section - added `put` context param to override the source param - created makefile for ease of development / testing - bumped `jq` version in makefile --- Dockerfile | 2 +- Makefile | 18 ++++++ README.md | 110 ++++++++++++++------------------- bin/check | 2 +- bin/common.sh | 2 + bin/out | 8 ++- test/all.sh | 2 +- test/check/simple.sh | 2 +- test/out/default-target-url.sh | 2 +- test/out/put_context.sh | 89 ++++++++++++++++++++++++++ test/out/simple.sh | 2 +- test/out/target-url-path.sh | 2 +- test/out/wait-for-status.sh | 2 +- 13 files changed, 170 insertions(+), 73 deletions(-) create mode 100644 Makefile create mode 100644 test/out/put_context.sh diff --git a/Dockerfile b/Dockerfile index f818801..b58ff1e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ FROM alpine RUN apk --no-cache add curl ca-certificates gettext \ - && curl -Ls https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64 > /usr/bin/jq && chmod +x /usr/bin/jq + && curl -Ls https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 > /usr/bin/jq && chmod +x /usr/bin/jq ADD bin /opt/resource diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..2ba4669 --- /dev/null +++ b/Makefile @@ -0,0 +1,18 @@ +SHELL=/bin/bash + +root_mkfile := $(abspath $(lastword $(MAKEFILE_LIST))) +root_dir := $(dir $(root_mkfile)) + +.PHONY: resource-dev +resource-dev: + @docker run \ + -v $(root_dir):/resource \ + --rm -i -t dpb587/github-status-resource:master \ + /bin/ash + +.PHONY: test +test: + @docker run \ + -v $(root_dir):/resource \ + --rm -i -t dpb587/github-status-resource:master \ + /resource/test/all.sh diff --git a/README.md b/README.md index dd8d2d1..56a532b 100644 --- a/README.md +++ b/README.md @@ -2,25 +2,22 @@ A [Concourse](http://concourse.ci/) [resource](http://concourse.ci/resources.html) to interact with the [GitHub Status](https://developer.github.com/v3/repos/statuses/) type. - ## Configuration * **`repository`** - the owner and repository name, slash delimited (e.g. `dpb587/github-status-resource`) * **`access_token`** - GitHub API access token from a user with write access to the repository (minimum token scope of `repo:status`) * `branch` - the branch currently being monitored (default: `master`) + * `base_context` - prefix for the context label (default: `concourse-ci`) * `context` - a label to differentiate this status from the status of other systems (default: `default`) * `endpoint` - GitHub API endpoint (default: `https://api.github.com`) * `skip_ssl_verification` - Disable certificate validation for GitHub API calls (default: `false`) - ## Behavior - ### `check` Triggers when the status of the branch for the configured context has been updated. - ### `in` Lookup the state of a status. @@ -31,7 +28,6 @@ Lookup the state of a status. * `/target_url` - the target URL associated with the status * `/updated_at` - when the status was last updated - ### `out` Update the status of a commit. Optionally include a description and target URL which will be referenced from GitHub. @@ -43,78 +39,65 @@ Parameters: * `description` - a short description of the status * `description_path` - path to an input file whose data is the value of `description` * `target_url` - the target URL to associate with the status (default: concourse build link) - + * `context` - overrides the source context value (default: `""`) ## Example A typical use case is to update the status of a commit as it traverses your pipeline. The following example marks the commit as pending before unit tests start. Once unit tests finish, the status is updated to either success or failure depending on how the task completes. - --- - jobs: - - name: "unit-tests" - plan: - - get: "repo" - trigger: true - - put: "repo-status" # + - params: { state: "pending", commit: "repo" } # + - - task: "unit-tests" - file: "repo/ci/unit-tests/task.yml" - on_failure: - - put: "repo-status" # + - params: { state: "failure", commit: "repo" } # + - - put: "repo-status" # + - params: { state: "success", commit: "repo" } # + - resources: - - name: "repo" - type: "git" - source: - uri: {{repo_uri}} - branch: {{repo_branch}} - - name: "repo-status" # + - type: "github-status" # + - source: # + - repository: {{repo_github_path}} # + - access_token: {{repo_github_token}} # + +```yaml +jobs: + - name: "unit-tests" + plan: + - get: "repo" + trigger: true + - put: "repo-status" # + + params: { state: "pending", commit: "repo" } # + + - task: "unit-tests" + file: "repo/ci/unit-tests/task.yml" + on_failure: + - put: "repo-status" # + + params: { state: "failure", commit: "repo" } # + + - put: "repo-status" # + + params: { state: "success", commit: "repo" } # + +resources: + - name: "repo" + type: "git" + source: + uri: {{repo_uri}} + branch: {{repo_branch}} + - name: "repo-status" # + + type: "github-status" # + + source: # + + repository: {{repo_github_path}} # + + access_token: {{repo_github_token}} # + +``` When testing pull requests, use the PR ref as the `branch`. For example, if testing PR #12345 to your repository, your resource might look like... - name: "pr-status" - type: "github-status" - source: - repository: {{repo_github_path}} - access_token: {{repo_github_token}} - branch: "pull/12345/head" # + +```yaml +name: "pr-status" +type: "github-status" +source: + repository: {{repo_github_path}} + access_token: {{repo_github_token}} + branch: "pull/12345/head" # + +``` For another pipeline example, see [`ci/pipelines/main.yml`](ci/pipelines/main.yml) which operates against this repository. - ## Installation -This resource is not included with the standard Concourse release. Use one of the following methods to make this resource available to your pipelines. - - -### Deployment-wide - -To install on all Concourse workers, update your deployment manifest properties to include a new `groundcrew.resource_types` entry... - - properties: - groundcrew: - additional_resource_types: - - image: "docker:///dpb587/github-status-resource#master" # + - type: "github-status" # + - - -### Pipeline-specific - -To use on a single pipeline, update your pipeline to include a new `resource_types` entry... - - resource_types: - - name: "github-status" # + - type: "docker-image" # + - source: # + - repository: "dpb587/github-status-resource" # + - tag: "master" # + +This resource is not included with the standard Concourse release. Add it to your pipeline's `resource_types` definition. +```yaml +resource_types: + - name: "github-status" + type: "docker-image" + source: + repository: "dpb587/github-status-resource" + tag: "master" +``` ## References @@ -123,7 +106,6 @@ To use on a single pipeline, update your pipeline to include a new `resource_typ * [Enabling required status checks (help.github.com)](https://help.github.com/articles/enabling-required-status-checks/) * [Personal Access Tokens (github.com)](https://github.com/settings/tokens) - ## License [MIT License](./LICENSE) diff --git a/bin/check b/bin/check index 1b39db7..7c57c8d 100755 --- a/bin/check +++ b/bin/check @@ -7,6 +7,6 @@ load_source curlgh "$source_endpoint/repos/$source_repository/commits/$source_branch/status" \ | jq -c \ - --arg context "$source_context" \ + --arg context "$source_base_context/$source_context" \ '.sha as $commit | .statuses | map(select( $context == .context )) | map({ "commit": $commit, "status": ( .id | tostring ) })' \ >&3 diff --git a/bin/common.sh b/bin/common.sh index 45b30b0..32f0ee3 100644 --- a/bin/common.sh +++ b/bin/common.sh @@ -24,6 +24,7 @@ load_source () { "source_repository": .source.repository, "source_access_token": .source.access_token, "source_branch": ( .source.branch // "master" ), + "source_base_context": ( .source.base_context // "concourse-ci" ), "source_context": ( .source.context // "default" ), "source_endpoint": ( .source.endpoint // "https://api.github.com" ), "skip_ssl_verification": ( .source.skip_ssl_verification // "false" ) @@ -40,6 +41,7 @@ buildtpl () { BUILD_NAME="${BUILD_NAME:-}" \ BUILD_JOB_NAME="${BUILD_JOB_NAME:-}" \ BUILD_PIPELINE_NAME="${BUILD_PIPELINE_NAME:-}" \ + BUILD_TEAM_NAME="${BUILD_TEAM_NAME:-}" \ ATC_EXTERNAL_URL="${ATC_EXTERNAL_URL:-}" \ $envsubst } diff --git a/bin/out b/bin/out index a3107bf..cb4d2f8 100755 --- a/bin/out +++ b/bin/out @@ -8,6 +8,7 @@ load_source eval $( jq -r '{ "params_commit": .params.commit, "params_state": .params.state, + "params_context": ( .params.context // "" ), "params_description": ( .params.description // "" ), "params_description_path": ( .params.description_path // "" ), "params_target_url": ( .params.target_url // ""), @@ -73,11 +74,16 @@ fi # execute # +cntxt="$source_context" +if [[ -n "$params_context" ]] ; then + cntxt="$params_context" +fi + jq -c -n \ --arg state "$params_state" \ --arg target_url "$( echo "$target_url" | buildtpl )" \ --arg description "$( cat $description_path )" \ - --arg context "$source_context" \ + --arg context "$(echo "$source_base_context/$cntxt" | buildtpl)" \ '{ "context": $context, "description": $description, diff --git a/test/all.sh b/test/all.sh index df3ac5a..53607a5 100755 --- a/test/all.sh +++ b/test/all.sh @@ -7,7 +7,7 @@ cd $dir export TMPDIR="${TMPDIR:-/tmp}" -for test in $( find test -type f -perm +111 -print | grep -v 'test/all.sh' ) ; do +for test in $( find test -type f -print | grep -v 'test/all.sh' ) ; do echo "==> $test" $test result=$? diff --git a/test/check/simple.sh b/test/check/simple.sh index b15423c..b6c641e 100755 --- a/test/check/simple.sh +++ b/test/check/simple.sh @@ -30,7 +30,7 @@ HTTP/1.0 200 OK "description": "Testing has completed successfully", "id": 2, "url": "https://api.github.com/repos/octocat/Hello-World/statuses/2", - "context": "test-context" + "context": "concourse-ci/test-context" } ] } diff --git a/test/out/default-target-url.sh b/test/out/default-target-url.sh index d236314..aa94767 100755 --- a/test/out/default-target-url.sh +++ b/test/out/default-target-url.sh @@ -62,7 +62,7 @@ if ! grep -q '^POST /repos/dpb587/test-repo/statuses/a1b2c3d4e5 ' $TMPDIR/http.r exit 1 fi -if ! grep -q '^{"context":"test-context","description":"test-description","state":"success","target_url":"http://localhost/builds/123"}$' $TMPDIR/http.req-$$ ; then +if ! grep -q '^{"context":"concourse-ci/test-context","description":"test-description","state":"success","target_url":"http://localhost/builds/123"}$' $TMPDIR/http.req-$$ ; then echo "FAILURE: Unexpected request body" cat $TMPDIR/http.req-$$ exit 1 diff --git a/test/out/put_context.sh b/test/out/put_context.sh new file mode 100644 index 0000000..1bf05d2 --- /dev/null +++ b/test/out/put_context.sh @@ -0,0 +1,89 @@ +#!/bin/sh + +set -eu + +DIR=$( dirname "$0" )/../.. + +echo 'a1b2c3d4e5' > $TMPDIR/commit + +cat < $TMPDIR/http.req-$$ & +HTTP/1.0 200 OK + +{ + "created_at": "2012-07-20T01:19:13Z", + "updated_at": "2012-07-20T02:19:13Z", + "state": "success", + "target_url": "https://ci.example.com/1000/output", + "description": "Build has completed successfully", + "id": 1, + "url": "https://api.github.com/repos/octocat/Hello-World/statuses/1", + "context": "continuous-integration/jenkins", + "creator": { + "login": "octocat", + "id": 1, + "avatar_url": "https://github.com/images/error/octocat_happy.gif", + "gravatar_id": "", + "url": "https://api.github.com/users/octocat", + "html_url": "https://github.com/octocat", + "followers_url": "https://api.github.com/users/octocat/followers", + "following_url": "https://api.github.com/users/octocat/following{/other_user}", + "gists_url": "https://api.github.com/users/octocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/octocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/octocat/subscriptions", + "organizations_url": "https://api.github.com/users/octocat/orgs", + "repos_url": "https://api.github.com/users/octocat/repos", + "events_url": "https://api.github.com/users/octocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/octocat/received_events", + "type": "User", + "site_admin": false + } +} +EOF + +BUILD_ID=123 BUILD_JOB_NAME=myjob $DIR/bin/out > $TMPDIR/resource-$$ < Date: Fri, 21 Feb 2020 17:21:32 -0500 Subject: [PATCH 2/6] Temporarily bump the limit to 100 to avoid following cursors --- bin/in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/in b/bin/in index 9b6710b..b0d5e56 100755 --- a/bin/in +++ b/bin/in @@ -16,7 +16,7 @@ eval $( jq -r '{ # lookup # -curlgh "$source_endpoint/repos/$source_repository/commits/$version_commit/status" \ +curlgh "$source_endpoint/repos/$source_repository/commits/$version_commit/status?per_page=100" \ | jq -c \ --arg status "$version_status" \ '{ From 0efae2cea64b16a7e7f46d0b7303b1f4c63f9370 Mon Sep 17 00:00:00 2001 From: Mark Stansberry Date: Thu, 27 Feb 2020 15:17:49 -0500 Subject: [PATCH 3/6] Add multi-page status results support --- bin/common.sh | 17 +++++++++++++++++ bin/in | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/bin/common.sh b/bin/common.sh index 32f0ee3..bd5e9b5 100644 --- a/bin/common.sh +++ b/bin/common.sh @@ -54,3 +54,20 @@ curlgh () { fi curl $skip_verify_arg -s -H "Authorization: token $source_access_token" $@ } + +curlgh_all_pages_status () { + results='' + page=1 + present='true' + while [ "$present" = "true" ]; do + current_results=$(curlgh "$@?page=$page" | jq .) + if [ "$(echo $current_results | jq .statuses)" != "[]" ]; then + results=$(jq -s '.[0] as $o1 | .[1] as $o2 | ($o1 + $o2) | .statuses = ($o1.statuses + $o2.statuses)' <(echo $results) <(echo $current_results)) + page=$(($page+1)) + else + present='false' + fi + done + + echo "$results" +} diff --git a/bin/in b/bin/in index b0d5e56..ae413f1 100755 --- a/bin/in +++ b/bin/in @@ -16,7 +16,7 @@ eval $( jq -r '{ # lookup # -curlgh "$source_endpoint/repos/$source_repository/commits/$version_commit/status?per_page=100" \ +curlgh_all_pages_status "$source_endpoint/repos/$source_repository/commits/$version_commit/status" \ | jq -c \ --arg status "$version_status" \ '{ From 23bf3723bd2c28dc12400afcd05e28eacdb380cb Mon Sep 17 00:00:00 2001 From: Dennis Wong Date: Thu, 27 Feb 2020 22:41:48 -0500 Subject: [PATCH 4/6] Identity function from jq should not change the output --- bin/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/common.sh b/bin/common.sh index bd5e9b5..c65ea17 100644 --- a/bin/common.sh +++ b/bin/common.sh @@ -60,7 +60,7 @@ curlgh_all_pages_status () { page=1 present='true' while [ "$present" = "true" ]; do - current_results=$(curlgh "$@?page=$page" | jq .) + current_results=$(curlgh "$@?page=$page") if [ "$(echo $current_results | jq .statuses)" != "[]" ]; then results=$(jq -s '.[0] as $o1 | .[1] as $o2 | ($o1 + $o2) | .statuses = ($o1.statuses + $o2.statuses)' <(echo $results) <(echo $current_results)) page=$(($page+1)) From 320683ef5688025eaa42797329cc85d1eefb1b2c Mon Sep 17 00:00:00 2001 From: Dennis Wong Date: Thu, 27 Feb 2020 23:05:04 -0500 Subject: [PATCH 5/6] Alternative method for combining statuses array across multiple queries --- bin/common.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/bin/common.sh b/bin/common.sh index c65ea17..c084b4c 100644 --- a/bin/common.sh +++ b/bin/common.sh @@ -61,8 +61,12 @@ curlgh_all_pages_status () { present='true' while [ "$present" = "true" ]; do current_results=$(curlgh "$@?page=$page") - if [ "$(echo $current_results | jq .statuses)" != "[]" ]; then - results=$(jq -s '.[0] as $o1 | .[1] as $o2 | ($o1 + $o2) | .statuses = ($o1.statuses + $o2.statuses)' <(echo $results) <(echo $current_results)) + + # If key "statuses" is not present, stop iterating loop + statuses=$(echo $current_results | jq -c '.statuses // []') + if [ "$statuses" != "[]" ]; then + # Identify "statuses" array in `current_results` and append it to "statuses" array in `results` + results=$(echo "$results" | jq --argjson s "$statuses" '.statuses += $s') page=$(($page+1)) else present='false' From 507e2b1e73cd2414d48f5db3cdff1c83f318af67 Mon Sep 17 00:00:00 2001 From: Dennis Wong Date: Thu, 27 Feb 2020 23:08:34 -0500 Subject: [PATCH 6/6] Save inital response so there is an array to append to in future iterations of loop --- bin/common.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/bin/common.sh b/bin/common.sh index c084b4c..fb98a35 100644 --- a/bin/common.sh +++ b/bin/common.sh @@ -57,17 +57,25 @@ curlgh () { curlgh_all_pages_status () { results='' - page=1 + page=0 present='true' while [ "$present" = "true" ]; do + page=$(($page+1)) current_results=$(curlgh "$@?page=$page") + # Save the first query as the return value, in case the response + # body is not expected, the response body can still be returned + # and behave like curlgh would in the non-happy path. + if [ -z "$results" ]; then + results="$current_results" + continue + fi + # If key "statuses" is not present, stop iterating loop statuses=$(echo $current_results | jq -c '.statuses // []') if [ "$statuses" != "[]" ]; then # Identify "statuses" array in `current_results` and append it to "statuses" array in `results` results=$(echo "$results" | jq --argjson s "$statuses" '.statuses += $s') - page=$(($page+1)) else present='false' fi