Add support for pushing metrics to a remote server (#3)

* Bump to Go 1.15
* Add support for pushing metrics to a remote server

Author: James Alseth

.github/test/policy/always_warn.rego

@@ -1,3 +1,10 @@
 package always_warn
 
-warn["a warning! you should probably fix this"]
+warn[msg] {
+    msg := {
+        "msg": "a warning! you should probably fix this",
+        "details": {"policyID": "P0000"}
+    }
+
+    true
+}

.github/workflows/pull_request.yaml

@@ -9,7 +9,7 @@ jobs:
       - name: setup go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.14.x
+          go-version: 1.15.x
 
       - name: checkout
         uses: actions/checkout@v2
@@ -20,9 +20,6 @@ jobs:
       - name: test build
         run: go build -o build/action-conftest
 
-      - name: test docker build
-        run: docker build .
-
   test-run-action:
     name: Test the Action
     needs: go-tests

.gitignore

@@ -1,2 +1,3 @@
 build
 gcs.json
+*.env

Dockerfile

@@ -1,10 +1,7 @@
 FROM instrumenta/conftest:v0.20.0 as conftest
 
-FROM golang:1.14-alpine as builder
-COPY main.go .
-RUN go build -o /entrypoint
-
-FROM alpine:3
+FROM golang:1.15-alpine as builder
 COPY --from=conftest /conftest /usr/local/bin/conftest
-COPY --from=builder /entrypoint /usr/local/bin/entrypoint
-CMD [ "/usr/local/bin/entrypoint" ]
+COPY main.go .
+RUN go build -o /entrypoint main.go
+CMD [ "/entrypoint" ]

README.md

@@ -1,24 +1,30 @@
 # action-conftest
 
-A GitHub Action for easily using [conftest](https://github.com/open-policy-agent/conftest) in your CI. It allows for pulling policies from another source and can surface the violations and warnings into the comments of the pull request.
+A GitHub Action for easily using [conftest](https://github.com/open-policy-agent/conftest) in your CI. It allows for pulling policies from another source and can surface the violations and warnings into the comments of the pull request. Additionally, the action can submit metrics for the results of the tests to a remote server for analysis of the rate of failures and warnings, which is useful when deploying new policies.
 
 **NOTE:** This action only supports pull secrets for S3, GCS, and HTTP remotes. If you are pulling from an OCI registry, it is assumed that you have already authenticated using `docker login` in a previous step in the GitHub Actions `job` and you should not supply a `pull-secret` argument.
 
 ## Options
 
-| Option         | Description                                    | Default | Required               |
-|----------------|------------------------------------------------|---------|------------------------|
-| files          | Files for Conftest to test (space delimited)   |         | yes                    |
-| policy         | Where to find the policy folder or file        | policy  | no                     |
-| data           | Files or folder with supplemental test data    |         | no                     |
-| all-namespaces | Whether to use all namespaces in testing       | true    | no                     |
-| combine        | Whether to combine input files                 | false   | no                     |
-| pull-url       | URL to pull policies from                      |         | no                     |
-| pull-secret    | Secret that allows the policies to be pulled   |         | no                     |
-| add-comment    | Whether or not to add a comment to the PR      | true    | no                     |
-| docs-url       | Documentation URL to link to in the PR comment |         | no                     |
-| gh-token       | Token to authorize adding the PR comment       |         | if add-comment is true |
-| gh-comment-url | The URL of the comments for the PR             |         | if add-comment is true |
+| Option          | Description                                                     | Default  | Required               |
+|-----------------|-----------------------------------------------------------------|----------|------------------------|
+| files           | Files and/or folders for Conftest to test (space delimited)     |          | yes                    |
+| policy          | Where to find the policy folder or file                         | policy   | no                     |
+| data            | Files or folder with supplemental test data                     |          | no                     |
+| all-namespaces  | Whether to use all namespaces in testing                        | true     | no                     |
+| combine         | Whether to combine input files                                  | false    | no                     |
+| pull-url        | URL to pull policies from                                       |          | no                     |
+| pull-secret     | Secret that allows the policies to be pulled                    |          | no                     |
+| add-comment     | Whether or not to add a comment to the PR                       | true     | no                     |
+| docs-url        | Documentation URL to link to in the PR comment                  |          | no                     |
+| no-fail         | Always returns an exit code of 0 (no error)                     | false    | no                     |
+| gh-token        | Token to authorize adding the PR comment                        |          | if add-comment is true |
+| gh-comment-url  | URL of the comments for the PR                                  |          | if add-comment is true |
+| metrics-url     | URL to POST the results to for metrics                          |          | no                     |
+| metrics-source  | Unique ID for the source of the metrics (usually the repo name) |          | if metrics-url is set  |
+| metrics-details | Whether to include the full test results in the metrics         | false    | no
+| metrics-token   | Bearer token for submitting the metrics                         |          | no                     |
+| policy-id-key   | Name of the key in the details object that stores the policy ID | policyID | if metrics-url is set  |
 
 ## Example Usage
 
@@ -36,7 +42,7 @@ jobs:
       - name: checkout
         uses: actions/checkout@v2
       - name: conftest
-        uses: YubicoLabs/action-conftest@v1
+        uses: YubicoLabs/action-conftest@v2
         with:
           files: some_deployment.yaml another_resource.yaml
           gh-token: ${{ secrets.GITHUB_TOKEN }}
@@ -57,11 +63,32 @@ jobs:
       - name: checkout
         uses: actions/checkout@v2
       - name: conftest
-        uses: YubicoLabs/action-conftest@v1
+        uses: YubicoLabs/action-conftest@v2
         with:
           files: some_deployment.yaml another_resource.yaml
           pull-url: gcs::https://www.googleapis.com/storage/v1/bucket_name/policy
           pull-secret: ${{ secrets.POLICY_PULL_SECRET }}
           gh-token: ${{ secrets.GITHUB_TOKEN }}
           gh-comment-url: ${{ github.event.pull_request.comments_url }}
 ```
+
+### Submitting metrics to a remote server
+
+```yaml
+name: conftest-push-metrics
+on: [pull_request]
+jobs:
+  conftest:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+      - name: conftest
+        uses: YubicoLabs/action-conftest@v2
+        with:
+          files: some_deployment.yaml another_resource.yaml
+          gh-token: ${{ secrets.GITHUB_TOKEN }}
+          gh-comment-url: ${{ github.event.pull_request.comments_url }}
+          metrics-url: https://your.com/metrics/endpoints/conftest
+          metrics-source: your-repo-name
+```

action.yml

@@ -5,7 +5,7 @@ branding:
   color: "green"
 inputs: 
   files:
-    description: "Files for Conftest to test (space delimited)"
+    description: "Files and/or folders for Conftest to test (space delimited)"
     required: true
   policy:
     description: "Where to find the policy folder or file"
@@ -25,11 +25,24 @@ inputs:
     description: "Whether or not to add a comment to the PR"
     default: true
   docs-url:
-    description: "The URL where users can find out more about the policies"
+    description: "URL where users can find out more about the policies"
+  no-fail:
+    description: "Always returns an exit code of 0 (no error)"
   gh-token:
-    description: "The token that allows us to post a comment in the PR"
+    description: "Token that allows us to post a comment in the PR"
   gh-comment-url:
-    description: "The URL of the comments for the PR"
+    description: "URL of the comments for the PR"
+  metrics-url:
+    description: "URL to POST the results to for metrics"
+  metrics-source:
+    description: "Unique identifier for the source of the submission"
+  metrics-details:
+    description: "Whether to include the full test results in the metrics"
+  metrics-token:
+    description: "Bearer token for submitting metrics"
+  policy-id-key:
+    description: "Name of the key in the details object that stores the policy ID"
+    default: "policyID"
 runs:
   using: 'docker'
   image: 'Dockerfile'
@@ -43,5 +56,11 @@ runs:
     PULL_SECRET: ${{ inputs.pull-secret }}
     ADD_COMMENT: ${{ inputs.add-comment }}
     DOCS_URL: ${{ inputs.docs-url }}
+    NO_FAIL: ${{ inputs.no-fail }}
     GITHUB_TOKEN: ${{ inputs.gh-token }}
     GITHUB_COMMENT_URL: ${{ inputs.gh-comment-url }}
+    METRICS_URL: ${{ inputs.metrics-url }}
+    METRICS_SOURCE: ${{ inputs.metrics-source }}
+    METRICS_DETAILS: ${{ inputs.metrics-details }}
+    METRICS_TOKEN: ${{ inputs.metrics-token }}
+    POLICY_ID_KEY: ${{ inputs.policy-id-key }}