Initial commit

Signed-off-by: Christopher Maier <[email protected]>

Author: christophermaier

.buildkite/pipeline.verify.yml

@@ -0,0 +1,20 @@
+---
+env:
+  PANTS_CONFIG_FILES: "['pants.toml', 'pants.ci.toml']"
+
+steps:
+  - label: ":lint-roller::bash: Lint Shell"
+    command:
+      - make lint-shell
+
+  - label: ":lint-roller::buildkite: Lint Plugin"
+    command:
+      - make lint-plugin
+
+  - label: ":bash: Unit Tests"
+    command:
+      - make test-shell
+
+  - label: ":buildkite: Plugin Tests"
+    command:
+      - make test-plugin

.gitignore

@@ -0,0 +1,2 @@
+.pants.d
+.pids

Makefile

@@ -0,0 +1,44 @@
+COMPOSE_USER=$(shell id -u):$(shell id -g)
+
+# Linting
+########################################################################
+
+.PHONY: lint
+lint: lint-plugin lint-shell
+
+.PHONY: lint-plugin
+lint-plugin:
+	docker-compose run --rm plugin-linter
+
+.PHONY: lint-shell
+lint-shell:
+	./pants lint ::
+
+# Formatting
+########################################################################
+
+.PHONY: format
+format: format-shell
+
+.PHONY: format-shell
+format-shell:
+	./pants fmt ::
+
+# Testing
+########################################################################
+
+.PHONY: test
+test: test-plugin test-shell
+
+.PHONY: test-plugin
+test-plugin:
+	docker-compose run --rm plugin-tester
+
+.PHONY: test-shell
+test-shell:
+	./pants test ::
+
+########################################################################
+
+.PHONY: all
+all: format lint test

README.md

@@ -0,0 +1,113 @@
+# Grapl Artifacts Buildkite Plugin
+
+Encapsulates logic used in Grapl's release pipelines for
+record-keeping around the versions of any artifacts that are
+generated.
+
+This is highly specific to how we run our CI/CD pipelines at Grapl; it
+is not intended to be broadly generalizable.
+
+## Example
+
+```yml
+steps:
+  - label: ":gear::packer: Convert Packer Manifests"
+    plugins:
+      - grapl-security/grapl-artifacts#v0.1.0:
+          action: convert_packer_manifests
+```
+
+```yml
+steps:
+  - label: ":knot: Merge Artifact Files"
+    plugins:
+      - grapl-security/grapl-artifacts#v0.1.0:
+          action: merge_artifact_files
+```
+
+## Configuration
+
+### action (required, string)
+
+The name of the plugin action to run; currently supports the following:
+- `convert_packer_manifests`
+- `merge_artifact_files`
+
+#### `convert_packer_manifests`
+
+Extracts the AMI IDs from one or more Packer [manifest
+files](https://www.packer.io/docs/post-processors/manifest) into a
+simplified JSON form that we can consume more easily in downstream pipeline
+jobs.
+
+In particular, it will take a manifest file like this (call it
+`my-ami.packer-manifest.json`):
+
+```json
+{
+  "builds": [
+    {
+      "name": "amazon-linux-2-amd64-ami",
+      "builder_type": "amazon-ebs",
+      "build_time": 1626818948,
+      "files": null,
+      "artifact_id": "us-east-1:ami-aaaaaaaaaaaaaaaaa,us-east-2:ami-bbbbbbbbbbbbbbbbb,us-west-1:ami-ccccccccccccccccc,us-west-2:ami-ddddddddddddddddd",
+      "packer_run_uuid": "f101f1dd-5fdb-bc9e-b0f0-ee49267bf567",
+      "custom_data": null
+    }
+  ],
+  "last_run_uuid": "f101f1dd-5fdb-bc9e-b0f0-ee49267bf567"
+}
+```
+
+and turn it into `my-ami-${BUILDKITE_JOB_ID}.grapl-artifacts.json`:
+
+```json
+{
+  "my-ami.us-east-1": "ami-aaaaaaaaaaaaaaaaa",
+  "my-ami.us-east-2": "ami-bbbbbbbbbbbbbbbbb",
+  "my-ami.us-west-1": "ami-ccccccccccccccccc",
+  "my-ami.us-west-2": "ami-ddddddddddddddddd",
+}
+```
+
+The manifest file _must_ have the extension `.packer-manifest.json`;
+the resulting file will be named according to whatever the basename of
+the manifest file is.
+
+To generate manifests compatible with this plugin, use HCL like this
+in your Packer template:
+
+```hcl
+build {
+  # all your sources and provisioners go here...
+
+  post-processor "manifest" {
+    output = "my-ami.packer-manifest.json"
+  }
+}
+```
+
+All files previously uploaded using `buildkite-agent artifact upload`
+in the current pipeline and ending in `.packer-manifest.json` will be
+processed in this way; one `.grapl-artifacts.json` file per Packer
+manifest will be generated and uploaded for access in downstream jobs.
+
+#### `merge_artifact_files`
+
+Takes all `*.grapl-artifacts.json` files that have been previously
+uploaded in the current pipeline and merges them all into a single
+file for subsequent processing.
+
+Each file is assumed to contain a single, flat JSON object, with
+mutually disjoint keys.
+
+The final merged file is called `all_artifacts.json` and is uploaded
+for access in downstream jobs.
+
+## Building
+
+Requires `make`, `docker`, and `docker-compose`.
+
+`make all` will run all formatting, linting, and testing, though
+finer-grained targets are available.

bin/BUILD

@@ -0,0 +1 @@
+shell_library()

bin/convert_packer_manifests.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# shellcheck source-path=SCRIPTDIR
+source "$(dirname "${BASH_SOURCE[0]}")/../lib/artifacts.sh"
+# shellcheck source-path=SCRIPTDIR
+source "$(dirname "${BASH_SOURCE[0]}")/../lib/packer_artifacts.sh"
+
+# TODO: Could just use the artifacts plugin for this
+download_all_packer_manifests() {
+    buildkite-agent artifact download "*.${PACKER_MANIFEST_EXTENSION}" .
+}
+
+upload_all_grapl_artifacts() {
+    buildkite-agent artifact upload "*.${ARTIFACTS_FILE_EXTENSION}"
+}
+
+download_all_packer_manifests
+
+for packer_manifest_file in *."${PACKER_MANIFEST_EXTENSION}"; do
+    echo -e "--- Extracting AMI information from ${packer_manifest_file}"
+    slug="$(name_from_manifest_file "${packer_manifest_file}")"
+    extract_ami_information "${packer_manifest_file}" > "$(artifacts_file_for "${slug}")"
+done
+
+# To keep things simple, we _could_ use the artifacts plugin, or
+# artifacts_path key instead of this.
+#
+# Here, at least, we're keeping all the extension knowledge "in house"
+upload_all_grapl_artifacts

bin/merge_artifact_files.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+# Given a series of names of artifact files (i.e., files containing
+# flat JSON objects mapping artifact names to versions), retrieve them
+# from the Buildkite artifact storage facility, merge them all into a
+# single new artifact file, and then upload that for subsequent
+# processing.
+#
+# This will allow this logic to be reused across multiple pipelines,
+# regardless of how many artifact files they may generate.
+
+set -euo pipefail
+
+# shellcheck source-path=SCRIPTDIR
+source "$(dirname "${BASH_SOURCE[0]}")/../lib/artifacts.sh"
+
+echo "--- :buildkite: Downloading all artifact files"
+buildkite-agent artifact download "*.${ARTIFACTS_FILE_EXTENSION}" .
+
+merge_artifact_files > "${ALL_ARTIFACTS_JSON_FILE}"
+
+echo "--- :buildkite: Uploading ${ALL_ARTIFACTS_JSON_FILE} file"
+buildkite-agent artifact upload "${ALL_ARTIFACTS_JSON_FILE}"
+# This artifact then gets picked up by the "Create new release candidate" step in Buildkite

docker-compose.yml

@@ -0,0 +1,31 @@
+version: "3.8"
+
+x-common-variables:
+  read-only-workdir: &read-only-workdir
+    type: bind
+    source: .
+    target: /workdir
+    read_only: true
+  read-write-workdir: &read-write-workdir
+    type: bind
+    source: .
+    target: /workdir
+    read_only: false
+  read-only-plugin: &read-only-plugin
+    # Buildkite containers assume you mount into /plugin
+    type: bind
+    source: .
+    target: /plugin
+    read_only: true
+
+services:
+  plugin-tester:
+    image: buildkite/plugin-tester:latest # the only available tag
+    volumes:
+      - *read-only-plugin
+
+  plugin-linter:
+    image: buildkite/plugin-linter:latest # the only available tag
+    command: ["--id", "grapl-security/grapl-artifacts"]
+    volumes:
+      - *read-only-plugin

fixtures/BUILD

@@ -0,0 +1,3 @@
+resources(
+  sources=["*.json"]
+)

fixtures/multiple-runs-single-region.grapl-artifacts.json

@@ -0,0 +1,3 @@
+{
+    "multiple-runs-single-region.us-east-1": "ami-aaaaaaaaaaaaaaaaa"
+}