Add buildkite steps and github workflows (#5)

Author: abeatrix

.buildkite/check-image-names.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -euxo pipefail
+
+cd "$(dirname "${BASH_SOURCE[0]}")/.."
+ROOT="$(pwd)"
+
+pushd .buildkite/tools/check-image-names
+
+echo "--- Check to see if all manifests contain valid image names"
+go run check-image-names.go "${ROOT}"/base

.buildkite/ci-checkov.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+# Set this to fail on the install
+set -euxo pipefail
+
+# Install and run the plugin for checkov
+# Use the full path to run pip3.10
+pip3 install checkov
+
+# List of checks we do not want to run here
+# This is a living list and will see additions and mostly removals over time.
+SKIP_CHECKS="CKV_GCP_22,CKV_GCP_66,CKV_GCP_13,CKV_GCP_71,CKV_GCP_61,CKV_GCP_21,CKV_GCP_65,CKV_GCP_67,CKV_GCP_20,CKV_GCP_69,CKV_GCP_12,CKV_GCP_24,CKV_GCP_25,CKV_GCP_64,CKV_GCP_68,CKV2_AWS_5,CKV2_GCP_3,CKV2_GCP_5,CKV_AWS_23,CKV_GCP_70,CKV_GCP_62,CKV_GCP_62,CKV_GCP_62,CKV_GCP_62,CKV_GCP_29,CKV_GCP_39"
+
+set +x
+# In case no terraform code is present
+echo "--- Starting Checkov..."
+echo "Note: If there is no output below here then no terraform code was found to scan.  All good!"
+echo "==========================================================================================="
+
+# Set not to fail on non-zero exit code
+set +e
+# Run checkov
+python3 -m checkov.main --skip-check $SKIP_CHECKS --quiet --framework terraform --compact -d .
+
+# Options
+# --quiet: Only show failing tests
+# --compact: Do not show code snippets
+# --framework: Only scan terraform code
+
+# Capture the error code
+CHECKOV_EXIT_CODE="$?"
+
+# We check the exit code and display a warning if anything was found
+if [[ "$CHECKOV_EXIT_CODE" != 0 ]]; then
+  echo "^^^ +++"
+  echo "Possible Terraform security issues found. "
+  echo "Please refer to the Sourcegraph handbook for guidance: https://handbook.sourcegraph.com/product-engineering/engineering/cloud/security/checkov"
+fi

.buildkite/cleanup-disks.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -ex
+
+cd "$(dirname "${BASH_SOURCE[0]}")"/..
+
+export TEST_GCP_PROJECT=sourcegraph-ci
+export TEST_GCP_ZONE=us-central1-a
+
+# Temporary fix: delete unattached disks associated with these tests
+# https://github.com/sourcegraph/sourcegraph/issues/32916 will implement long-term fix
+gcloud compute disks list --filter="name:gke-ds-test AND NOT users:*" --format="value(name)" --project ${TEST_GCP_PROJECT} |
+  while read -r disk; do gcloud compute disks delete ${disk} --zone ${TEST_GCP_ZONE} --project ${TEST_GCP_PROJECT} --quiet; done
+
+# Also delete disks created by integration-restricted-test
+gcloud compute disks list --filter="description:ds-test-restricted AND NOT users:*" --format="value(name)" --project ${TEST_GCP_PROJECT} |
+  while read -r disk; do gcloud compute disks delete ${disk} --zone ${TEST_GCP_ZONE} --project ${TEST_GCP_PROJECT} --quiet; done

.buildkite/hooks/pre-command

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+set -eu
+pushd "$(dirname "${BASH_SOURCE[0]}")"/../..
+
+WORKDIR=$(pwd)
+
+echo "Installing asdf dependencies as defined in '${WORKDIR}/.tool-versions':"
+asdf install

.buildkite/install-yj.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+VERSION=1.0.0
+
+if [ ! "$(which yj)" ]; then
+  echo "Installing yj to /usr/local/bin"
+  wget https://github.com/sourcegraph/yj/releases/download/v${VERSION}/yj-${VERSION}-linux-amd64 -O /usr/local/bin/yj
+  chmod a+x /usr/local/bin/yj
+fi

.buildkite/pipeline.yaml

@@ -0,0 +1,24 @@
+steps:
+  - label: ":k8s:"
+    command: .buildkite/verify-yaml.sh
+    agents: { queue: standard }
+
+  - label: ":k8s:"
+    command: .buildkite/verify-label.sh
+    agents: { queue: standard }
+
+  - label: ":k8s:"
+    command: .buildkite/verify-rbac-labels.sh
+    agents: { queue: standard }
+
+  - label: ":git: :sleuth_or_spy:"
+    command: .buildkite/verify-release/verify-release.sh
+    agents: { queue: standard }
+
+  - label: ":k8s: :sleuth_or_spy:"
+    command: .buildkite/check-image-names.sh
+    agents: { queue: standard }
+
+  - label: ":k8s:"
+    command: .buildkite/verify-overlays.sh
+    agents: { queue: standard }

.buildkite/rbac.jq

@@ -0,0 +1,20 @@
+def rbacAPI:
+    .apiVersion and 
+    (.apiVersion | contains("rbac.authorization.k8s.io"));
+
+def rbacKind:
+    .kind as $k 
+    | ["Role", "RoleBinding", "ClusterRole", "ClusterRoleBinding", "ServiceAccount"]
+    | index($k);
+
+def isRBAC: 
+    rbacAPI or rbacKind;
+
+def hasLabel: 
+    if . | isRBAC then
+        .metadata.labels.category == "rbac" 
+    else
+        true
+    end;
+
+. | all(hasLabel)

.buildkite/shfmt.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -ex
+
+cd "$(dirname "${BASH_SOURCE[0]}")"/..
+
+shfmt -d .

.buildkite/tools/check-image-names/check-image-names.go

@@ -0,0 +1,90 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strings"
+
+	"github.com/pkg/errors"
+	"github.com/sourcegraph/sourcegraph/enterprise/dev/ci/images"
+)
+
+var (
+	i       = regexp.MustCompile(`index\.docker\.io\/sourcegraph/(?P<image>[a-z0-9-_.]+):[a-z0-9-_]+@sha256:[[:alnum:]]+`)
+	matches []string
+	data    []byte
+)
+
+func main() {
+
+	path := os.Args[1]
+
+	fmt.Print(CheckImages(path))
+
+}
+
+func CheckImages(path string) error {
+
+	err := filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			fmt.Println(err)
+			return err
+		}
+		if info.IsDir() {
+			return nil
+		}
+
+		if strings.HasPrefix(path, ".git") {
+			return nil
+		}
+
+		data, err = ioutil.ReadFile(path)
+		if err != nil {
+			return errors.Wrap(err, "when reading file contents")
+		}
+
+		// matchedImages contains all lines matching our regex. FindAllSubmatch returns
+		// a slice containing the full string, and the capture group `image` for each image in a file.
+		// We then loop over each slice, pull out the capture group and append it to a list of images
+		// to compare with upstream.
+		matchedImages := i.FindAllSubmatch(data, -1)
+		for _, match := range matchedImages {
+			matchd := string(match[1])
+			matches = append(matches, matchd)
+		}
+
+		return nil
+	})
+	matches = Unique(matches)
+	for i, image := range matches {
+		if image != images.DeploySourcegraphDockerImages[i] {
+			return fmt.Errorf("image: %s is not in the upstream list", image)
+		}
+	}
+
+	if err != nil {
+		fmt.Println(err)
+	}
+
+	return nil
+
+}
+
+func Unique(strSlice []string) []string {
+	keys := make(map[string]bool)
+	list := []string{}
+	for _, entry := range strSlice {
+		if _, found := keys[entry]; !found {
+			keys[entry] = true
+			list = append(list, entry)
+		}
+	}
+
+	sort.Strings(list)
+
+	return list
+}

.buildkite/tools/check-image-names/check-image-names_test.go

@@ -0,0 +1,16 @@
+package main
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCheckImages(t *testing.T) {
+
+	errorString := "image: foo is not in the upstream list"
+	err := CheckImages("tests")
+	assert.NotNil(t, err)
+	assert.Contains(t, err.Error(), errorString)
+
+}