ci(gocd): Add canary deployments (#209)

Adds canary deployments that have to run for 300 seconds before
proceeding with the full deploy. The pipeline automatically pauses on
failure.

Author: jan-auer

gocd/templates/bash/deploy.sh

@@ -4,7 +4,12 @@ eval $(regions-project-env-vars --region="${SENTRY_REGION}")
 
 /devinfra/scripts/get-cluster-credentials
 
+LABEL_SELECTOR="service=objectstore"
+if [[ -n "${K8S_ENVIRONMENT:-}" ]]; then
+  LABEL_SELECTOR="$LABEL_SELECTOR,environment=$K8S_ENVIRONMENT"
+fi
+
 k8s-deploy \
-    --label-selector="service=objectstore" \
-    --image="us-docker.pkg.dev/sentryio/objectstore-mr/image:${GO_REVISION_OBJECTSTORE_REPO}" \
+    --label-selector="$LABEL_SELECTOR" \
+    --image="us-docker.pkg.dev/sentryio/objectstore-mr/image:$GO_REVISION_OBJECTSTORE_REPO" \
     --container-name="objectstore"

gocd/templates/bash/wait-canary.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+# Wait for 5 minutes
+sleep 300

gocd/templates/pipelines/objectstore.libsonnet

@@ -1,6 +1,44 @@
 local utils = import '../libs/utils.libsonnet';
 local gocdtasks = import 'github.com/getsentry/gocd-jsonnet/libs/gocd-tasks.libsonnet';
 
+local deploy_canary(region) = [
+  {
+    'deploy-canary': {
+      fetch_materials: true,
+      jobs: {
+        create_sentry_release: {
+          environment_variables: {
+            SENTRY_ORG: 'sentry',
+            SENTRY_PROJECT: 'objectstore',
+            SENTRY_AUTH_TOKEN: '{{SECRET:[devinfra-sentryio][token]}}',
+            SENTRY_ENVIRONMENT: region + '-canary',
+          },
+          timeout: 60,
+          elastic_profile_id: 'objectstore',
+          tasks: [
+            gocdtasks.script(importstr '../bash/create-sentry-release.sh'),
+          ],
+        },
+        deploy: {
+          timeout: 300,
+          elastic_profile_id: 'objectstore',
+          environment_variables: {
+            K8S_ENVIRONMENT: 'canary',
+            PAUSE_MESSAGE: 'Pausing pipeline due to canary failure.',
+          },
+          tasks: [
+            gocdtasks.script(importstr '../bash/deploy.sh'),
+            gocdtasks.script(importstr '../bash/wait-canary.sh'),
+            // TODO: Add sentry error checks
+            // TODO: Add datadog monitors
+            utils.pause_on_failure(),
+          ],
+        },
+      },
+    },
+  },
+];
+
 local deploy_primary(region) = [
   {
     'deploy-primary': {
@@ -45,5 +83,5 @@ function(region) {
       destination: 'objectstore',
     },
   },
-  stages: utils.github_checks() + deploy_primary(region),
+  stages: utils.github_checks() + deploy_canary(region) + deploy_primary(region),
 }