Merge pull request #5 from buildkite-plugins/SUP-4893/Retry

Adds automatic retry logic with exponential backoff

Author: JoeColeman95

README.md

@@ -18,15 +18,15 @@ A `pipeline.yml` like this will read each secret out into a ENV variable:
 steps:
   - command: echo "The content of ANIMAL is \$ANIMAL"
     plugins:
-      - secrets#v1.0.0:
+      - secrets#v1.0.1:
           variables:
             ANIMAL: llamas
             FOO: bar
 ```
 
 ### Multiple
 
-Create a single Buildkite secret with one variable per line, encoded as base64 for storage. 
+Create a single Buildkite secret with one variable per line, encoded as base64 for storage.
 
 For example, setting three variables looks like this in a file:
 
@@ -50,7 +50,7 @@ job environment using a pipeline.yml like this:
 steps:
   - command: build.sh
     plugins:
-      - secrets#v1.0.0:
+      - secrets#v1.0.1:
           env: "llamas"
 ```
 
@@ -63,8 +63,36 @@ The secret key name to fetch multiple from Buildkite secrets.
 Specify a dictionary of `key: value` pairs to inject as environment variables, where the key is the name of the
 environment variable to be set, and the value is the Buildkite Secret key.
 
+### `retry-max-attempts` (optional, number, default: 5)
+
+Maximum number of retry attempts for transient failures when fetching secrets (e.g., 5xx server errors, network issues).
+
+### `retry-base-delay` (optional, number, default: 2)
+
+Base delay in seconds for exponential backoff between retry attempts.
+
+## Retry Behavior
+
+This plugin implements automatic retry logic with exponential backoff for secret calls. This will occur for 5xx server errors or any local network issues. If a 4xx code is received, a fast failure will be served.
+
+By default, the base delay will be 2 seconds, with a maximum of 5 retries.
+
+### Example with Custom Retry
+
+```yaml
+steps:
+  - command: build.sh
+    plugins:
+      - secrets#v1.0.1:
+          env: "llamas"
+          retry-max-attempts: 10
+          retry-base-delay: 2
+```
+
 ## Testing
+
 You can run the tests using `docker-compose`:
+
 ```bash
 docker compose run --rm tests
 ```

hooks/environment

@@ -1,11 +1,66 @@
 #!/bin/bash
 set -euo pipefail
 
+calculate_backoff_delay() {
+  local BASE_DELAY="$1"
+  local ATTEMPT="$2"
+  local DELAY=$((BASE_DELAY * (2 ** (ATTEMPT - 1))))
+  local JITTER=$((RANDOM % (DELAY / 4 + 1)))
+  local TOTAL_DELAY=$((DELAY + JITTER))
+
+  if [ "$TOTAL_DELAY" -gt 60 ]; then
+    TOTAL_DELAY=60
+  fi
+
+  echo "$TOTAL_DELAY"
+}
+
+buildkite_agent_secret_get_with_retry() {
+  local KEY="$1"
+  local MAX_ATTEMPTS="${BUILDKITE_PLUGIN_SECRETS_RETRY_MAX_ATTEMPTS:-5}"
+  local BASE_DELAY="${BUILDKITE_PLUGIN_SECRETS_RETRY_BASE_DELAY:-2}"
+  local ATTEMPT=1
+  local EXIT_CODE
+  local OUTPUT
+
+  while [ "$ATTEMPT" -le "$MAX_ATTEMPTS" ]; do
+    set +e
+    OUTPUT=$(buildkite-agent secret get "${KEY}" 2>&1)
+    EXIT_CODE=$?
+    set -e
+
+    if [ "$EXIT_CODE" -eq 0 ]; then
+      echo "$OUTPUT"
+      return 0
+    fi
+
+    if echo "$OUTPUT" | grep -qiE "(not found|unauthorized|forbidden|bad request)"; then
+      echo "$OUTPUT"
+      return "$EXIT_CODE"
+    fi
+
+    if [ "$ATTEMPT" -lt "$MAX_ATTEMPTS" ]; then
+      local TOTAL_DELAY
+      TOTAL_DELAY=$(calculate_backoff_delay "$BASE_DELAY" "$ATTEMPT")
+
+      echo "Failed to fetch secret $KEY (attempt $ATTEMPT/$MAX_ATTEMPTS). Retrying in ${TOTAL_DELAY}s..." >&2
+      echo "Error: $OUTPUT" >&2
+
+      sleep "$TOTAL_DELAY"
+      ATTEMPT=$((ATTEMPT + 1))
+    else
+      echo "Failed to fetch secret $KEY after $MAX_ATTEMPTS attempts" >&2
+      echo "Error: $OUTPUT" >&2
+      return "$EXIT_CODE"
+    fi
+  done
+}
+
 # downloads the secret by provided key using the buildkite-agent secret command
 downloadSecret() {
     local key=$1
 
-    if ! secret=$(buildkite-agent secret get "${key}"); then
+    if ! secret=$(buildkite_agent_secret_get_with_retry "${key}"); then
         echo "not found ${key}"
     else
         echo "${secret}"
@@ -57,7 +112,7 @@ processVariables() {
     key="${param/BUILDKITE_PLUGIN_SECRETS_VARIABLES_/}"
     path="${!param}"
 
-    if ! value=$(buildkite-agent secret get "${path}"); then
+    if ! value=$(buildkite_agent_secret_get_with_retry "${path}"); then
         echo "⚠️ Unable to find secret at ${path}"
         exit 1
     else

plugin.yml

@@ -10,4 +10,10 @@ configuration:
       type: string
     variables:
       type: object
+    retry-max-attempts:
+      type: number
+      default: 5
+    retry-base-delay:
+      type: number
+      default: 2
 additionalProperties: false

tests/environment-hook.bats

@@ -7,6 +7,7 @@ setup() {
 
   export BUILDKITE_PIPELINE_SLUG=testpipe
   export BUILDKITE_PLUGIN_SECRETS_DUMP_ENV=true
+  export BUILDKITE_PLUGIN_SECRETS_RETRY_BASE_DELAY=0
 }
 
 @test "Download default env from Buildkite secrets" {
@@ -86,11 +87,60 @@ setup() {
 
     stub buildkite-agent \
         "secret get env : echo 'not found'" \
-        "secret get best : exit 1"
+        "secret get best : echo 'not found' && exit 1"
 
     run bash -c "$PWD/hooks/environment"
 
     assert_failure
     assert_output --partial "⚠️ Unable to find secret at"
+    refute_output --partial "Retrying"
+    unstub buildkite-agent
+}
+
+@test "Retry on transient failure" {
+    export TESTDATA='Rk9PPWJhcgpCQVI9QmF6ClNFQ1JFVD1sbGFtYXMK'
+    export BUILDKITE_PLUGIN_SECRETS_RETRY_MAX_ATTEMPTS=3
+
+    stub buildkite-agent \
+        "secret get env : exit 1" \
+        "secret get env : echo ${TESTDATA}"
+
+    run bash -c "$PWD/hooks/environment"
+
+    assert_success
+    assert_output --partial "FOO=bar"
+    assert_output --partial "Failed to fetch secret env (attempt 1/3)"
+    unstub buildkite-agent
+}
+
+@test "Fails after max attempts" {
+    export BUILDKITE_PLUGIN_SECRETS_VARIABLES_ANIMAL="best"
+    export BUILDKITE_PLUGIN_SECRETS_RETRY_MAX_ATTEMPTS=3
+
+    stub buildkite-agent \
+        "secret get env : echo 'not found'" \
+        "secret get best : exit 1" \
+        "secret get best : exit 1" \
+        "secret get best : exit 1"
+
+    run bash -c "$PWD/hooks/environment"
+
+    assert_failure
+    assert_output --partial "Failed to fetch secret best after 3 attempts"
+    unstub buildkite-agent
+}
+
+@test "No retry on 4xx" {
+    export BUILDKITE_PLUGIN_SECRETS_VARIABLES_ANIMAL="best"
+    export BUILDKITE_PLUGIN_SECRETS_RETRY_MAX_ATTEMPTS=3
+
+    stub buildkite-agent \
+        "secret get env : echo 'not found'" \
+        "secret get best : echo 'unauthorized' && exit 1"
+
+    run bash -c "$PWD/hooks/environment"
+
+    assert_failure
+    refute_output --partial "Retrying"
     unstub buildkite-agent
 }