Refactor KMS E2E tests (#3684)

* Move KMS tests from shell script to Go test suite

Signed-off-by: Colleen Murphy <[email protected]>

* Make KMS E2E tests hermetic

Use the scaffolding setup action to set up a local sigstore cluster and
run the KMS tests against the local rekor instance instead of the
public, production instance.

Signed-off-by: Colleen Murphy <[email protected]>

* Move KMS E2E workflow into main e2e-tests file

Condense the E2E tests into fewer workflow files. There are no unique
conditions that require them to be in separate files. Condensing them
makes them easier to discover, and makes the Actions tab in GitHub
cleaner because there are fewer workflows to sort through.

Signed-off-by: Colleen Murphy <[email protected]>

---------

Signed-off-by: Colleen Murphy <[email protected]>

Author: cmurphy

.github/workflows/e2e-tests-kms.yml

@@ -61,3 +61,49 @@ jobs:
       - name: Run pkcs11 end-to-end tests
         shell: bash
         run: ./test/e2e_test_pkcs11.sh
+
+  e2e-kms:
+    runs-on: ubuntu-latest
+    services:
+      vault:
+        image: hashicorp/vault:latest
+        env:
+          VAULT_DEV_ROOT_TOKEN_ID: root
+        options: >-
+          --health-cmd "VAULT_ADDR=http://127.0.0.1:8200 vault status"
+          --health-interval 1s
+          --health-timeout 5s
+          --health-retries 5
+          --restart always
+        ports:
+          - 8200:8200
+
+    env:
+      VAULT_TOKEN: "root"
+      VAULT_ADDR: "http://localhost:8200"
+      COSIGN_YES: "true"
+      SCAFFOLDING_RELEASE_VERSION: "v0.6.17"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+      - uses: cpanato/vault-installer@ac6d910a90d64f78ef773afe83887a35c95245c6 # v1.0.3
+        with:
+          vault-release: '1.14.1'
+
+      - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
+        with:
+          go-version: '1.21'
+          check-latest: true
+
+      - uses: imjasonh/setup-crane@00c9e93efa4e1138c9a7a5c594acd6c75a2fbf0c # v0.3
+
+      - name: Install cluster + sigstore
+        uses: sigstore/scaffolding/actions/setup@main
+        with:
+          version: ${{ env.SCAFFOLDING_RELEASE_VERSION }}
+
+      - name: enable vault transit
+        run: vault secrets enable transit
+
+      - name: Acceptance Tests
+        run: go test -tags=e2e,kms -v ./test/...

.github/workflows/e2e-tests.yml

@@ -0,0 +1,98 @@
+// Copyright 2024 The Sigstore Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build e2e && kms
+
+package test
+
+import (
+	"context"
+	"os"
+	"path"
+	"testing"
+
+	"github.com/sigstore/cosign/v2/cmd/cosign/cli/generate"
+	"github.com/sigstore/cosign/v2/cmd/cosign/cli/options"
+	"github.com/sigstore/cosign/v2/cmd/cosign/cli/sign"
+	"github.com/sigstore/cosign/v2/pkg/cosign/env"
+	_ "github.com/sigstore/sigstore/pkg/signature/kms/hashivault"
+)
+
+const (
+	rekorURLVar = "REKOR_URL"
+	testKMSVar  = "TEST_KMS"
+	defaultKMS  = "hashivault://transit"
+)
+
+func TestSecretsKMS(t *testing.T) {
+	ctx := context.Background()
+
+	repo, stop := reg(t)
+	defer stop()
+	td := t.TempDir()
+
+	imgName := path.Join(repo, "cosign-attach-e2e")
+	_, _, cleanup := mkimage(t, imgName)
+	defer cleanup()
+
+	kms := os.Getenv(testKMSVar)
+	if kms == "" {
+		kms = defaultKMS
+	}
+
+	prefix := path.Join(td, "test-kms")
+
+	must(generate.GenerateKeyPairCmd(ctx, kms, prefix, nil), t)
+
+	pubKey := prefix + ".pub"
+	privKey := kms
+
+	// Verify should fail at first
+	mustErr(verify(pubKey, imgName, true, nil, "", false), t)
+
+	rekorURL := os.Getenv(rekorURLVar)
+
+	must(downloadAndSetEnv(t, rekorURL+"/api/v1/log/publicKey", env.VariableSigstoreRekorPublicKey.String(), td), t)
+
+	// Now sign and verify with the KMS key
+	ko := options.KeyOpts{
+		KeyRef:           privKey,
+		RekorURL:         rekorURL,
+		SkipConfirmation: true,
+	}
+	so := options.SignOptions{
+		Upload:     true,
+		TlogUpload: true,
+	}
+	must(sign.SignCmd(ro, ko, so, []string{imgName}), t)
+	must(verify(pubKey, imgName, true, nil, "", false), t)
+
+	// Sign and verify with annotations
+	mustErr(verify(pubKey, imgName, true, map[string]any{"foo": "bar"}, "", false), t)
+	soAnno := options.SignOptions{
+		Upload:     true,
+		TlogUpload: true,
+		AnnotationOptions: options.AnnotationOptions{
+			Annotations: []string{"foo=bar"},
+		},
+	}
+	must(sign.SignCmd(ro, ko, soAnno, []string{imgName}), t)
+	must(verify(pubKey, imgName, true, map[string]any{"foo": "bar"}, "", false), t)
+
+	// Store signatures in a different repo
+	t.Setenv("COSIGN_REPOSITORY", path.Join(repo, "subbedrepo"))
+	must(sign.SignCmd(ro, ko, so, []string{imgName}), t)
+	must(verify(pubKey, imgName, true, nil, "", false), t)
+	os.Unsetenv("COSIGN_REPOSITORY")
+}

test/e2e_kms_test.go

@@ -13,7 +13,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-//go:build e2e && !cross
+//go:build e2e && !cross && !kms
 
 package test
 
@@ -2201,40 +2201,3 @@ func getOIDCToken() (string, error) {
 	}
 	return string(body), nil
 }
-
-func setLocalEnv(t *testing.T, dir string) error {
-	// fulcio repo is downloaded to the user's home directory by e2e_test.sh
-	home, err := os.UserHomeDir()
-	if err != nil {
-		return fmt.Errorf("error getting home directory: %w", err)
-	}
-	t.Setenv(env.VariableSigstoreCTLogPublicKeyFile.String(), path.Join(home, "fulcio/config/ctfe/pubkey.pem"))
-	err = downloadAndSetEnv(t, fulcioURL+"/api/v1/rootCert", env.VariableSigstoreRootFile.String(), dir)
-	if err != nil {
-		return fmt.Errorf("error setting %s env var: %w", env.VariableSigstoreRootFile.String(), err)
-	}
-	err = downloadAndSetEnv(t, rekorURL+"/api/v1/log/publicKey", env.VariableSigstoreRekorPublicKey.String(), dir)
-	if err != nil {
-		return fmt.Errorf("error setting %s env var: %w", env.VariableSigstoreRekorPublicKey.String(), err)
-	}
-	return nil
-}
-
-func downloadAndSetEnv(t *testing.T, url, envVar, dir string) error {
-	resp, err := http.Get(url)
-	if err != nil {
-		return fmt.Errorf("error downloading file: %w", err)
-	}
-	defer resp.Body.Close()
-	f, err := os.CreateTemp(dir, "")
-	if err != nil {
-		return fmt.Errorf("error creating temp file: %w", err)
-	}
-	defer f.Close()
-	_, err = io.Copy(f, resp.Body)
-	if err != nil {
-		return fmt.Errorf("error writing to file: %w", err)
-	}
-	t.Setenv(envVar, f.Name())
-	return nil
-}

test/e2e_test.go

@@ -20,9 +20,13 @@ package test
 import (
 	"context"
 	"crypto"
+	"fmt"
+	"io"
+	"net/http"
 	"net/http/httptest"
 	"net/url"
 	"os"
+	"path"
 	"path/filepath"
 	"testing"
 
@@ -39,6 +43,7 @@ import (
 	"github.com/sigstore/cosign/v2/cmd/cosign/cli/options"
 	cliverify "github.com/sigstore/cosign/v2/cmd/cosign/cli/verify"
 	"github.com/sigstore/cosign/v2/pkg/cosign"
+	"github.com/sigstore/cosign/v2/pkg/cosign/env"
 	ociremote "github.com/sigstore/cosign/v2/pkg/oci/remote"
 	sigs "github.com/sigstore/cosign/v2/pkg/signature"
 )
@@ -372,3 +377,42 @@ func registryClientOpts(ctx context.Context) []remote.Option {
 		remote.WithContext(ctx),
 	}
 }
+
+// setLocalEnv sets SIGSTORE_CT_LOG_PUBLIC_KEY_FILE, SIGSTORE_ROOT_FILE, and SIGSTORE_REKOR_PUBLIC_KEY for the locally running sigstore deployment.
+func setLocalEnv(t *testing.T, dir string) error {
+	// fulcio repo is downloaded to the user's home directory by e2e_test.sh
+	home, err := os.UserHomeDir()
+	if err != nil {
+		return fmt.Errorf("error getting home directory: %w", err)
+	}
+	t.Setenv(env.VariableSigstoreCTLogPublicKeyFile.String(), path.Join(home, "fulcio/config/ctfe/pubkey.pem"))
+	err = downloadAndSetEnv(t, fulcioURL+"/api/v1/rootCert", env.VariableSigstoreRootFile.String(), dir)
+	if err != nil {
+		return fmt.Errorf("error setting %s env var: %w", env.VariableSigstoreRootFile.String(), err)
+	}
+	err = downloadAndSetEnv(t, rekorURL+"/api/v1/log/publicKey", env.VariableSigstoreRekorPublicKey.String(), dir)
+	if err != nil {
+		return fmt.Errorf("error setting %s env var: %w", env.VariableSigstoreRekorPublicKey.String(), err)
+	}
+	return nil
+}
+
+// downloadAndSetEnv fetches a URL and sets the given environment variable to point to the downloaded file path.
+func downloadAndSetEnv(t *testing.T, url, envVar, dir string) error {
+	resp, err := http.Get(url)
+	if err != nil {
+		return fmt.Errorf("error downloading file: %w", err)
+	}
+	defer resp.Body.Close()
+	f, err := os.CreateTemp(dir, "")
+	if err != nil {
+		return fmt.Errorf("error creating temp file: %w", err)
+	}
+	defer f.Close()
+	_, err = io.Copy(f, resp.Body)
+	if err != nil {
+		return fmt.Errorf("error writing to file: %w", err)
+	}
+	t.Setenv(envVar, f.Name())
+	return nil
+}