verify: Periodically write valid signatures to an on cluster config map

As a cluster verifies signatures for release digests, flush those
to an on cluster config map that ensures that even if the remote endpoint
is down the cluster can still verify those signatures.

Author: smarterclayton

pkg/cvo/cvo.go

@@ -46,6 +46,7 @@ import (
 	"github.com/openshift/cluster-version-operator/pkg/payload/precondition"
 	preconditioncv "github.com/openshift/cluster-version-operator/pkg/payload/precondition/clusterversion"
 	"github.com/openshift/cluster-version-operator/pkg/verify"
+	"github.com/openshift/cluster-version-operator/pkg/verify/verifyconfigmap"
 )
 
 const (
@@ -136,6 +137,9 @@ type Operator struct {
 	// verifier, if provided, will be used to check an update before it is executed.
 	// Any error will prevent an update payload from being accessed.
 	verifier verify.Interface
+	// signatureStore, if set, will be used to periodically persist signatures to
+	// the cluster as a config map
+	signatureStore *verify.StorePersister
 
 	configSync ConfigSyncWorker
 	// statusInterval is how often the configSync worker is allowed to retrigger
@@ -238,17 +242,21 @@ func (optr *Operator) InitializeFromPayload(restConfig *rest.Config, burstRestCo
 	}
 	// XXX: set this to the cincinnati version in preference
 	if _, err := semver.Parse(update.ImageRef.Name); err != nil {
-		return fmt.Errorf("The local release contents name %q is not a valid semantic version - no current version will be reported: %v", update.ImageRef.Name, err)
+		return fmt.Errorf("the local release contents name %q is not a valid semantic version - no current version will be reported: %v", update.ImageRef.Name, err)
 	}
 
 	optr.releaseCreated = update.ImageRef.CreationTimestamp.Time
 	optr.releaseVersion = update.ImageRef.Name
 
 	// Wraps operator's HTTPClient method to allow releaseVerifier to create http client with up-to-date config.
 	clientBuilder := &verifyClientBuilder{builder: optr.HTTPClient}
+	configClient, err := coreclientsetv1.NewForConfig(restConfig)
+	if err != nil {
+		return fmt.Errorf("unable to create a configuration client: %v", err)
+	}
 
 	// attempt to load a verifier as defined in the payload
-	verifier, err := loadConfigMapVerifierDataFromUpdate(update, clientBuilder)
+	verifier, signatureStore, err := loadConfigMapVerifierDataFromUpdate(update, clientBuilder, configClient)
 	if err != nil {
 		return err
 	}
@@ -259,6 +267,7 @@ func (optr *Operator) InitializeFromPayload(restConfig *rest.Config, burstRestCo
 		verifier = verify.Reject
 	}
 	optr.verifier = verifier
+	optr.signatureStore = signatureStore
 
 	// after the verifier has been loaded, initialize the sync worker with a payload retriever
 	// which will consume the verifier
@@ -282,7 +291,7 @@ func (optr *Operator) InitializeFromPayload(restConfig *rest.Config, burstRestCo
 // It returns an error if the data is not valid, or no verifier if no config map is found. See the verify
 // package for more details on the algorithm for verification. If the annotation is set, a verifier or error
 // is always returned.
-func loadConfigMapVerifierDataFromUpdate(update *payload.Update, clientBuilder verify.ClientBuilder) (verify.Interface, error) {
+func loadConfigMapVerifierDataFromUpdate(update *payload.Update, clientBuilder verify.ClientBuilder, configMapClient coreclientsetv1.ConfigMapsGetter) (verify.Interface, *verify.StorePersister, error) {
 	configMapGVK := corev1.SchemeGroupVersion.WithKind("ConfigMap")
 	for _, manifest := range update.Manifests {
 		if manifest.GVK != configMapGVK {
@@ -294,15 +303,21 @@ func loadConfigMapVerifierDataFromUpdate(update *payload.Update, clientBuilder v
 		src := fmt.Sprintf("the config map %s/%s", manifest.Obj.GetNamespace(), manifest.Obj.GetName())
 		data, _, err := unstructured.NestedStringMap(manifest.Obj.Object, "data")
 		if err != nil {
-			return nil, errors.Wrapf(err, "%s is not valid: %v", src, err)
+			return nil, nil, errors.Wrapf(err, "%s is not valid: %v", src, err)
 		}
 		verifier, err := verify.NewFromConfigMap(src, data, clientBuilder)
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
-		return verifier, nil
+
+		// allow the verifier to consult the cluster for signature data, and also configure
+		// a process that writes signatures back to that store
+		signatureStore := verifyconfigmap.NewStore(configMapClient, nil)
+		verifier = verifier.WithStores(signatureStore)
+		persister := verify.NewSignatureStorePersister(signatureStore, verifier)
+		return verifier, persister, nil
 	}
-	return nil, nil
+	return nil, nil, nil
 }
 
 // Run runs the cluster version operator until stopCh is completed. Workers is ignored for now.
@@ -339,6 +354,9 @@ func (optr *Operator) Run(ctx context.Context, workers int) {
 			utilruntime.HandleError(fmt.Errorf("unable to perform final sync: %v", err))
 		}
 	}, time.Second, stopCh)
+	if optr.signatureStore != nil {
+		go optr.signatureStore.Run(ctx, optr.minimumUpdateCheckInterval*2)
+	}
 
 	<-stopCh
 

pkg/cvo/cvo_test.go

@@ -28,6 +28,7 @@ import (
 	"k8s.io/apimachinery/pkg/watch"
 	"k8s.io/client-go/discovery"
 	"k8s.io/client-go/rest"
+	kfake "k8s.io/client-go/kubernetes/fake"
 	ktesting "k8s.io/client-go/testing"
 	"k8s.io/client-go/util/workqueue"
 	"k8s.io/klog"
@@ -3417,13 +3418,17 @@ func Test_loadReleaseVerifierFromConfigMap(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := loadConfigMapVerifierDataFromUpdate(tt.update, verify.DefaultClient)
+			f := kfake.NewSimpleClientset()
+			got, store, err := loadConfigMapVerifierDataFromUpdate(tt.update, verify.DefaultClient, f.CoreV1())
 			if (err != nil) != tt.wantErr {
 				t.Fatalf("loadReleaseVerifierFromPayload() error = %v, wantErr %v", err, tt.wantErr)
 			}
 			if (got != nil) != tt.want {
 				t.Fatal(got)
 			}
+			if tt.want && store == nil {
+				t.Fatalf("expected valid store")
+			}
 			if err != nil {
 				return
 			}

pkg/verify/persist.go

@@ -0,0 +1,53 @@
+package verify
+
+import (
+	"context"
+	"time"
+
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/klog"
+)
+
+// SignatureSource provides a set of signatures by digest to save.
+type SignatureSource interface {
+	// Signatures returns a list of valid signatures for release digests.
+	Signatures() map[string][][]byte
+}
+
+// PersistentSignatureStore is a store that can save signatures for
+// later recovery.
+type PersistentSignatureStore interface {
+	// Store saves the provided signatures or return an error. If context
+	// reaches its deadline the store should be cancelled.
+	Store(ctx context.Context, signatures map[string][][]byte) error
+}
+
+// StorePersister saves signatures into store periodically.
+type StorePersister struct {
+	store      PersistentSignatureStore
+	signatures SignatureSource
+}
+
+// NewSignatureStorePersister creates an instance that can save signatures into the destination
+// store.
+func NewSignatureStorePersister(dst PersistentSignatureStore, src SignatureSource) *StorePersister {
+	return &StorePersister{
+		store:      dst,
+		signatures: src,
+	}
+}
+
+// Run flushes signatures to the provided store every interval or until the context is finished.
+// After context is done, it runs one more time to attempt to flush the current state. It does not
+// return until that last store completes.
+func (p *StorePersister) Run(ctx context.Context, interval time.Duration) {
+	wait.Until(func() {
+		if err := p.store.Store(ctx, p.signatures.Signatures()); err != nil {
+			klog.Warningf("Unable to save signatures: %v", err)
+		}
+	}, interval, ctx.Done())
+
+	if err := p.store.Store(context.Background(), p.signatures.Signatures()); err != nil {
+		klog.Warningf("Unable to save signatures during final flush: %v", err)
+	}
+}

pkg/verify/verify.go

@@ -80,7 +80,7 @@ func NewReleaseVerifier(verifiers map[string]openpgp.EntityList, locations []*ur
 }
 
 // WithStores copies the provided verifier and adds any provided stores to the list.
-func (v *ReleaseVerifier) WithStores(stores []SignatureStore) *ReleaseVerifier {
+func (v *ReleaseVerifier) WithStores(stores ...SignatureStore) *ReleaseVerifier {
 	return &ReleaseVerifier{
 		verifiers:      v.verifiers,
 		locations:      v.locations,

pkg/verify/verifyconfigmap/store.go

@@ -0,0 +1,135 @@
+package verifyconfigmap
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"golang.org/x/time/rate"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
+	"k8s.io/client-go/util/retry"
+)
+
+// ReleaseLabelConfigMap is a label applied to a configmap inside the
+// openshift-config-managed namespace that indicates it contains signatures
+// for release image digests. Any binaryData key that starts with the digest
+// is added to the list of signatures checked.
+const ReleaseLabelConfigMap = "release.openshift.io/verification-signatures"
+
+// Store abstracts retrieving signatures from config maps on a cluster.
+type Store struct {
+	client corev1client.ConfigMapsGetter
+	ns     string
+
+	limiter *rate.Limiter
+	lock    sync.Mutex
+	last    []corev1.ConfigMap
+}
+
+// NewStore returns a store that can retrieve or persist signatures on a
+// cluster. If limiter is not specified it defaults to one call every 30 seconds.
+func NewStore(client corev1client.ConfigMapsGetter, limiter *rate.Limiter) *Store {
+	if limiter == nil {
+		limiter = rate.NewLimiter(rate.Every(30*time.Second), 1)
+	}
+	return &Store{
+		client: client,
+		ns:     "openshift-config-managed",
+	}
+}
+
+// String displays information about this source for human review.
+func (s *Store) String() string {
+	return fmt.Sprintf("config maps in %s with label %q", s.ns, ReleaseLabelConfigMap)
+}
+
+// rememberMostRecentConfigMaps stores a set of config maps containing
+// signatures.
+func (s *Store) rememberMostRecentConfigMaps(last []corev1.ConfigMap) {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+	s.last = last
+}
+
+// mostRecentConfigMaps returns the last cached version of config maps
+// containing signatures.
+func (s *Store) mostRecentConfigMaps() []corev1.ConfigMap {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+	return s.last
+}
+
+// DigestSignatures returns a list of signatures that match the request
+// digest out of config maps labelled with ReleaseLabelConfigMap in the
+// openshift-config-managed namespace.
+func (s *Store) DigestSignatures(ctx context.Context, digest string) ([][]byte, error) {
+	// avoid repeatedly reloading config maps
+	items := s.mostRecentConfigMaps()
+	r := s.limiter.Reserve()
+	if items == nil || r.OK() {
+		configMaps, err := s.client.ConfigMaps(s.ns).List(metav1.ListOptions{
+			LabelSelector: ReleaseLabelConfigMap,
+		})
+		if err != nil {
+			s.rememberMostRecentConfigMaps([]corev1.ConfigMap{})
+			return nil, err
+		}
+		items = configMaps.Items
+		s.rememberMostRecentConfigMaps(configMaps.Items)
+	}
+
+	var signatures [][]byte
+	for _, cm := range items {
+		for k, v := range cm.BinaryData {
+			if strings.HasPrefix(k, digest) {
+				signatures = append(signatures, v)
+			}
+		}
+	}
+	return signatures, nil
+}
+
+// Store attempts to persist the provided signatures into a form DigestSignatures will
+// retrieve.
+func (s *Store) Store(ctx context.Context, signaturesByDigest map[string][][]byte) error {
+	cm := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: s.ns,
+			Name:      "signatures-managed",
+			Labels: map[string]string{
+				ReleaseLabelConfigMap: "",
+			},
+		},
+		BinaryData: make(map[string][]byte),
+	}
+	for digest, signatures := range signaturesByDigest {
+		for i := 0; i < len(signatures); i++ {
+			cm.BinaryData[fmt.Sprintf("%s-%d", digest, i)] = signatures[i]
+		}
+	}
+	return retry.OnError(
+		retry.DefaultRetry,
+		func(err error) bool { return errors.IsConflict(err) || errors.IsAlreadyExists(err) },
+		func() error {
+			existing, err := s.client.ConfigMaps(s.ns).Get(cm.Name, metav1.GetOptions{})
+			if errors.IsNotFound(err) {
+				_, err := s.client.ConfigMaps(s.ns).Create(cm)
+				return err
+			}
+			if err != nil {
+				return err
+			}
+			existing.Labels = cm.Labels
+			existing.BinaryData = cm.BinaryData
+			existing.Data = cm.Data
+			_, err = s.client.ConfigMaps(s.ns).Update(existing)
+			return err
+		},
+	)
+}