Merge pull request kubernetes#128152 from stlaz/ensure-secret-images

Multi-tenancy in accessing node images via Pod API

Author: k8s-ci-robot

pkg/credentialprovider/keyring.go

@@ -17,14 +17,19 @@ limitations under the License.
 package credentialprovider
 
 import (
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
 	"net"
 	"net/url"
 	"path/filepath"
 	"sort"
 	"strings"
 
 	"k8s.io/apimachinery/pkg/util/sets"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/klog/v2"
+	"k8s.io/kubernetes/pkg/features"
 )
 
 // DockerKeyring tracks a set of docker registry credentials, maintaining a
@@ -35,13 +40,13 @@ import (
 //     most specific match for a given image
 //   - iterating a map does not yield predictable results
 type DockerKeyring interface {
-	Lookup(image string) ([]AuthConfig, bool)
+	Lookup(image string) ([]TrackedAuthConfig, bool)
 }
 
 // BasicDockerKeyring is a trivial map-backed implementation of DockerKeyring
 type BasicDockerKeyring struct {
 	index []string
-	creds map[string][]AuthConfig
+	creds map[string][]TrackedAuthConfig
 }
 
 // providersDockerKeyring is an implementation of DockerKeyring that
@@ -50,6 +55,47 @@ type providersDockerKeyring struct {
 	Providers []DockerConfigProvider
 }
 
+// TrackedAuthConfig wraps the AuthConfig and adds information about the source
+// of the credentials.
+type TrackedAuthConfig struct {
+	AuthConfig
+	AuthConfigHash string
+
+	Source *CredentialSource
+}
+
+// NewTrackedAuthConfig initializes the TrackedAuthConfig structure by adding
+// the source information to the supplied AuthConfig. It also counts a hash of the
+// AuthConfig and keeps it in the returned structure.
+//
+// The supplied CredentialSource is only used when the "KubeletEnsureSecretPulledImages"
+// is enabled, the same applies for counting the hash.
+func NewTrackedAuthConfig(c *AuthConfig, src *CredentialSource) *TrackedAuthConfig {
+	if c == nil {
+		panic("cannot construct TrackedAuthConfig with a nil AuthConfig")
+	}
+
+	authConfig := &TrackedAuthConfig{
+		AuthConfig: *c,
+	}
+
+	if utilfeature.DefaultFeatureGate.Enabled(features.KubeletEnsureSecretPulledImages) {
+		authConfig.Source = src
+		authConfig.AuthConfigHash = hashAuthConfig(c)
+	}
+	return authConfig
+}
+
+type CredentialSource struct {
+	Secret SecretCoordinates
+}
+
+type SecretCoordinates struct {
+	UID       string
+	Namespace string
+	Name      string
+}
+
 // AuthConfig contains authorization information for connecting to a Registry
 // This type mirrors "github.com/docker/docker/api/types.AuthConfig"
 type AuthConfig struct {
@@ -72,11 +118,13 @@ type AuthConfig struct {
 	RegistryToken string `json:"registrytoken,omitempty"`
 }
 
-// Add add some docker config in basic docker keyring
-func (dk *BasicDockerKeyring) Add(cfg DockerConfig) {
+// Add inserts the docker config `cfg` into the basic docker keyring. It attaches
+// the `src` information that describes where the docker config `cfg` comes from.
+// `src` is nil if the docker config is globally available on the node.
+func (dk *BasicDockerKeyring) Add(src *CredentialSource, cfg DockerConfig) {
 	if dk.index == nil {
 		dk.index = make([]string, 0)
-		dk.creds = make(map[string][]AuthConfig)
+		dk.creds = make(map[string][]TrackedAuthConfig)
 	}
 	for loc, ident := range cfg {
 		creds := AuthConfig{
@@ -111,7 +159,9 @@ func (dk *BasicDockerKeyring) Add(cfg DockerConfig) {
 		} else {
 			key = parsed.Host
 		}
-		dk.creds[key] = append(dk.creds[key], creds)
+		trackedCreds := NewTrackedAuthConfig(&creds, src)
+
+		dk.creds[key] = append(dk.creds[key], *trackedCreds)
 		dk.index = append(dk.index, key)
 	}
 
@@ -235,9 +285,9 @@ func URLsMatch(globURL *url.URL, targetURL *url.URL) (bool, error) {
 // Lookup implements the DockerKeyring method for fetching credentials based on image name.
 // Multiple credentials may be returned if there are multiple potentially valid credentials
 // available.  This allows for rotation.
-func (dk *BasicDockerKeyring) Lookup(image string) ([]AuthConfig, bool) {
+func (dk *BasicDockerKeyring) Lookup(image string) ([]TrackedAuthConfig, bool) {
 	// range over the index as iterating over a map does not provide a predictable ordering
-	ret := []AuthConfig{}
+	ret := []TrackedAuthConfig{}
 	for _, k := range dk.index {
 		// both k and image are schemeless URLs because even though schemes are allowed
 		// in the credential configurations, we remove them in Add.
@@ -257,30 +307,32 @@ func (dk *BasicDockerKeyring) Lookup(image string) ([]AuthConfig, bool) {
 		}
 	}
 
-	return []AuthConfig{}, false
+	return []TrackedAuthConfig{}, false
 }
 
 // Lookup implements the DockerKeyring method for fetching credentials
 // based on image name.
-func (dk *providersDockerKeyring) Lookup(image string) ([]AuthConfig, bool) {
+func (dk *providersDockerKeyring) Lookup(image string) ([]TrackedAuthConfig, bool) {
 	keyring := &BasicDockerKeyring{}
 
 	for _, p := range dk.Providers {
-		keyring.Add(p.Provide(image))
+		// TODO: the source should probably change once we depend on service accounts (KEP-4412).
+		//       Perhaps `Provide()` should return the source modified to accommodate this?
+		keyring.Add(nil, p.Provide(image))
 	}
 
 	return keyring.Lookup(image)
 }
 
 // FakeKeyring a fake config credentials
 type FakeKeyring struct {
-	auth []AuthConfig
+	auth []TrackedAuthConfig
 	ok   bool
 }
 
 // Lookup implements the DockerKeyring method for fetching credentials based on image name
 // return fake auth and ok
-func (f *FakeKeyring) Lookup(image string) ([]AuthConfig, bool) {
+func (f *FakeKeyring) Lookup(image string) ([]TrackedAuthConfig, bool) {
 	return f.auth, f.ok
 }
 
@@ -289,8 +341,8 @@ type UnionDockerKeyring []DockerKeyring
 
 // Lookup implements the DockerKeyring method for fetching credentials based on image name.
 // return each credentials
-func (k UnionDockerKeyring) Lookup(image string) ([]AuthConfig, bool) {
-	authConfigs := []AuthConfig{}
+func (k UnionDockerKeyring) Lookup(image string) ([]TrackedAuthConfig, bool) {
+	authConfigs := []TrackedAuthConfig{}
 	for _, subKeyring := range k {
 		if subKeyring == nil {
 			continue
@@ -302,3 +354,14 @@ func (k UnionDockerKeyring) Lookup(image string) ([]AuthConfig, bool) {
 
 	return authConfigs, (len(authConfigs) > 0)
 }
+
+func hashAuthConfig(creds *AuthConfig) string {
+	credBytes, err := json.Marshal(creds)
+	if err != nil {
+		return ""
+	}
+
+	hash := sha256.New()
+	hash.Write([]byte(credBytes))
+	return hex.EncodeToString(hash.Sum(nil))
+}

pkg/credentialprovider/keyring_test.go

@@ -21,6 +21,10 @@ import (
 	"fmt"
 	"reflect"
 	"testing"
+
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	featuregatetesting "k8s.io/component-base/featuregate/testing"
+	"k8s.io/kubernetes/pkg/features"
 )
 
 func TestURLsMatch(t *testing.T) {
@@ -222,7 +226,7 @@ func TestDockerKeyringForGlob(t *testing.T) {
 		if cfg, err := ReadDockerConfigFileFromBytes([]byte(sampleDockerConfig)); err != nil {
 			t.Errorf("Error processing json blob %q, %v", sampleDockerConfig, err)
 		} else {
-			keyring.Add(cfg)
+			keyring.Add(nil, cfg)
 		}
 
 		creds, ok := keyring.Lookup(test.targetURL + "/foo/bar")
@@ -290,7 +294,7 @@ func TestKeyringMiss(t *testing.T) {
 		if cfg, err := ReadDockerConfigFileFromBytes([]byte(sampleDockerConfig)); err != nil {
 			t.Errorf("Error processing json blob %q, %v", sampleDockerConfig, err)
 		} else {
-			keyring.Add(cfg)
+			keyring.Add(nil, cfg)
 		}
 
 		_, ok := keyring.Lookup(test.lookupURL + "/foo/bar")
@@ -318,7 +322,7 @@ func TestKeyringMissWithDockerHubCredentials(t *testing.T) {
 	if cfg, err := ReadDockerConfigFileFromBytes([]byte(sampleDockerConfig)); err != nil {
 		t.Errorf("Error processing json blob %q, %v", sampleDockerConfig, err)
 	} else {
-		keyring.Add(cfg)
+		keyring.Add(nil, cfg)
 	}
 
 	val, ok := keyring.Lookup("world.mesos.org/foo/bar")
@@ -344,7 +348,7 @@ func TestKeyringHitWithUnqualifiedDockerHub(t *testing.T) {
 	if cfg, err := ReadDockerConfigFileFromBytes([]byte(sampleDockerConfig)); err != nil {
 		t.Errorf("Error processing json blob %q, %v", sampleDockerConfig, err)
 	} else {
-		keyring.Add(cfg)
+		keyring.Add(nil, cfg)
 	}
 
 	creds, ok := keyring.Lookup("google/docker-registry")
@@ -353,7 +357,7 @@ func TestKeyringHitWithUnqualifiedDockerHub(t *testing.T) {
 		return
 	}
 	if len(creds) > 1 {
-		t.Errorf("Got more hits than expected: %s", creds)
+		t.Errorf("Got more hits than expected: %v", creds)
 	}
 	val := creds[0]
 
@@ -385,7 +389,7 @@ func TestKeyringHitWithUnqualifiedLibraryDockerHub(t *testing.T) {
 	if cfg, err := ReadDockerConfigFileFromBytes([]byte(sampleDockerConfig)); err != nil {
 		t.Errorf("Error processing json blob %q, %v", sampleDockerConfig, err)
 	} else {
-		keyring.Add(cfg)
+		keyring.Add(nil, cfg)
 	}
 
 	creds, ok := keyring.Lookup("jenkins")
@@ -394,7 +398,7 @@ func TestKeyringHitWithUnqualifiedLibraryDockerHub(t *testing.T) {
 		return
 	}
 	if len(creds) > 1 {
-		t.Errorf("Got more hits than expected: %s", creds)
+		t.Errorf("Got more hits than expected: %v", creds)
 	}
 	val := creds[0]
 
@@ -426,7 +430,7 @@ func TestKeyringHitWithQualifiedDockerHub(t *testing.T) {
 	if cfg, err := ReadDockerConfigFileFromBytes([]byte(sampleDockerConfig)); err != nil {
 		t.Errorf("Error processing json blob %q, %v", sampleDockerConfig, err)
 	} else {
-		keyring.Add(cfg)
+		keyring.Add(nil, cfg)
 	}
 
 	creds, ok := keyring.Lookup(url + "/google/docker-registry")
@@ -435,7 +439,7 @@ func TestKeyringHitWithQualifiedDockerHub(t *testing.T) {
 		return
 	}
 	if len(creds) > 2 {
-		t.Errorf("Got more hits than expected: %s", creds)
+		t.Errorf("Got more hits than expected: %v", creds)
 	}
 	val := creds[0]
 
@@ -498,20 +502,24 @@ func TestProvidersDockerKeyring(t *testing.T) {
 }
 
 func TestDockerKeyringLookup(t *testing.T) {
+	// turn on the ensure secret pulled images feature to get the hashes with the creds
+	featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KubeletEnsureSecretPulledImages, true)
 	ada := AuthConfig{
 		Username: "ada",
 		Password: "smash", // Fake value for testing.
 		Email:    "[email protected]",
 	}
+	adaHash := "353258b53f5e9a57b059eab3f05312fc35bbeb874f08ce101e7bf0bf46977423"
 
 	grace := AuthConfig{
 		Username: "grace",
 		Password: "squash", // Fake value for testing.
 		Email:    "[email protected]",
 	}
+	graceHash := "f949b3837a1eb733a951b6aeda0b3327c09ec50c917de9ca35818e8fbf567e29"
 
 	dk := &BasicDockerKeyring{}
-	dk.Add(DockerConfig{
+	dk.Add(nil, DockerConfig{
 		"bar.example.com/pong": DockerConfigEntry{
 			Username: grace.Username,
 			Password: grace.Password,
@@ -526,27 +534,27 @@ func TestDockerKeyringLookup(t *testing.T) {
 
 	tests := []struct {
 		image string
-		match []AuthConfig
+		match []TrackedAuthConfig
 		ok    bool
 	}{
 		// direct match
-		{"bar.example.com", []AuthConfig{ada}, true},
+		{"bar.example.com", []TrackedAuthConfig{{AuthConfig: ada, AuthConfigHash: adaHash}}, true},
 
 		// direct match deeper than other possible matches
-		{"bar.example.com/pong", []AuthConfig{grace, ada}, true},
+		{"bar.example.com/pong", []TrackedAuthConfig{{AuthConfig: grace, AuthConfigHash: graceHash}, {AuthConfig: ada, AuthConfigHash: adaHash}}, true},
 
 		// no direct match, deeper path ignored
-		{"bar.example.com/ping", []AuthConfig{ada}, true},
+		{"bar.example.com/ping", []TrackedAuthConfig{{AuthConfig: ada, AuthConfigHash: adaHash}}, true},
 
 		// match first part of path token
-		{"bar.example.com/pongz", []AuthConfig{grace, ada}, true},
+		{"bar.example.com/pongz", []TrackedAuthConfig{{AuthConfig: grace, AuthConfigHash: graceHash}, {AuthConfig: ada, AuthConfigHash: adaHash}}, true},
 
 		// match regardless of sub-path
-		{"bar.example.com/pong/pang", []AuthConfig{grace, ada}, true},
+		{"bar.example.com/pong/pang", []TrackedAuthConfig{{AuthConfig: grace, AuthConfigHash: graceHash}, {AuthConfig: ada, AuthConfigHash: adaHash}}, true},
 
 		// no host match
-		{"example.com", []AuthConfig{}, false},
-		{"foo.example.com", []AuthConfig{}, false},
+		{"example.com", []TrackedAuthConfig{}, false},
+		{"foo.example.com", []TrackedAuthConfig{}, false},
 	}
 
 	for i, tt := range tests {
@@ -565,14 +573,17 @@ func TestDockerKeyringLookup(t *testing.T) {
 // by images that only match the hostname.
 // NOTE: the above covers the case of a more specific match trumping just hostname.
 func TestIssue3797(t *testing.T) {
+	// turn on the ensure secret pulled images feature to get the hashes with the creds
+	featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KubeletEnsureSecretPulledImages, true)
 	rex := AuthConfig{
 		Username: "rex",
 		Password: "tiny arms", // Fake value for testing.
 		Email:    "[email protected]",
 	}
+	rexHash := "899748fec74c8dd761845fca727f4249b05be275ff24026676fcd4351f656363"
 
 	dk := &BasicDockerKeyring{}
-	dk.Add(DockerConfig{
+	dk.Add(nil, DockerConfig{
 		"https://quay.io/v1/": DockerConfigEntry{
 			Username: rex.Username,
 			Password: rex.Password,
@@ -582,15 +593,15 @@ func TestIssue3797(t *testing.T) {
 
 	tests := []struct {
 		image string
-		match []AuthConfig
+		match []TrackedAuthConfig
 		ok    bool
 	}{
 		// direct match
-		{"quay.io", []AuthConfig{rex}, true},
+		{"quay.io", []TrackedAuthConfig{{AuthConfig: rex, AuthConfigHash: rexHash}}, true},
 
 		// partial matches
-		{"quay.io/foo", []AuthConfig{rex}, true},
-		{"quay.io/foo/bar", []AuthConfig{rex}, true},
+		{"quay.io/foo", []TrackedAuthConfig{{AuthConfig: rex, AuthConfigHash: rexHash}}, true},
+		{"quay.io/foo/bar", []TrackedAuthConfig{{AuthConfig: rex, AuthConfigHash: rexHash}}, true},
 	}
 
 	for i, tt := range tests {

pkg/credentialprovider/plugin/plugins.go

@@ -87,11 +87,12 @@ func NewExternalCredentialProviderDockerKeyring(podNamespace, podName, podUID, s
 	return keyring
 }
 
-func (k *externalCredentialProviderKeyring) Lookup(image string) ([]credentialprovider.AuthConfig, bool) {
+func (k *externalCredentialProviderKeyring) Lookup(image string) ([]credentialprovider.TrackedAuthConfig, bool) {
 	keyring := &credentialprovider.BasicDockerKeyring{}
 
 	for _, p := range k.providers {
-		keyring.Add(p.Provide(image))
+		// TODO: modify the credentialprovider.CredentialSource to contain the SA/pod information
+		keyring.Add(nil, p.Provide(image))
 	}
 
 	return keyring.Lookup(image)