Use an uncached client to retrieve composed resources not found in the cache (crossplane#6261)

* Do not cache Unstructured resources

Fixes crossplane#6260

Signed-off-by: Bob Haddleton <[email protected]>

* Use a no-cache client to retrieve not-found composed resources

Signed-off-by: Bob Haddleton <[email protected]>

* refactor variable names

Signed-off-by: Bob Haddleton <[email protected]>

---------

Signed-off-by: Bob Haddleton <[email protected]>

Author: bobh66

.gitignore

@@ -9,6 +9,8 @@ cover.out
 /.vendor-new
 .DS_Store
 Zone.Identifier
+.tmp-earthly-out/
+build/
 
 # gitlab example
 # exclude files generate by running the example

cmd/crossplane/core/core.go

@@ -346,7 +346,7 @@ func (c *startCommand) Run(s *runtime.Scheme, log logging.Logger) error { //noli
 		log.Info("API extensions cache stopped")
 	}()
 
-	cl, err := client.New(mgr.GetConfig(), client.Options{
+	cached, err := client.New(mgr.GetConfig(), client.Options{
 		HTTPClient: mgr.GetHTTPClient(),
 		Scheme:     mgr.GetScheme(),
 		Mapper:     mgr.GetRESTMapper(),
@@ -364,13 +364,25 @@ func (c *startCommand) Run(s *runtime.Scheme, log logging.Logger) error { //noli
 		return errors.Wrap(err, "cannot create client for API extension controllers")
 	}
 
+	// Create a separate no-cache client for use when the composite controller does not find an Unstructured
+	// resource that it expects to find in the cache.
+	uncached, err := client.New(mgr.GetConfig(), client.Options{
+		HTTPClient: mgr.GetHTTPClient(),
+		Scheme:     mgr.GetScheme(),
+		Mapper:     mgr.GetRESTMapper(),
+	})
+	if err != nil {
+		return errors.Wrap(err, "cannot create uncached client for API extension controllers")
+	}
+
 	// It's important the engine's client is wrapped with unstructured.NewClient
 	// because controller-runtime always caches *unstructured.Unstructured, not
 	// our wrapper types like *composite.Unstructured. This client takes care of
 	// automatically wrapping and unwrapping *unstructured.Unstructured.
 	ce := engine.New(mgr,
 		engine.TrackInformers(ca, mgr.GetScheme()),
-		unstructured.NewClient(cl),
+		unstructured.NewClient(cached),
+		unstructured.NewClient(uncached),
 		engine.WithLogger(log),
 	)
 

internal/controller/apiextensions/composite/composition_functions.go

@@ -217,18 +217,18 @@ func WithManagedFieldsUpgrader(u ManagedFieldsUpgrader) FunctionComposerOption {
 
 // NewFunctionComposer returns a new Composer that supports composing resources using
 // both Patch and Transform (P&T) logic and a pipeline of Composition Functions.
-func NewFunctionComposer(kube client.Client, r FunctionRunner, o ...FunctionComposerOption) *FunctionComposer {
-	f := NewSecretConnectionDetailsFetcher(kube)
+func NewFunctionComposer(cached, uncached client.Client, r FunctionRunner, o ...FunctionComposerOption) *FunctionComposer {
+	f := NewSecretConnectionDetailsFetcher(cached)
 
 	c := &FunctionComposer{
-		client: kube,
+		client: cached,
 
 		composite: xr{
 			ConnectionDetailsFetcher:         f,
-			ComposedResourceObserver:         NewExistingComposedResourceObserver(kube, f),
-			ComposedResourceGarbageCollector: NewDeletingComposedResourceGarbageCollector(kube),
-			NameGenerator:                    names.NewNameGenerator(kube),
-			ManagedFieldsUpgrader:            NewPatchingManagedFieldsUpgrader(kube),
+			ComposedResourceObserver:         NewExistingComposedResourceObserver(cached, uncached, f),
+			ComposedResourceGarbageCollector: NewDeletingComposedResourceGarbageCollector(cached),
+			NameGenerator:                    names.NewNameGenerator(cached),
+			ManagedFieldsUpgrader:            NewPatchingManagedFieldsUpgrader(cached),
 		},
 
 		pipeline: r,
@@ -580,14 +580,15 @@ func ComposedFieldOwnerName(xr *composite.Unstructured) string {
 // any existing composed resources from the API server. It also loads their
 // connection details.
 type ExistingComposedResourceObserver struct {
-	resource client.Reader
+	cached   client.Reader
+	uncached client.Reader
 	details  managed.ConnectionDetailsFetcher
 }
 
 // NewExistingComposedResourceObserver returns a ComposedResourceGetter that
 // fetches an XR's existing composed resources.
-func NewExistingComposedResourceObserver(c client.Reader, f managed.ConnectionDetailsFetcher) *ExistingComposedResourceObserver {
-	return &ExistingComposedResourceObserver{resource: c, details: f}
+func NewExistingComposedResourceObserver(c, uc client.Reader, f managed.ConnectionDetailsFetcher) *ExistingComposedResourceObserver {
+	return &ExistingComposedResourceObserver{cached: c, uncached: uc, details: f}
 }
 
 // ObserveComposedResources begins building composed resource state by
@@ -613,10 +614,14 @@ func (g *ExistingComposedResourceObserver) ObserveComposedResources(ctx context.
 
 		r := composed.New(composed.FromReference(ref))
 		nn := types.NamespacedName{Namespace: ref.Namespace, Name: ref.Name}
-		err := g.resource.Get(ctx, nn, r)
+		err := g.cached.Get(ctx, nn, r)
 		if kerrors.IsNotFound(err) {
-			// We believe we created this resource, but it doesn't exist.
-			continue
+			// We believe we created this resource, but it is not in the cache yet?  Try again without the cache.
+			err = g.uncached.Get(ctx, nn, r)
+			if kerrors.IsNotFound(err) {
+				// We believe we created this resource, but it no longer exists.
+				continue
+			}
 		}
 		if err != nil {
 			return nil, errors.Wrap(err, errGetComposed)