allow precreating ACM resources (#96)

* allow precreating ACM resources

* Update docs/ACMRegistration.md

Co-authored-by: Ian Miller <34246471+imiller0@users.noreply.github.com>

---------

Co-authored-by: Ian Miller <34246471+imiller0@users.noreply.github.com>

Author: loganmc10

docs/ACMRegistration.md

@@ -59,3 +59,38 @@ oc apply -f /tmp/acm-secret.yaml
 ```
 
 Now your target cluster has a Secret than will allow it to authenticate to the ACM cluster and register itself. Once the registration succeeds, the secret is deleted from the target cluster.
+
+## Pre-creating the ManagedCluster and (optionally) KlusterletAddonConfig
+For use cases where flexible and ongoing control over the `ManagedCluster` and (optionally) the `klusterletAddonConfig` CRs is required you may pre-create these artifacts on the ACM hub cluster prior to relocation. This allows the user to control the lifetime and content (eg custom labels on the ManagedCluster) CRs beyond the initial installation phase. 
+
+When these CRs are pre-created for a cluster you can omit the `managedClusterSet` and `klusterletAddonConfig` fields from the relocation CR. In this case, the Service Account that you create only needs permissions to "get" Secrets for the cluster namespace created by the ManagedCluster.
+
+For example (run these commands on your ACM cluster):
+```
+oc create sa -n multicluster-engine acm-registration-sa
+
+cat << EOF | oc apply -f -
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: secret-reader
+rules:
+- apiGroups: [""]
+  resources: ["secrets"]
+  verbs: ["get"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: read-secrets
+  namespace: <managed-cluster-name>
+subjects:
+- kind: ServiceAccount
+  name: acm-registration-sa
+  namespace: multicluster-engine
+roleRef:
+  kind: ClusterRole
+  name: secret-reader
+  apiGroup: rbac.authorization.k8s.io
+EOF
+```

internal/acm/reconcile.go

@@ -106,12 +106,11 @@ func Reconcile(ctx context.Context, c client.Client, scheme *runtime.Scheme, rel
 	}
 
 	// the acmSecret holds the credentials for the ACM cluster
-	// these credentials should allow the following:
-	// Creating ManagedClusters (these are cluster scoped resources)
-	// Creating KlusterletAddonConfigs (these are namespace scoped resources)
-	// Getting Secrets (these are namespace scoped resources)
-	//
-	// the secret is deleted once registration succeeds
+	// there are 2 options:
+	// 1. Have this operator create the ManagedCluster and (optionally) KlusterletAddonConfig.
+	//    In this case, the acmSecret token should have open-cluster-management:managedclusterset:admin:default (ClusterRole) permissions.
+	// 2. Pre-create the ManagedCluster and (optionally) KlusterletAddonConfig.
+	//    In this case, the acmSecret token should have permissions to "get" Secrets for the namespace created by the ManagedCluster.
 	acmSecret := &corev1.Secret{}
 	if err := c.Get(ctx, types.NamespacedName{Name: relocation.Spec.ACMRegistration.ACMSecret.Name, Namespace: relocation.Spec.ACMRegistration.ACMSecret.Namespace}, acmSecret); err != nil {
 		return err
@@ -177,18 +176,26 @@ func Reconcile(ctx context.Context, c client.Client, scheme *runtime.Scheme, rel
 		},
 	}
 	if err := acmClient.Create(ctx, managedCluster); err != nil {
-		if !errors.IsAlreadyExists(err) {
+		if errors.IsForbidden(err) {
+			logger.Info("could not create ManagedCluster, proceeding anyway (it may have been pre-created)")
+		} else if !errors.IsAlreadyExists(err) {
 			return err
 		}
 	}
 
+	startTime := time.Now()
 	logger.Info("getting ACM import secret")
 	importSecret := &corev1.Secret{}
 	for {
 		if err := acmClient.Get(ctx, types.NamespacedName{Name: fmt.Sprintf("%s-import", relocation.Spec.ACMRegistration.ClusterName), Namespace: relocation.Spec.ACMRegistration.ClusterName}, importSecret); err != nil {
 			// after the ManagedCluster is created, it can take some time for this secret and the RBAC roles to be created
 			if errors.IsNotFound(err) || errors.IsForbidden(err) {
 				time.Sleep(time.Second * 10)
+
+				// we set a 5 minute timeout in case the ACM import secret can never be pulled
+				if time.Since(startTime) > time.Minute*5 {
+					return fmt.Errorf("could not get ACM import secret")
+				}
 				continue
 			}
 			return err
@@ -259,7 +266,7 @@ func Reconcile(ctx context.Context, c client.Client, scheme *runtime.Scheme, rel
 
 	logger.Info("waiting for Klusterlet to become Available")
 	// wait for the Klusterlet to become Available
-	startTime := time.Now()
+	startTime = time.Now()
 	for {
 		if checkKlusterlet(ctx, c, relocation, logger) == nil {
 			return nil