Merge pull request #1136 from hongkailiu/OTA-1427

OTA-1427: USC: Maintain status insights for Nodes

Author: openshift-merge-bot[bot]

install/0000_00_update-status-controller_02_rbac-DevPreviewNoUpgrade.yaml

@@ -113,6 +113,41 @@ rules:
   - get
 ---
 apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: update-status-controller-node-informer
+  annotations:
+    kubernetes.io/description: Role that allows the update status controller to watch and read node resources
+    include.release.openshift.io/self-managed-high-availability: "true"
+    release.openshift.io/feature-set: DevPreviewNoUpgrade
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - machineconfiguration.openshift.io
+  resources:
+  - machineconfigs
+  - machineconfigpools
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - config.openshift.io
+  resources:
+  - clusterversions
+  resourceNames:
+  - version
+  verbs:
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
   name: update-status-controller-library
@@ -181,3 +216,20 @@ roleRef:
   kind: ClusterRole
   name: update-status-controller-control-plane-informer
   apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: update-status-controller-node-informer
+  annotations:
+    kubernetes.io/description: Grant the update status controller permission to read cluster resources (temporary, until we have UpdateInformer producers)
+    include.release.openshift.io/self-managed-high-availability: "true"
+    release.openshift.io/feature-set: DevPreviewNoUpgrade
+subjects:
+- kind: ServiceAccount
+  name: update-status-controller
+  namespace: openshift-update-status-controller
+roleRef:
+  kind: ClusterRole
+  name: update-status-controller-node-informer
+  apiGroup: rbac.authorization.k8s.io

pkg/updatestatus/controlplaneinformer.go

@@ -62,9 +62,9 @@ func newControlPlaneInformerController(
 
 	controller := factory.New().
 		// call sync on ClusterVersion changes
-		WithInformersQueueKeysFunc(configApiQueueKeys, cvInformer).
+		WithInformersQueueKeysFunc(controlPlaneInformerQueueKeys, cvInformer).
 		// call sync on ClusterOperator changes with a filter
-		WithFilteredEventsInformersQueueKeysFunc(configApiQueueKeys, clusterOperatorEventFilterFunc, coInformer).
+		WithFilteredEventsInformersQueueKeysFunc(controlPlaneInformerQueueKeys, clusterOperatorEventFilterFunc, coInformer).
 		WithSync(c.sync).
 		ToController("ControlPlaneInformer", c.recorder)
 
@@ -97,7 +97,7 @@ const (
 func (c *controlPlaneInformerController) sync(ctx context.Context, syncCtx factory.SyncContext) error {
 	queueKey := syncCtx.QueueKey()
 
-	t, name, err := parseQueueKey(queueKey)
+	t, name, err := parseControlPlaneInformerQueueKey(queueKey)
 	if err != nil {
 		return fmt.Errorf("failed to parse queue key: %w", err)
 	}
@@ -470,15 +470,15 @@ func versionsFromHistory(history []configv1.UpdateHistory) ControlPlaneUpdateVer
 	return versions
 }
 
-func parseQueueKey(queueKey string) (string, string, error) {
+func parseControlPlaneInformerQueueKey(queueKey string) (string, string, error) {
 	splits := strings.Split(queueKey, "/")
 	if len(splits) != 2 {
 		return "", "", fmt.Errorf("invalid queue key: %s", queueKey)
 	}
 	return splits[0], splits[1], nil
 }
 
-func configApiQueueKeys(object runtime.Object) []string {
+func controlPlaneInformerQueueKeys(object runtime.Object) []string {
 	if object == nil {
 		return nil
 	}

pkg/updatestatus/controlplaneinformer_test.go

@@ -215,7 +215,7 @@ func Test_sync_with_cv(t *testing.T) {
 				now:             func() metav1.Time { return now },
 			}
 
-			queueKey := configApiQueueKeys(cv)[0]
+			queueKey := controlPlaneInformerQueueKeys(cv)[0]
 
 			err := controller.sync(context.Background(), newTestSyncContext(queueKey))
 			if err != nil {
@@ -391,14 +391,14 @@ func Test_configApiQueueKeys(t *testing.T) {
 				}
 			}()
 
-			actual := configApiQueueKeys(tc.object)
+			actual := controlPlaneInformerQueueKeys(tc.object)
 
 			if diff := cmp.Diff(tc.expected, actual); diff != "" {
 				t.Errorf("%s: key differs from expected:\n%s", tc.name, diff)
 			}
 
 			if !tc.expectedPanic && len(actual) > 0 {
-				kind, name, err := parseQueueKey(actual[0])
+				kind, name, err := parseControlPlaneInformerQueueKey(actual[0])
 				if err != nil {
 					t.Errorf("%s: unexpected error raised:\n%v", tc.name, err)
 				}
@@ -504,7 +504,7 @@ func Test_sync_with_co(t *testing.T) {
 				now:              func() metav1.Time { return now },
 			}
 
-			queueKey := configApiQueueKeys(co)[0]
+			queueKey := controlPlaneInformerQueueKeys(co)[0]
 
 			err := controller.sync(context.Background(), newTestSyncContext(queueKey))
 			if err != nil {

pkg/updatestatus/mco/controller_common_constants.go

@@ -0,0 +1,83 @@
+// Taken from: https://github.com/openshift/machine-config-operator/blob/11d5151a784c7d4be5255ea41acfbf5092eda592/pkg/controller/common/constants.go
+// TODO: Replace this file with the original MCO code when transitioning to server-side
+package mco
+
+const (
+	// MCONamespace is the namespace that should be used for all API objects owned by the MCO by default
+	MCONamespace = "openshift-machine-config-operator"
+
+	// GeneratedByControllerVersionAnnotationKey is used to tag the machineconfigs generated by the controller with the version of the controller.
+	GeneratedByControllerVersionAnnotationKey = "machineconfiguration.openshift.io/generated-by-controller-version"
+
+	// ReleaseImageVersionAnnotationKey is used to tag the rendered machineconfigs & controller config with the release image version.
+	ReleaseImageVersionAnnotationKey = "machineconfiguration.openshift.io/release-image-version"
+
+	// OSImageURLOverriddenKey is used to tag a rendered machineconfig when OSImageURL has been overridden from default using machineconfig
+	OSImageURLOverriddenKey = "machineconfiguration.openshift.io/os-image-url-overridden"
+
+	// ControllerConfigName is the name of the ControllerConfig object that controllers use
+	ControllerConfigName = "machine-config-controller"
+
+	// KernelTypeDefault denominates the default kernel type
+	KernelTypeDefault = "default"
+
+	// KernelTypeRealtime denominates the realtime kernel type
+	KernelTypeRealtime = "realtime"
+
+	// KernelType64kPages denominates the 64k pages kernel
+	KernelType64kPages = "64k-pages"
+
+	// MasterLabel defines the label associated with master node. The master taint uses the same label as taint's key
+	MasterLabel = "node-role.kubernetes.io/master"
+
+	// MCNameSuffixAnnotationKey is used to keep track of the machine config name associated with a CR
+	MCNameSuffixAnnotationKey = "machineconfiguration.openshift.io/mc-name-suffix"
+
+	// MaxMCNameSuffix is the maximum value of the name suffix of the machine config associated with kubeletconfig and containerruntime objects
+	MaxMCNameSuffix int = 9
+
+	// ClusterFeatureInstanceName is a singleton name for featureGate configuration
+	ClusterFeatureInstanceName = "cluster"
+
+	// ClusterNodeInstanceName is a singleton name for node configuration
+	ClusterNodeInstanceName = "cluster"
+
+	// MachineConfigPoolMaster is the MachineConfigPool name given to the master
+	MachineConfigPoolMaster = "master"
+
+	// MachineConfigPoolWorker is the MachineConfigPool name given to the worker
+	MachineConfigPoolWorker = "worker"
+
+	// LayeringEnabledPoolLabel is the label that enables the "layered" workflow path for a pool.
+	LayeringEnabledPoolLabel = "machineconfiguration.openshift.io/layering-enabled"
+
+	// ExperimentalNewestLayeredImageEquivalentConfigAnnotationKey is the annotation that signifies which rendered config
+	// TODO(zzlotnik): Determine if we should use this still.
+	ExperimentalNewestLayeredImageEquivalentConfigAnnotationKey = "machineconfiguration.openshift.io/newestImageEquivalentConfig"
+
+	OSImageBuildPodLabel = "machineconfiguration.openshift.io/buildPod"
+
+	// InternalMCOIgnitionVersion is the ignition version that the MCO converts everything to internally. The intent here is that
+	// we should be able to update this constant when we bump the internal ignition version instead of having to hunt down all of
+	// the version references and figure out "was this supposed to be explicitly 3.4.0 or just the default version which happens
+	// to be 3.4.0 currently". Ideally if you find an explicit "3.4.0", it's supposed to be "3.4.0" version. If it's this constant,
+	// it's supposed to be the internal default version.
+	InternalMCOIgnitionVersion = "3.4.0"
+
+	// MachineConfigRoleLabel is the role on MachineConfigs, used to select for pools
+	MachineConfigRoleLabel = "machineconfiguration.openshift.io/role"
+
+	// BootImagesConfigMapName is a Configmap of golden bootimages, updated by CVO on an upgrade
+	BootImagesConfigMapName = "coreos-bootimages"
+
+	// MCOVersionHashKey is the key for indexing the MCO git version hash stored in the bootimages configmap
+	MCOVersionHashKey = "MCOVersionHash"
+
+	// MCOReleaseImageVersionKey is the key for indexing the MCO release version stored in the bootimages configmap
+	MCOReleaseImageVersionKey = "MCOReleaseImageVersion"
+
+	ServiceCARotateAnnotation = "machineconfiguration.openshift.io/service-ca-rotate"
+
+	ServiceCARotateTrue  = "true"
+	ServiceCARotateFalse = "false"
+)

pkg/updatestatus/mco/daemon_constants.go

@@ -0,0 +1,110 @@
+// Taken from: https://github.com/openshift/machine-config-operator/blob/11d5151a784c7d4be5255ea41acfbf5092eda592/pkg/daemon/constants/constants.go
+// TODO: Replace this file with the original MCO code when transitioning to server-side
+package mco
+
+const (
+	// XXX
+	//
+	// Add a constant here, if and only if: it's exported (of course) and it's reused across the entire project.
+	// Otherwise, prefer an unexported const in a specific package.
+	//
+	// XXX
+
+	// CurrentImageAnnotationKey is used to get the current OS image pullspec for a machine
+	CurrentImageAnnotationKey = "machineconfiguration.openshift.io/currentImage"
+	// DesiredImageAnnotationKey is used to specify the desired OS image pullspec for a machine
+	DesiredImageAnnotationKey = "machineconfiguration.openshift.io/desiredImage"
+
+	// CurrentMachineConfigAnnotationKey is used to fetch current MachineConfig for a machine
+	CurrentMachineConfigAnnotationKey = "machineconfiguration.openshift.io/currentConfig"
+	// DesiredMachineConfigAnnotationKey is used to specify the desired MachineConfig for a machine
+	DesiredMachineConfigAnnotationKey = "machineconfiguration.openshift.io/desiredConfig"
+	// MachineConfigDaemonStateAnnotationKey is used to fetch the state of the daemon on the machine.
+	MachineConfigDaemonStateAnnotationKey = "machineconfiguration.openshift.io/state"
+	// DesiredDrainerAnnotationKey is set by the MCD to indicate drain/uncordon requests
+	DesiredDrainerAnnotationKey = "machineconfiguration.openshift.io/desiredDrain"
+	// LastAppliedDrainerAnnotationKey is set by the controller to indicate the last request applied
+	LastAppliedDrainerAnnotationKey = "machineconfiguration.openshift.io/lastAppliedDrain"
+	// DrainerStateDrain is used for drainer annotation as a value to indicate needing a drain
+	DrainerStateDrain = "drain"
+	// DrainerStateUncordon is used for drainer annotation as a value to indicate needing an uncordon
+	DrainerStateUncordon = "uncordon"
+	// ClusterControlPlaneTopologyAnnotationKey is set by the node controller by reading value from
+	// controllerConfig. MCD uses the annotation value to decide drain action on the node.
+	ClusterControlPlaneTopologyAnnotationKey = "machineconfiguration.openshift.io/controlPlaneTopology"
+	// OpenShiftOperatorManagedLabel is used to filter out kube objects that don't need to be synced by the MCO
+	OpenShiftOperatorManagedLabel = "openshift.io/operator-managed"
+	// ControllerConfigResourceVersionKey is used for the certificate writer to indicate the last controllerconfig object it synced upon
+	ControllerConfigResourceVersionKey = "machineconfiguration.openshift.io/lastSyncedControllerConfigResourceVersion"
+	// ControllerConfigSyncServerCA is used to determine if we have already synced the server CA for this version of the controller config
+	ControllerConfigSyncServerCA = "machineconfiguration.openshift.io/lastObservedServerCAAnnotation"
+	// GeneratedByVersionAnnotationKey is used to tag the controllerconfig to synchronize the MCO and MCC
+	GeneratedByVersionAnnotationKey = "machineconfiguration.openshift.io/generated-by-version"
+
+	// MachineConfigDaemonStateWorking is set by daemon when it is beginning to apply an update.
+	MachineConfigDaemonStateWorking = "Working"
+	// MachineConfigDaemonStateDone is set by daemon when it is done applying an update.
+	MachineConfigDaemonStateDone = "Done"
+	// MachineConfigDaemonStateDegraded is set by daemon when an error not caused by a bad MachineConfig
+	// is thrown during an update.
+	MachineConfigDaemonStateDegraded = "Degraded"
+	// MachineConfigDaemonRebooting is used to indicate a reboot is either queued or is in progress.
+	MachineConfigDaemonStateRebooting = "Rebooting"
+	// MachineConfigDaemonStateUnreconcilable is set by the daemon when a MachineConfig cannot be applied.
+	MachineConfigDaemonStateUnreconcilable = "Unreconcilable"
+	// MachineConfigDaemonReasonAnnotationKey is set by the daemon when it needs to report a human readable reason for its state. E.g. when state flips to degraded/unreconcilable.
+	MachineConfigDaemonReasonAnnotationKey = "machineconfiguration.openshift.io/reason"
+	// MachineConfigDaemonPostConfigAction is set by the daemon when it needs to report a human readable post config action that takes place during update.
+	MachineConfigDaemonPostConfigAction = "machineconfiguration.openshift.io/post-config-action"
+	// MachineConfigDaemonFinalizeFailureAnnotationKey is set by the daemon when ostree fails to finalize
+	MachineConfigDaemonFinalizeFailureAnnotationKey = "machineconfiguration.openshift.io/ostree-finalize-staged-failure"
+	// InitialNodeAnnotationsFilePath defines the path at which it will find the node annotations it needs to set on the node once it comes up for the first time.
+	// The Machine Config Server writes the node annotations to this path.
+	InitialNodeAnnotationsFilePath = "/etc/machine-config-daemon/node-annotations.json"
+	// InitialNodeAnnotationsBakPath defines the path of InitialNodeAnnotationsFilePath when the initial bootstrap is done. We leave it around for debugging and reconciling.
+	InitialNodeAnnotationsBakPath = "/etc/machine-config-daemon/node-annotation.json.bak"
+
+	// IgnitionSystemdPresetFile is where Ignition writes initial enabled/disabled systemd unit configs
+	// This should be removed on boot after MCO takes over, so if any of these are deleted we can go back
+	// to initial system settings
+	IgnitionSystemdPresetFile = "/etc/systemd/system-preset/20-ignition.preset"
+
+	// EtcPivotFile is used by the `pivot` command
+	// For more information, see https://github.com/openshift/pivot/pull/25/commits/c77788a35d7ee4058d1410e89e6c7937bca89f6c#diff-04c6e90faac2675aa89e2176d2eec7d8R44
+	EtcPivotFile = "/etc/pivot/image-pullspec"
+
+	// MachineConfigEncapsulatedPath contains all of the data from a MachineConfig object
+	// except the Spec/Config object; this supports inverting+encapsulating a MachineConfig
+	// object so that Ignition can process it on first boot, and then the MCD can act on
+	// non-Ignition fields such as the osImageURL and kernelArguments.
+	MachineConfigEncapsulatedPath = "/etc/ignition-machine-config-encapsulated.json"
+
+	// MachineConfigEncapsulatedBakPath defines the path where the machineconfigdaemom-firstboot.service
+	// will leave a copy of the encapsulated MachineConfig in MachineConfigEncapsulatedPath after
+	// processing for debugging and auditing purposes.
+	MachineConfigEncapsulatedBakPath = "/etc/ignition-machine-config-encapsulated.json.bak"
+
+	// MachineConfigDaemonForceFile if present causes the MCD to skip checking the validity of the
+	// "currentConfig" state.  Create this file (empty contents is fine) if you wish the MCD
+	// to proceed and attempt to "reconcile" to the new "desiredConfig" state regardless.
+	MachineConfigDaemonForceFile = "/run/machine-config-daemon-force"
+
+	// coreUser is "core" and currently the only permissible user name
+	CoreUserName  = "core"
+	CoreGroupName = "core"
+
+	// changes to registries.conf will cause a crio reload and require extra logic about whether to drain
+	ContainerRegistryConfPath = "/etc/containers/registries.conf"
+
+	// SSH Keys for user "core" will only be written at /home/core/.ssh
+	CoreUserSSHPath = "/home/" + CoreUserName + "/.ssh"
+
+	// SSH keys in RHCOS 8 will be written to /home/core/.ssh/authorized_keys
+	RHCOS8SSHKeyPath = CoreUserSSHPath + "/authorized_keys"
+
+	// SSH keys in RHCOS 9 / FCOS / SCOS will be written to /home/core/.ssh/authorized_keys.d/ignition
+	RHCOS9SSHKeyPath = CoreUserSSHPath + "/authorized_keys.d/ignition"
+
+	// CRIOServiceName is used to specify reloads and restarts of the CRI-O service
+	CRIOServiceName = "crio"
+)