agent-day2: retrieve ignition endpoint to add a new node

This patch allows using the secure port, if configured, in the target cluster when adding a new node

Author: andfasano

data/data/agent/systemd/units/agent-import-cluster.service.template

@@ -16,7 +16,7 @@ EnvironmentFile=/usr/local/share/assisted-service/assisted-service.env
 EnvironmentFile=/etc/assisted/add-nodes.env
 ExecStartPre=/bin/rm -f %t/%n.ctr-id
 ExecStartPre=/usr/local/bin/wait-for-assisted-service.sh
-ExecStart=podman run --net host --cidfile=%t/%n.ctr-id --cgroups=no-conmon --log-driver=journald --rm --pod-id-file=%t/assisted-service-pod.pod-id --replace --name=agent-import-cluster -v /etc/assisted/manifests:/manifests -v /etc/assisted/extra-manifests:/extra-manifests {{ if .HaveMirrorConfig }}-v /etc/containers:/etc/containers{{ end }} {{.CaBundleMount}} --env SERVICE_BASE_URL --env OPENSHIFT_INSTALL_RELEASE_IMAGE_MIRROR --env CLUSTER_ID --env CLUSTER_NAME --env CLUSTER_API_VIP_DNS_NAME $SERVICE_IMAGE /usr/local/bin/agent-installer-client importCluster
+ExecStart=podman run --net host --cidfile=%t/%n.ctr-id --cgroups=no-conmon --log-driver=journald --rm --pod-id-file=%t/assisted-service-pod.pod-id --replace --name=agent-import-cluster -v /etc/assisted/clusterconfig:/clusterconfig -v /etc/assisted/manifests:/manifests -v /etc/assisted/extra-manifests:/extra-manifests {{ if .HaveMirrorConfig }}-v /etc/containers:/etc/containers{{ end }} {{.CaBundleMount}} --env SERVICE_BASE_URL --env OPENSHIFT_INSTALL_RELEASE_IMAGE_MIRROR --env CLUSTER_ID --env CLUSTER_NAME --env CLUSTER_API_VIP_DNS_NAME $SERVICE_IMAGE /usr/local/bin/agent-installer-client importCluster
 ExecStop=/usr/bin/podman stop --ignore --cidfile=%t/%n.ctr-id
 ExecStopPost=/usr/bin/podman rm -f --ignore --cidfile=%t/%n.ctr-id
 

pkg/asset/agent/image/ignition.go

@@ -2,6 +2,7 @@ package image
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"net"
 	"net/url"
@@ -46,6 +47,7 @@ const nmConnectionsPath = "/etc/assisted/network"
 const extraManifestPath = "/etc/assisted/extra-manifests"
 const registriesConfPath = "/etc/containers/registries.conf"
 const registryCABundlePath = "/etc/pki/ca-trust/source/anchors/domain.crt"
+const clusterConfigPath = "/etc/assisted/clusterconfig"
 
 // Ignition is an asset that generates the agent installer ignition file.
 type Ignition struct {
@@ -195,6 +197,10 @@ func (a *Ignition) Generate(dependencies asset.Parents) error {
 		streamGetter = func(ctx context.Context) (*stream.Stream, error) {
 			return clusterInfo.OSImage, nil
 		}
+		// If defined, add the ignition endpoints
+		if err := addDay2IgnitionEndpoints(&config, *clusterInfo); err != nil {
+			return err
+		}
 
 	default:
 		return fmt.Errorf("AgentWorkflowType value not supported: %s", agentWorkflow.Workflow)
@@ -531,6 +537,35 @@ func addHostConfig(config *igntypes.Config, agentHosts *agentconfig.AgentHosts)
 	return nil
 }
 
+func addDay2IgnitionEndpoints(config *igntypes.Config, clusterInfo joiner.ClusterInfo) error {
+	if clusterInfo.IgnitionEndpointWorker == nil {
+		return nil
+	}
+
+	user := "root"
+	mode := 0644
+	config.Storage.Directories = append(config.Storage.Directories, igntypes.Directory{
+		Node: igntypes.Node{
+			Path: clusterConfigPath,
+			User: igntypes.NodeUser{
+				Name: &user,
+			},
+			Overwrite: util.BoolToPtr(true),
+		},
+		DirectoryEmbedded1: igntypes.DirectoryEmbedded1{
+			Mode: &mode,
+		},
+	})
+
+	workerIgnitionBytes, err := json.Marshal(clusterInfo.IgnitionEndpointWorker)
+	if err != nil {
+		return err
+	}
+	workerIgnitionFile := ignition.FileFromBytes(path.Join(clusterConfigPath, "worker-ignition-endpoint.json"), user, mode, workerIgnitionBytes)
+	config.Storage.Files = append(config.Storage.Files, workerIgnitionFile)
+	return nil
+}
+
 func addExtraManifests(config *igntypes.Config, extraManifests *manifests.ExtraManifests) error {
 
 	user := "root"

pkg/asset/agent/joiner/clusterinfo.go

@@ -4,7 +4,9 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"strings"
 
+	igntypes "github.com/coreos/ignition/v2/config/v3_2/types"
 	"github.com/coreos/stream-metadata-go/arch"
 	"github.com/coreos/stream-metadata-go/stream"
 	"k8s.io/apimachinery/pkg/api/errors"
@@ -15,6 +17,7 @@ import (
 	"sigs.k8s.io/yaml"
 
 	hiveext "github.com/openshift/assisted-service/api/hiveextension/v1beta1"
+	"github.com/openshift/assisted-service/models"
 	configclient "github.com/openshift/client-go/config/clientset/versioned"
 	"github.com/openshift/installer/pkg/asset"
 	"github.com/openshift/installer/pkg/asset/agent"
@@ -45,6 +48,7 @@ type ClusterInfo struct {
 	SSHKey                        string
 	OSImage                       *stream.Stream
 	OSImageLocation               string
+	IgnitionEndpointWorker        *models.IgnitionEndpoint
 }
 
 var _ asset.WritableAsset = (*ClusterInfo)(nil)
@@ -106,6 +110,10 @@ func (ci *ClusterInfo) Generate(dependencies asset.Parents) error {
 	if err != nil {
 		return err
 	}
+	err = ci.retrieveIgnitionEndpointWorker()
+	if err != nil {
+		return err
+	}
 
 	ci.Namespace = "cluster0"
 
@@ -269,6 +277,48 @@ func (ci *ClusterInfo) retrieveOsImage() error {
 	return nil
 }
 
+// This method retrieves, if present, the secured ignition endpoint - along with its ca certificate.
+// These information will be used to configure subsequently the imported Assisted Service cluster,
+// so that the secure port (22623) could be used by the nodes to fetch the worker ignition.
+func (ci *ClusterInfo) retrieveIgnitionEndpointWorker() error {
+	workerUserDataManaged, err := ci.Client.CoreV1().Secrets("openshift-machine-api").Get(context.Background(), "worker-user-data-managed", metav1.GetOptions{})
+	if err != nil {
+		if errors.IsNotFound(err) {
+			return nil
+		}
+		return err
+	}
+	userData := workerUserDataManaged.Data["userData"]
+
+	config := &igntypes.Config{}
+	err = json.Unmarshal(userData, config)
+	if err != nil {
+		return err
+	}
+
+	// Check if there is at least a CA certificate in the ignition
+	if len(config.Ignition.Security.TLS.CertificateAuthorities) == 0 {
+		return nil
+	}
+
+	// Get the first source and ca certificate (and strip the base64 data header)
+	ignEndpointURL := config.Ignition.Config.Merge[0].Source
+	caCertSource := *config.Ignition.Security.TLS.CertificateAuthorities[0].Source
+
+	hdrIndex := strings.Index(caCertSource, ",")
+	if hdrIndex == -1 {
+		return fmt.Errorf("error while parsing ignition endpoints ca certificate, invalid data url format")
+	}
+	caCert := caCertSource[hdrIndex+1:]
+
+	ci.IgnitionEndpointWorker = &models.IgnitionEndpoint{
+		URL:           ignEndpointURL,
+		CaCertificate: &caCert,
+	}
+
+	return nil
+}
+
 // Files returns the files generated by the asset.
 func (*ClusterInfo) Files() []*asset.File {
 	return []*asset.File{}

pkg/asset/agent/joiner/clusterinfo_test.go

@@ -10,10 +10,12 @@ import (
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/kubernetes/fake"
+	"k8s.io/utils/ptr"
 	"sigs.k8s.io/yaml"
 
 	configv1 "github.com/openshift/api/config/v1"
 	"github.com/openshift/assisted-service/api/hiveextension/v1beta1"
+	"github.com/openshift/assisted-service/models"
 	fakeclientconfig "github.com/openshift/client-go/config/clientset/versioned/fake"
 	"github.com/openshift/installer/pkg/asset"
 	"github.com/openshift/installer/pkg/asset/agent/workflow"
@@ -114,6 +116,17 @@ func TestClusterInfo_Generate(t *testing.T) {
 						"stream": makeCoreOsBootImages(t, buildStreamData()),
 					},
 				},
+				&corev1.Secret{
+					ObjectMeta: v1.ObjectMeta{
+						Name:      "worker-user-data-managed",
+						Namespace: "openshift-machine-api",
+					},
+					Data: map[string][]byte{
+						"userData": []byte(`{"ignition":{"config":{"merge":[{"source":"https://192.168.111.5:22623/config/worker","verification":{}}],
+"replace":{"verification":{}}},"proxy":{},"security":{"tls":{"certificateAuthorities":[{"source":"data:text/plain;charset=utf-8;base64,LS0tL_FakeCertificate_LS0tCg==",
+"verification":{}}]}},"timeouts":{},"version":"3.4.0"},"kernelArguments":{},"passwd":{},"storage":{},"systemd":{}}`),
+					},
+				},
 			},
 			expectedClusterInfo: ClusterInfo{
 				ClusterID:    "1b5ba46b-7e56-47b1-a326-a9eebddfb38c",
@@ -142,6 +155,10 @@ func TestClusterInfo_Generate(t *testing.T) {
 				SSHKey:          "my-ssh-key",
 				OSImage:         buildStreamData(),
 				OSImageLocation: "http://my-coreosimage-url/416.94.202402130130-0",
+				IgnitionEndpointWorker: &models.IgnitionEndpoint{
+					URL:           ptr.To("https://192.168.111.5:22623/config/worker"),
+					CaCertificate: ptr.To("LS0tL_FakeCertificate_LS0tCg=="),
+				},
 			},
 		},
 	}
@@ -179,6 +196,7 @@ func TestClusterInfo_Generate(t *testing.T) {
 			assert.Equal(t, tc.expectedClusterInfo.SSHKey, clusterInfo.SSHKey)
 			assert.Equal(t, tc.expectedClusterInfo.OSImageLocation, clusterInfo.OSImageLocation)
 			assert.Equal(t, tc.expectedClusterInfo.OSImage, clusterInfo.OSImage)
+			assert.Equal(t, tc.expectedClusterInfo.IgnitionEndpointWorker, clusterInfo.IgnitionEndpointWorker)
 		})
 	}
 }