Merge pull request openshift#7061 from zaneb/rendezvous-host-env

openshift-merge-robot · web-flow · commit c2b00a80a545 · 2023-05-30T07:14:03.000-04:00
AGENT-555: Move Rendezvous Host config to separate file
diff --git a/data/data/agent/files/etc/assisted/agent-installer.env.template b/data/data/agent/files/etc/assisted/agent-installer.env.template
@@ -1,13 +1 @@
-# If NODE_ZERO_IP is changed, additional files need to be updated
-# before starting up the agent-based installer services:
-# 
-# In /usr/local/share/assisted-service/assisted-service.env,
-# IMAGE_SERVICE_BASE_URL and SERVICE_BASE_URL must be updated.
-#
-# In /usr/local/share/assisted-service/images.env,
-# ASSISTED_SERVICE_HOST must be updated.
-#   
-NODE_ZERO_IP={{.NodeZeroIP}}
-LD_LIBRARY_PATH=/usr/local/bin/
 RELEASE_IMAGE={{.ReleaseImage}}
-AGENT_TUI_LOG_PATH=/var/log/agent/agent-tui.log
diff --git a/data/data/agent/files/usr/local/bin/common.sh b/data/data/agent/files/usr/local/bin/common.sh
diff --git a/data/data/agent/files/usr/local/bin/set-node-zero.sh b/data/data/agent/files/usr/local/bin/set-node-zero.sh
@@ -3,9 +3,19 @@
 set -e
 
 # shellcheck disable=SC1091
-source common.sh
+source /etc/assisted/rendezvous-host.env
 echo "NODE_ZERO_IP: $NODE_ZERO_IP"
 
+is_node_zero() {
+    local is_rendezvous_host
+    is_rendezvous_host=$(ip -j address | jq "[.[].addr_info] | flatten | map(.local==\"$NODE_ZERO_IP\") | any")
+    if [[ "${is_rendezvous_host}" == "true" ]]; then
+        echo 1
+    else
+        echo 0
+    fi
+}
+
 timeout=$((SECONDS + 30))
 
 while [[ $SECONDS -lt $timeout ]]
@@ -29,7 +39,7 @@ if [ "${IS_NODE_ZERO}" = "true" ]; then
     cat >"${NODE0_PATH}" <<EOF
 # This file exists if the agent-based installer has determined the host is node 0.
 # The host is determined to be node 0 when one of its network interfaces has an 
-# IP address matching NODE_ZERO_IP in /etc/assisted/agent-installer.env. 
+# IP address matching NODE_ZERO_IP in /etc/assisted/rendezvous-host.env. 
 # The MAC address of the network interface matching NODE_ZERO_IP is noted below 
 # as BOOTSTRAP_HOST_MAC.
 #
diff --git a/data/data/agent/files/usr/local/bin/start-agent.sh b/data/data/agent/files/usr/local/bin/start-agent.sh
@@ -1,8 +1,5 @@
 #!/bin/bash
 
-# shellcheck disable=SC1091
-source common.sh
-
 >&2 echo "Waiting for infra-env-id to be available"
 INFRA_ENV_ID=""
 until [[ $INFRA_ENV_ID != "" && $INFRA_ENV_ID != "null" ]]; do
diff --git a/data/data/agent/files/usr/local/bin/start-cluster-installation.sh.template b/data/data/agent/files/usr/local/bin/start-cluster-installation.sh.template
@@ -1,11 +1,8 @@
 #!/bin/bash
 set -e
 
-source common.sh
 source issue_status.sh
 
-wait_for_assisted_service
-
 BASE_URL="${SERVICE_BASE_URL}api/assisted-install/v2"
 
 cluster_id=""
diff --git a/data/data/agent/files/usr/local/bin/wait-for-assisted-service.sh b/data/data/agent/files/usr/local/bin/wait-for-assisted-service.sh
@@ -1,7 +1,8 @@
 #!/bin/bash
 set -e
 
-# shellcheck disable=SC1091
-source common.sh
-
-wait_for_assisted_service
+echo "Waiting for assisted-service to be ready"
+until curl --output /dev/null --silent --fail "${SERVICE_BASE_URL}/api/assisted-install/v2/infra-envs"; do
+    printf '.'
+    sleep 5
+done
diff --git a/data/data/agent/files/usr/local/share/assisted-service/assisted-service.env.template b/data/data/agent/files/usr/local/share/assisted-service/assisted-service.env.template
@@ -10,13 +10,11 @@ DUMMY_IGNITION=false
 ENABLE_SINGLE_NODE_DNSMASQ=true
 EPHEMERAL_INSTALLER_CLUSTER_TLS_CERTS_OVERRIDE_DIR=/opt/agent/tls
 HW_VALIDATOR_REQUIREMENTS=[{"version":"default","master":{"cpu_cores":4,"ram_mib":16384,"disk_size_gb":100,"installation_disk_speed_threshold_ms":10,"network_latency_threshold_ms":100,"packet_loss_percentage":0},"worker":{"cpu_cores":2,"ram_mib":8192,"disk_size_gb":100,"installation_disk_speed_threshold_ms":10,"network_latency_threshold_ms":1000,"packet_loss_percentage":10},"sno":{"cpu_cores":8,"ram_mib":16384,"disk_size_gb":100,"installation_disk_speed_threshold_ms":10}}]
-IMAGE_SERVICE_BASE_URL=http://{{.NodeZeroIP}}:8888
 INSTALL_INVOKER=agent-installer
 IPV6_SUPPORT=true
 NTP_DEFAULT_SERVER=
 PUBLIC_CONTAINER_REGISTRIES={{.PublicContainerRegistries}}
 RELEASE_IMAGES={{.ReleaseImages}}
 OPENSHIFT_INSTALL_RELEASE_IMAGE_MIRROR={{.ReleaseImageMirror}}
-SERVICE_BASE_URL={{.ServiceBaseURL}}
 STORAGE=filesystem
 INFRA_ENV_ID={{.InfraEnvID}}
diff --git a/data/data/agent/files/usr/local/share/assisted-service/images.env.template b/data/data/agent/files/usr/local/share/assisted-service/images.env.template
@@ -1,3 +1,3 @@
-ASSISTED_SERVICE_HOST={{.AssistedServiceHost}}
+ASSISTED_SERVICE_HOST=localhost:8090
 ASSISTED_SERVICE_SCHEME={{.ServiceProtocol}}
 OS_IMAGES=[{"openshift_version":"{{.OSImage.OpenshiftVersion}}","cpu_architecture":"{{.OSImage.CPUArchitecture}}","url":"{{.OSImage.URL}}","version":"{{.OSImage.Version}}"}]
diff --git a/data/data/agent/systemd/units/agent-interactive-console.service b/data/data/agent/systemd/units/agent-interactive-console.service
@@ -10,6 +10,8 @@ Type=oneshot
 RemainAfterExit=true
 TTYPath=/dev/tty15
 EnvironmentFile=/etc/assisted/agent-installer.env
+Environment=LD_LIBRARY_PATH=/usr/local/bin/
+Environment=AGENT_TUI_LOG_PATH=/var/log/agent/agent-tui.log
 ExecStartPre=/usr/bin/kill -s SIGRTMIN+21 1
 ExecStartPre=/usr/bin/chvt 15
 ExecStartPre=mkdir -p /var/log/agent
diff --git a/data/data/agent/systemd/units/agent.service.template b/data/data/agent/systemd/units/agent.service.template
@@ -4,6 +4,7 @@ Restart=always
 RestartSec=3
 StartLimitInterval=0
 TimeoutStartSec=3000
+EnvironmentFile=/etc/assisted/rendezvous-host.env
 ExecStartPre=/usr/local/bin/extract-agent.sh
 ExecStart=/usr/local/bin/start-agent.sh
 
diff --git a/data/data/agent/systemd/units/apply-host-config.service b/data/data/agent/systemd/units/apply-host-config.service
@@ -8,6 +8,7 @@ ConditionPathExists=/etc/assisted/node0
 
 [Service]
 Environment=PODMAN_SYSTEMD_UNIT=%n
+EnvironmentFile=/etc/assisted/rendezvous-host.env
 EnvironmentFile=/usr/local/share/assisted-service/agent-images.env
 EnvironmentFile=/usr/local/share/assisted-service/assisted-service.env
 ExecStartPre=/bin/rm -f %t/%n.ctr-id
diff --git a/data/data/agent/systemd/units/assisted-service.service.template b/data/data/agent/systemd/units/assisted-service.service.template
@@ -13,7 +13,7 @@ Restart=on-failure
 TimeoutStartSec=500
 TimeoutStopSec=300
 ExecStartPre=/bin/rm -f %t/%n.ctr-id
-ExecStart=/usr/bin/podman run --net host --cidfile=%t/%n.ctr-id --cgroups=no-conmon --log-driver=journald --rm --pod-id-file=%t/assisted-service-pod.pod-id --sdnotify=conmon --replace -d --name=service -v /opt/agent/tls:/opt/agent/tls:z {{ if .HaveMirrorConfig }}-v /etc/containers:/etc/containers{{ end }} -v /etc/pki/ca-trust:/etc/pki/ca-trust --env-file=/usr/local/share/assisted-service/assisted-service.env --env-file=/usr/local/share/assisted-service/images.env --env-file=/etc/assisted/node0 --env-file=/usr/local/share/assisted-service/agent-images.env $SERVICE_IMAGE
+ExecStart=/usr/bin/podman run --net host --cidfile=%t/%n.ctr-id --cgroups=no-conmon --log-driver=journald --rm --pod-id-file=%t/assisted-service-pod.pod-id --sdnotify=conmon --replace -d --name=service -v /opt/agent/tls:/opt/agent/tls:z {{ if .HaveMirrorConfig }}-v /etc/containers:/etc/containers{{ end }} -v /etc/pki/ca-trust:/etc/pki/ca-trust --env-file=/etc/assisted/rendezvous-host.env --env-file=/usr/local/share/assisted-service/assisted-service.env --env-file=/usr/local/share/assisted-service/images.env --env-file=/etc/assisted/node0 --env-file=/usr/local/share/assisted-service/agent-images.env $SERVICE_IMAGE
 ExecStop=/usr/bin/podman stop --ignore --cidfile=%t/%n.ctr-id
 ExecStopPost=/usr/bin/podman rm -f --ignore --cidfile=%t/%n.ctr-id
 Type=notify
diff --git a/data/data/agent/systemd/units/create-cluster-and-infraenv.service.template b/data/data/agent/systemd/units/create-cluster-and-infraenv.service.template
@@ -8,6 +8,7 @@ ConditionPathExists=/etc/assisted/node0
 [Service]
 Environment=PODMAN_SYSTEMD_UNIT=%n
 Environment=OPENSHIFT_INSTALL_RELEASE_IMAGE_MIRROR={{.ReleaseImageMirror}}
+EnvironmentFile=/etc/assisted/rendezvous-host.env
 EnvironmentFile=/usr/local/share/assisted-service/agent-images.env
 EnvironmentFile=/usr/local/share/assisted-service/assisted-service.env
 ExecStartPre=/bin/rm -f %t/%n.ctr-id
diff --git a/data/data/agent/systemd/units/start-cluster-installation.service b/data/data/agent/systemd/units/start-cluster-installation.service
@@ -7,6 +7,9 @@ After=network-online.target apply-host-config.service
 ConditionPathExists=/etc/assisted/node0
 
 [Service]
+EnvironmentFile=/usr/local/share/assisted-service/assisted-service.env
+EnvironmentFile=/etc/assisted/rendezvous-host.env
+ExecStartPre=/usr/local/bin/wait-for-assisted-service.sh
 ExecStart=/usr/local/bin/start-cluster-installation.sh
 
 KillMode=none
diff --git a/pkg/asset/agent/image/ignition.go b/pkg/asset/agent/image/ignition.go
@@ -34,6 +34,7 @@ import (
 	"github.com/openshift/installer/pkg/version"
 )
 
+const rendezvousHostEnvPath = "/etc/assisted/rendezvous-host.env"
 const manifestPath = "/etc/assisted/manifests"
 const hostnamesPath = "/etc/assisted/hostnames"
 const nmConnectionsPath = "/etc/assisted/network"
@@ -50,10 +51,7 @@ type Ignition struct {
 // files.
 type agentTemplateData struct {
 	ServiceProtocol           string
-	ServiceBaseURL            string
 	PullSecret                string
-	NodeZeroIP                string
-	AssistedServiceHost       string
 	APIVIP                    string
 	ControlPlaneAgents        int
 	WorkerAgents              int
@@ -162,7 +160,6 @@ func (a *Ignition) Generate(dependencies asset.Parents) error {
 	agentTemplateData := getTemplateData(
 		clusterName,
 		agentManifests.GetPullSecretData(),
-		nodeZeroIP,
 		releaseImageList,
 		agentManifests.ClusterImageSet.Spec.ReleaseImage,
 		releaseImageMirror,
@@ -178,6 +175,11 @@ func (a *Ignition) Generate(dependencies asset.Parents) error {
 		return err
 	}
 
+	rendezvousHostFile := ignition.FileFromString(rendezvousHostEnvPath,
+		"root", 0644,
+		getRendezvousHostEnv(agentTemplateData.ServiceProtocol, nodeZeroIP))
+	config.Storage.Files = append(config.Storage.Files, rendezvousHostFile)
+
 	// Set up bootstrap service recording
 	if err := bootstrap.AddStorageFiles(&config,
 		"/usr/local/bin/bootstrap-service-record.sh",
@@ -260,24 +262,15 @@ func (a *Ignition) Generate(dependencies asset.Parents) error {
 	return nil
 }
 
-func getTemplateData(name, pullSecret, nodeZeroIP, releaseImageList, releaseImage,
+func getTemplateData(name, pullSecret, releaseImageList, releaseImage,
 	releaseImageMirror string, haveMirrorConfig bool, publicContainerRegistries string,
 	agentClusterInstall *hiveext.AgentClusterInstall,
 	infraEnvID string,
 	osImage *models.OsImage,
 	proxy *v1beta1.Proxy) *agentTemplateData {
-	serviceBaseURL := url.URL{
-		Scheme: "http",
-		Host:   net.JoinHostPort(nodeZeroIP, "8090"),
-		Path:   "/",
-	}
-
 	return &agentTemplateData{
-		ServiceProtocol:           serviceBaseURL.Scheme,
-		ServiceBaseURL:            serviceBaseURL.String(),
+		ServiceProtocol:           "http",
 		PullSecret:                pullSecret,
-		NodeZeroIP:                serviceBaseURL.Hostname(),
-		AssistedServiceHost:       serviceBaseURL.Host,
 		APIVIP:                    agentClusterInstall.Spec.APIVIP,
 		ControlPlaneAgents:        agentClusterInstall.Spec.ProvisionRequirements.ControlPlaneAgents,
 		WorkerAgents:              agentClusterInstall.Spec.ProvisionRequirements.WorkerAgents,
@@ -293,6 +286,24 @@ func getTemplateData(name, pullSecret, nodeZeroIP, releaseImageList, releaseImag
 	}
 }
 
+func getRendezvousHostEnv(serviceProtocol, nodeZeroIP string) string {
+	serviceBaseURL := url.URL{
+		Scheme: serviceProtocol,
+		Host:   net.JoinHostPort(nodeZeroIP, "8090"),
+		Path:   "/",
+	}
+	imageServiceBaseURL := url.URL{
+		Scheme: serviceProtocol,
+		Host:   net.JoinHostPort(nodeZeroIP, "8888"),
+		Path:   "/",
+	}
+
+	return fmt.Sprintf(`NODE_ZERO_IP=%s
+SERVICE_BASE_URL=%s
+IMAGE_SERVICE_BASE_URL=%s
+`, nodeZeroIP, serviceBaseURL.String(), imageServiceBaseURL.String())
+}
+
 func addStaticNetworkConfig(config *igntypes.Config, staticNetworkConfig []*models.HostStaticNetworkConfig) (err error) {
 	if len(staticNetworkConfig) == 0 {
 		return nil
diff --git a/pkg/asset/agent/image/ignition_test.go b/pkg/asset/agent/image/ignition_test.go
@@ -45,7 +45,6 @@ func TestIgnition_getTemplateData(t *testing.T) {
 		},
 	}
 	pullSecret := "pull-secret"
-	nodeZeroIP := "192.168.111.80"
 	agentClusterInstall := &hiveext.AgentClusterInstall{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "test-agent-cluster-install",
@@ -87,13 +86,10 @@ func TestIgnition_getTemplateData(t *testing.T) {
 	}
 	clusterName := "test-agent-cluster-install.test"
 
-	templateData := getTemplateData(clusterName, pullSecret, nodeZeroIP, releaseImageList, releaseImage, releaseImageMirror, haveMirrorConfig, publicContainerRegistries, agentClusterInstall, infraEnvID, osImage, proxy)
+	templateData := getTemplateData(clusterName, pullSecret, releaseImageList, releaseImage, releaseImageMirror, haveMirrorConfig, publicContainerRegistries, agentClusterInstall, infraEnvID, osImage, proxy)
 	assert.Equal(t, clusterName, templateData.ClusterName)
 	assert.Equal(t, "http", templateData.ServiceProtocol)
-	assert.Equal(t, "http://"+nodeZeroIP+":8090/", templateData.ServiceBaseURL)
 	assert.Equal(t, pullSecret, templateData.PullSecret)
-	assert.Equal(t, nodeZeroIP, templateData.NodeZeroIP)
-	assert.Equal(t, nodeZeroIP+":8090", templateData.AssistedServiceHost)
 	assert.Equal(t, agentClusterInstall.Spec.APIVIP, templateData.APIVIP)
 	assert.Equal(t, agentClusterInstall.Spec.ProvisionRequirements.ControlPlaneAgents, templateData.ControlPlaneAgents)
 	assert.Equal(t, agentClusterInstall.Spec.ProvisionRequirements.WorkerAgents, templateData.WorkerAgents)
@@ -107,6 +103,14 @@ func TestIgnition_getTemplateData(t *testing.T) {
 	assert.Equal(t, proxy, templateData.Proxy)
 }
 
+func TestIgnition_getRendezvousHostEnv(t *testing.T) {
+	nodeZeroIP := "2001:db8::dead:beef"
+	rendezvousHostEnv := getRendezvousHostEnv("http", nodeZeroIP)
+	assert.Equal(t,
+		"NODE_ZERO_IP="+nodeZeroIP+"\nSERVICE_BASE_URL=http://["+nodeZeroIP+"]:8090/\nIMAGE_SERVICE_BASE_URL=http://["+nodeZeroIP+"]:8888/\n",
+		rendezvousHostEnv)
+}
+
 func TestIgnition_addStaticNetworkConfig(t *testing.T) {
 	_, execErr := exec.LookPath("nmstatectl")
 	if execErr != nil {
@@ -313,7 +317,6 @@ func generatedFiles(otherFiles ...string) []string {
 		"/etc/containers/containers.conf",
 		"/root/.docker/config.json",
 		"/root/assisted.te",
-		"/usr/local/bin/common.sh",
 		"/usr/local/bin/agent-gather",
 		"/usr/local/bin/extract-agent.sh",
 		"/usr/local/bin/get-container-images.sh",
@@ -328,6 +331,7 @@ func generatedFiles(otherFiles ...string) []string {
 		"/usr/local/bin/bootstrap-service-record.sh",
 		"/usr/local/bin/release-image.sh",
 		"/usr/local/bin/release-image-download.sh",
+		"/etc/assisted/rendezvous-host.env",
 		"/etc/assisted/manifests/agent-config.yaml",
 		"/etc/assisted/network/host0/eth0.nmconnection",
 		"/etc/assisted/network/host0/mac_interface.ini",
