Modify startup scripts for ovn-controller-ovs

In order to minimize the downtime during update of the
ovn-controller-ovs pods we're modifying the update strategy so
it doesn't accept any Unavailable pod. This means that it will
create new ovn-controller-ovs while the old one is running.

This commit enables that two ovn-controller-ovs pods can coexists.
It accomplish this by modifying the start-up scripts of all containers
so it checks if a pod is already running and slowly stops in a
controlled fashon the old pods while it starts the new ones.

The logic is done with a temporary file created during the init
container that will inform the ovsdb-server/ovs-vswitchd containers
if they're on an update scenario or a normal one. The temporary file
is deleted after the end of the ovs-vswitchd so when the ovnController
CR is deleted, the volumes won't have any leftovers.

Related: OSPRH-11636
Jira: OSPRH-10821
Depends-on: lib-common#611

Author: averdagu

pkg/ovncontroller/daemonset.go

@@ -24,6 +24,7 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/utils/ptr"
 )
 
@@ -236,6 +237,27 @@ func CreateOVSDaemonSet(
 	envVars := map[string]env.Setter{}
 	envVars["CONFIG_HASH"] = env.SetValue(configHash)
 
+	volumes := []corev1.Volume{}
+	mounts := []corev1.VolumeMount{}
+
+	// add OVN dbs cert and CA
+	if instance.Spec.TLS.Enabled() {
+		svc := tls.Service{
+			SecretName: *instance.Spec.TLS.GenericService.SecretName,
+			CertMount:  ptr.To(ovn_common.OVNDbCertPath),
+			KeyMount:   ptr.To(ovn_common.OVNDbKeyPath),
+			CaMount:    ptr.To(ovn_common.OVNDbCaCertPath),
+		}
+		volumes = append(volumes, svc.CreateVolume(ovnv1.ServiceNameOVS))
+		mounts = append(mounts, svc.CreateVolumeMounts(ovnv1.ServiceNameOVS)...)
+
+		// add CA bundle if defined
+		if instance.Spec.TLS.CaBundleSecretName != "" {
+			volumes = append(volumes, instance.Spec.TLS.CreateVolume())
+			mounts = append(mounts, instance.Spec.TLS.CreateVolumeMounts(nil)...)
+		}
+	}
+
 	initContainers := []corev1.Container{
 		{
 			Name:    "ovsdb-server-init",
@@ -250,7 +272,7 @@ func CreateOVSDaemonSet(
 				Privileged: &privileged,
 			},
 			Env:          env.MergeEnvs([]corev1.EnvVar{}, envVars),
-			VolumeMounts: GetOVSDbVolumeMounts(),
+			VolumeMounts: append(GetOVSDbVolumeMounts(), mounts...),
 		},
 	}
 
@@ -276,7 +298,7 @@ func CreateOVSDaemonSet(
 				Privileged: &privileged,
 			},
 			Env:          env.MergeEnvs([]corev1.EnvVar{}, envVars),
-			VolumeMounts: GetOVSDbVolumeMounts(),
+			VolumeMounts: append(GetOVSDbVolumeMounts(), mounts...),
 			// TODO: consider the fact that resources are now double booked
 			Resources:                instance.Spec.Resources,
 			LivenessProbe:            ovsDbLivenessProbe,
@@ -303,7 +325,7 @@ func CreateOVSDaemonSet(
 				Privileged: &privileged,
 			},
 			Env:          env.MergeEnvs([]corev1.EnvVar{}, envVars),
-			VolumeMounts: GetVswitchdVolumeMounts(),
+			VolumeMounts: append(GetVswitchdVolumeMounts(), mounts...),
 			// TODO: consider the fact that resources are now double booked
 			Resources:                instance.Spec.Resources,
 			LivenessProbe:            ovsVswitchdLivenessProbe,
@@ -312,6 +334,9 @@ func CreateOVSDaemonSet(
 		},
 	}
 
+	maxUnavailable := intstr.FromInt32(0)
+	maxSurge := intstr.FromInt32(1)
+
 	daemonset := &appsv1.DaemonSet{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      ovnv1.ServiceNameOVS,
@@ -327,9 +352,17 @@ func CreateOVSDaemonSet(
 				},
 				Spec: corev1.PodSpec{
 					ServiceAccountName: instance.RbacResourceName(),
+					HostPID:            true,
 					InitContainers:     initContainers,
 					Containers:         containers,
-					Volumes:            GetOVSVolumes(instance.Name, instance.Namespace),
+					Volumes:            append(GetOVSVolumes(instance.Name, instance.Namespace), volumes...),
+				},
+			},
+			UpdateStrategy: appsv1.DaemonSetUpdateStrategy{
+				Type: appsv1.RollingUpdateDaemonSetStrategyType,
+				RollingUpdate: &appsv1.RollingUpdateDaemonSet{
+					MaxUnavailable: &maxUnavailable,
+					MaxSurge:       &maxSurge,
 				},
 			},
 		},

templates/ovncontroller/bin/functions

@@ -29,6 +29,15 @@ FLOWS_RESTORE_SCRIPT=$ovs_dir/flows-script
 FLOWS_RESTORE_DIR=$ovs_dir/saved-flows
 SAFE_TO_STOP_OVSDB_SERVER_SEMAPHORE=$ovs_dir/is_safe_to_stop_ovsdb_server
 
+# Variables declaration used by start-up optimization
+ovs_vswitchd_pid_file=/var/run/openvswitch/ovs-vswitchd.pid
+ovsdb_server_pid_file=/var/run/openvswitch/ovsdb-server.pid
+update_semaphore_file=/var/lib/openvswitch/update
+stop_vswitchd_script_file=/usr/local/bin/container-scripts/stop-vswitchd.sh
+stop_ovsdb_server_script_file=/usr/local/bin/container-scripts/stop-ovsdb-server.sh
+skip_ovsdb_server_stop_file=/var/lib/openvswitch/skip_stop_ovsdbserver
+skip_vswitchd_stop_file=/var/lib/openvswitch/skip_stop_vswitchd
+
 function cleanup_ovsdb_server_semaphore() {
     rm -f $SAFE_TO_STOP_OVSDB_SERVER_SEMAPHORE 2>&1 > /dev/null
 }

templates/ovncontroller/bin/init-ovsdb-server.sh

@@ -23,10 +23,25 @@ trap wait_for_db_creation EXIT
 if ! [ -s ${DB_FILE} ]; then
     rm -f ${DB_FILE}
 fi
-# Initialize or upgrade database if needed
-CTL_ARGS="--system-id=random --no-ovs-vswitchd"
-/usr/share/openvswitch/scripts/ovs-ctl start $CTL_ARGS
-/usr/share/openvswitch/scripts/ovs-ctl stop $CTL_ARGS
 
-wait_for_db_creation
-trap - EXIT
+# Check if it's a normal start or an update
+# Normal start: ovsdb-server & ovs-vswitchd are not running, start normal
+# Update: ovsdb-server & ovs-vswitchd still running, need different approach
+if [ -f $ovs_vswitchd_pid_file ] || [ -f $ovsdb_server_pid_file ]; then
+    # Some process it's running, it's an update. Create semaphore
+    echo "UPDATE" > $update_semaphore_file
+    # No need to initializice ovs-vswitchd in this path, as this has done before
+    # TODO: check what happens if during the update an update to the ovs db is needed
+else
+    # In case something went wrong last run, ensure that semaphor_file is not present in this path
+    if [ -f $update_semaphore_file ]; then
+        rm $update_semaphore_file
+    fi
+    # Initialize or upgrade database if needed
+    CTL_ARGS="--system-id=random --no-ovs-vswitchd"
+    /usr/share/openvswitch/scripts/ovs-ctl start $CTL_ARGS
+    /usr/share/openvswitch/scripts/ovs-ctl stop $CTL_ARGS
+
+    wait_for_db_creation
+    trap - EXIT
+fi

templates/ovncontroller/bin/start-ovsdb-server.sh

@@ -20,6 +20,40 @@ source $(dirname $0)/functions
 # Remove the obsolete semaphore file in case it still exists.
 cleanup_ovsdb_server_semaphore
 
+# Check if we're on the update path
+if [ -f $update_semaphore_file ]; then
+echo "In the middle of an upgrade"
+    # Need to stop vsitchd
+    echo "Stopping vswitchd"
+    bash $stop_vswitchd_script_file
+    # with this script the current lflows should be already stored in a file
+    # and vswitchd should be stopped.
+    # Need to wait until vswitchd is stoped in order to stop also the ovsdb-server
+    while true; do
+        if [ ! -f $ovs_vswitchd_pid_file ]; then
+            break
+        fi
+        sleep 0.1
+    done
+    # Ovs-vswtichd was already restarted, need to skip the preStop from the openshift
+    # lifecicle when the old pod gets deleted
+    echo "Creating flag file to skip ovs-vswitchd stop"
+    touch $skip_vswitchd_stop_file
+    # Run stop-ovsdbserver script to ensure lflows semaphor is cleaned correctly
+    bash $stop_ovsdb_server_script_file
+    # Need to create a flag-file to skip ovsdb-server stop
+    # to avoid triggering it again when openshift triggers the preStop script.
+    echo "Creating flag file to skip ovsdb-server stop"
+    touch $skip_ovsdb_server_stop_file
+    # Ensure that ovsdb-server is stopped
+    while true; do
+        if [ ! -f $ovsdb_server_pid_file ]; then
+            break
+        fi
+        sleep 0.1
+    done
+fi
+
 # Start the service
 ovsdb-server ${DB_FILE} \
     --pidfile \

templates/ovncontroller/bin/start-vswitchd.sh

@@ -15,6 +15,18 @@
 # under the License.
 
 source $(dirname $0)/functions
+
+# If we're on an update wait until past vswitchd process is stopped correctly
+if [ -f $update_semaphore_file ]; then
+    # In the middle of an update, wait until vswitchd is already stopped
+    while true; do
+        if [ ! -f $ovs_vswitchd_pid_file ]; then
+            break
+        fi
+        sleep 0.1
+    done
+fi
+
 wait_for_ovsdb_server
 
 # The order - first wait for db server, then set -ex - is important. Otherwise,
@@ -49,6 +61,10 @@ cleanup_flows_backup
 # Now, inform vswitchd that we are done.
 ovs-vsctl remove open_vswitch . other_config flow-restore-wait
 
+# At this point, ovsdb-server and vswitchd are already running, update (if it was the case)
+# is already done. Delete update file
+rm $update_semaphore_file || true
+
 # This is container command script. Block it from exiting, otherwise k8s will
 # restart the container again.
 sleep infinity

templates/ovncontroller/bin/stop-ovsdb-server.sh

@@ -17,6 +17,12 @@
 set -ex
 source $(dirname $0)/functions
 
+# If file is present, skip stop script
+if [ -f $skip_ovsdb_server_stop_file ]; then
+    rm $skip_ovsdb_server_stop_file
+    exit 0
+fi
+
 # The ovs_vswitchd container has to terminate before ovsdb-server because it
 # needs access to db in its preStop script. The preStop script backs up flows
 # for restoration during the next startup. This semaphore ensures the vswitchd

templates/ovncontroller/bin/stop-vswitchd.sh

@@ -17,6 +17,12 @@
 set -ex
 source $(dirname $0)/functions
 
+# If file is present, skip stop script
+if [ -f $skip_vswitchd_stop_file ]; then
+    rm $skip_vswitchd_stop_file
+    exit 0
+fi
+
 # Clean up any previously created flow backups to avoid conflict with newly
 # generated backup.
 cleanup_flows_backup