Modify startup scripts for ovn-controller-ovs

In order to minimize the downtime during update of the
ovn-controller-ovs pods we're modifying the update strategy so
it doesn't accept any Unavailable pod. This means that it will
create new ovn-controller-ovs while the old one is running.

This commit enables that two ovn-controller-ovs pods can coexists.
It accomplish this by modifying the start-up scripts of all containers
so it checks if a pod is already running and slowly stops in a
controlled fashon the old pods while it starts the new ones.

The logic is done with a temporary file created during the init
container that will inform the ovsdb-server/ovs-vswitchd containers
if they're on an update scenario or a normal one. The temporary file
is deleted after the end of the ovs-vswitchd so when the ovnController
CR is deleted, the volumes won't have any leftovers.

Related: OSPRH-11636
Jira: OSPRH-10821
Depends-on: lib-common#611

Author: averdagu

controllers/ovncontroller_controller.go

@@ -744,6 +744,11 @@ func (r *OVNControllerReconciler) generateServiceConfigMaps(
 	} else {
 		templateParameters["OVNEncapNIC"] = "eth0"
 	}
+	if instance.Spec.TLS.Enabled() {
+		templateParameters["TLS"] = "Enabled"
+	} else {
+		templateParameters["TLS"] = "Disabled"
+	}
 	cms := []util.Template{
 		// ScriptsConfigMap
 		{

pkg/ovncontroller/daemonset.go

@@ -24,6 +24,7 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/utils/ptr"
 )
 
@@ -236,6 +237,27 @@ func CreateOVSDaemonSet(
 	envVars := map[string]env.Setter{}
 	envVars["CONFIG_HASH"] = env.SetValue(configHash)
 
+	volumes := []corev1.Volume{}
+	mounts := []corev1.VolumeMount{}
+
+	// add OVN dbs cert and CA
+	if instance.Spec.TLS.Enabled() {
+		svc := tls.Service{
+			SecretName: *instance.Spec.TLS.GenericService.SecretName,
+			CertMount:  ptr.To(ovn_common.OVNDbCertPath),
+			KeyMount:   ptr.To(ovn_common.OVNDbKeyPath),
+			CaMount:    ptr.To(ovn_common.OVNDbCaCertPath),
+		}
+		volumes = append(volumes, svc.CreateVolume(ovnv1.ServiceNameOVS))
+		mounts = append(mounts, svc.CreateVolumeMounts(ovnv1.ServiceNameOVS)...)
+
+		// add CA bundle if defined
+		if instance.Spec.TLS.CaBundleSecretName != "" {
+			volumes = append(volumes, instance.Spec.TLS.CreateVolume())
+			mounts = append(mounts, instance.Spec.TLS.CreateVolumeMounts(nil)...)
+		}
+	}
+
 	initContainers := []corev1.Container{
 		{
 			Name:    "ovsdb-server-init",
@@ -250,7 +272,7 @@ func CreateOVSDaemonSet(
 				Privileged: &privileged,
 			},
 			Env:          env.MergeEnvs([]corev1.EnvVar{}, envVars),
-			VolumeMounts: GetOVSDbVolumeMounts(),
+			VolumeMounts: append(GetOVSDbVolumeMounts(), mounts...),
 		},
 	}
 
@@ -276,7 +298,7 @@ func CreateOVSDaemonSet(
 				Privileged: &privileged,
 			},
 			Env:          env.MergeEnvs([]corev1.EnvVar{}, envVars),
-			VolumeMounts: GetOVSDbVolumeMounts(),
+			VolumeMounts: append(GetOVSDbVolumeMounts(), mounts...),
 			// TODO: consider the fact that resources are now double booked
 			Resources:                instance.Spec.Resources,
 			LivenessProbe:            ovsDbLivenessProbe,
@@ -303,7 +325,7 @@ func CreateOVSDaemonSet(
 				Privileged: &privileged,
 			},
 			Env:          env.MergeEnvs([]corev1.EnvVar{}, envVars),
-			VolumeMounts: GetVswitchdVolumeMounts(),
+			VolumeMounts: append(GetVswitchdVolumeMounts(), mounts...),
 			// TODO: consider the fact that resources are now double booked
 			Resources:                instance.Spec.Resources,
 			LivenessProbe:            ovsVswitchdLivenessProbe,
@@ -312,6 +334,9 @@ func CreateOVSDaemonSet(
 		},
 	}
 
+	maxUnavailable := intstr.FromInt32(0)
+	maxSurge := intstr.FromInt32(1)
+
 	daemonset := &appsv1.DaemonSet{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      ovnv1.ServiceNameOVS,
@@ -327,9 +352,17 @@ func CreateOVSDaemonSet(
 				},
 				Spec: corev1.PodSpec{
 					ServiceAccountName: instance.RbacResourceName(),
+					HostPID:            true,
 					InitContainers:     initContainers,
 					Containers:         containers,
-					Volumes:            GetOVSVolumes(instance.Name, instance.Namespace),
+					Volumes:            append(GetOVSVolumes(instance.Name, instance.Namespace), volumes...),
+				},
+			},
+			UpdateStrategy: appsv1.DaemonSetUpdateStrategy{
+				Type: appsv1.RollingUpdateDaemonSetStrategyType,
+				RollingUpdate: &appsv1.RollingUpdateDaemonSet{
+					MaxUnavailable: &maxUnavailable,
+					MaxSurge:       &maxSurge,
 				},
 			},
 		},

templates/ovncontroller/bin/functions

@@ -29,6 +29,15 @@ FLOWS_RESTORE_SCRIPT=$ovs_dir/flows-script
 FLOWS_RESTORE_DIR=$ovs_dir/saved-flows
 SAFE_TO_STOP_OVSDB_SERVER_SEMAPHORE=$ovs_dir/is_safe_to_stop_ovsdb_server
 
+# Variables declaration used by start-up optimization
+ovs_vswitchd_pid_file=/var/run/openvswitch/ovs-vswitchd.pid
+ovsdb_server_pid_file=/var/run/openvswitch/ovsdb-server.pid
+update_semaphore_file=/var/lib/openvswitch/update
+stop_vswitchd_script_file=/usr/local/bin/container-scripts/stop-vswitchd.sh
+stop_ovsdb_server_script_file=/usr/local/bin/container-scripts/stop-ovsdb-server.sh
+skip_ovsdb_server_stop_file=/var/lib/openvswitch/skip_stop_ovsdbserver
+skip_vswitchd_stop_file=/var/lib/openvswitch/skip_stop_vswitchd
+
 function cleanup_ovsdb_server_semaphore() {
     rm -f $SAFE_TO_STOP_OVSDB_SERVER_SEMAPHORE 2>&1 > /dev/null
 }

templates/ovncontroller/bin/init-ovsdb-server.sh

@@ -23,10 +23,54 @@ trap wait_for_db_creation EXIT
 if ! [ -s ${DB_FILE} ]; then
     rm -f ${DB_FILE}
 fi
-# Initialize or upgrade database if needed
-CTL_ARGS="--system-id=random --no-ovs-vswitchd"
-/usr/share/openvswitch/scripts/ovs-ctl start $CTL_ARGS
-/usr/share/openvswitch/scripts/ovs-ctl stop $CTL_ARGS
 
-wait_for_db_creation
-trap - EXIT
+# Check if it's a normal start or an update
+# Normal start: ovsdb-server & ovs-vswitchd are not running, start normal
+# Update: ovsdb-server & ovs-vswitchd still running, need different approach
+if [ -f $ovs_vswitchd_pid_file ] || [ -f $ovsdb_server_pid_file ]; then
+    # Some process it's running, it's an update. Create semaphore
+    echo "UPDATE" > $update_semaphore_file
+    # Due to modify the priority to minimize the dataplane outages we need to connect
+    # to the nb db, check if TLS is used or not
+    if [ {{ .TLS }} == "Enabled" ]; then
+        # TLS is used
+        TLSOptions="--certificate=/etc/pki/tls/certs/ovndb.crt --private-key=/etc/pki/tls/private/ovndb.key --ca-cert=/etc/pki/tls/certs/ovndbca.crt"
+        DBOptions="--db ssl:ovsdbserver-nb.openstack.svc.cluster.local:6641"
+    else
+        # Normal TCP is used
+        TLSOptions=""
+        DBOptions="--db tcp:ovsdbserver-nb.openstack.svc.cluster.local:6641"
+    fi
+    # No need to initializice ovs-vswitchd in this path, as this has done before
+    # TODO: check what happens if during the update an update to the ovs db is needed
+    chassis_id=$(ovs-vsctl get Open_Vswitch . external_ids:system-id)
+    nb_output=$(ovn-nbctl --no-leader-only $DBOptions $TLSOptions --columns=_uuid,priority find Gateway_Chassis chassis_name=$chassis_id)
+    # Check that nbctl was executed correctly
+    if [ $? -ne 0 ]; then
+        echo "ERROR: ovn-nbctl find command failed"
+        exit 1
+    fi
+    row_uuid=$(echo "$nb_output" | grep "_uuid" | cut -d':' -f2 | xargs)
+    priority=$(echo "$nb_output" | grep "priority" | cut -d':' -f2 | xargs)
+    # Save priority to be able to restore it later (It's overwritting, not appending, hence no check)
+    echo $priority > /var/lib/openvswitch/old_priority
+    # Set lower priority (lowest value possible 0)
+    ovn-nbctl --no-leader-only $DBOptions $TLSOptions set Gateway_Chassis $row_uuid priority=0
+    # Check that nbctl was executed correctly
+    if [ $? -ne 0 ]; then
+        echo "ERROR: ovn-nbctl set command failed"
+        exit 1
+    fi
+else
+    # In case something went wrong last run, ensure that semaphor_file is not present in this path
+    if [ -f $update_semaphore_file ]; then
+        rm $update_semaphore_file
+    fi
+    # Initialize or upgrade database if needed
+    CTL_ARGS="--system-id=random --no-ovs-vswitchd"
+    /usr/share/openvswitch/scripts/ovs-ctl start $CTL_ARGS
+    /usr/share/openvswitch/scripts/ovs-ctl stop $CTL_ARGS
+
+    wait_for_db_creation
+    trap - EXIT
+fi

templates/ovncontroller/bin/start-ovsdb-server.sh

@@ -20,6 +20,40 @@ source $(dirname $0)/functions
 # Remove the obsolete semaphore file in case it still exists.
 cleanup_ovsdb_server_semaphore
 
+# Check if we're on the update path
+if [ -f $update_semaphore_file ]; then
+    echo "In the middle of an upgrade"
+    # Need to stop vsitchd
+    echo "Stopping vswitchd"
+    bash $stop_vswitchd_script_file
+    # with this script the current lflows should be already stored in a file
+    # and vswitchd should be stopped.
+    # Need to wait until vswitchd is stoped in order to stop also the ovsdb-server
+    while true; do
+        if [ ! -f $ovs_vswitchd_pid_file ]; then
+            break
+        fi
+        sleep 0.1
+    done
+    # Ovs-vswtichd was already restarted, need to skip the preStop from the openshift
+    # lifecicle when the old pod gets deleted
+    echo "Creating flag file to skip ovs-vswitchd stop"
+    touch $skip_vswitchd_stop_file
+    # Run stop-ovsdbserver script to ensure lflows semaphor is cleaned correctly
+    bash $stop_ovsdb_server_script_file
+    # Need to create a flag-file to skip ovsdb-server stop
+    # to avoid triggering it again when openshift triggers the preStop script.
+    echo "Creating flag file to skip ovsdb-server stop"
+    touch $skip_ovsdb_server_stop_file
+    # Ensure that ovsdb-server is stopped
+    while true; do
+        if [ ! -f $ovsdb_server_pid_file ]; then
+            break
+        fi
+        sleep 0.1
+    done
+fi
+
 # Start the service
 ovsdb-server ${DB_FILE} \
     --pidfile \

templates/ovncontroller/bin/start-vswitchd.sh

@@ -15,6 +15,29 @@
 # under the License.
 
 source $(dirname $0)/functions
+
+# Check which connection is used
+if [ {{ .TLS }} == "Enabled" ]; then
+    # TLS is used
+    TLSOptions="--certificate=/etc/pki/tls/certs/ovndb.crt --private-key=/etc/pki/tls/private/ovndb.key --ca-cert=/etc/pki/tls/certs/ovndbca.crt"
+    DBOptions="--db ssl:ovsdbserver-nb.openstack.svc.cluster.local:6641"
+else
+    # Normal TCP is used
+    TLSOptions=""
+    DBOptions="--db tcp:ovsdbserver-nb.openstack.svc.cluster.local:6641"
+fi
+
+# If we're on an update wait until past vswitchd process is stopped correctly
+if [ -f $update_semaphore_file ]; then
+    # In the middle of an update, wait until vswitchd is already stopped
+    while true; do
+        if [ ! -f $ovs_vswitchd_pid_file ]; then
+            break
+        fi
+        sleep 0.1
+    done
+fi
+
 wait_for_ovsdb_server
 
 # The order - first wait for db server, then set -ex - is important. Otherwise,
@@ -28,6 +51,25 @@ ovs-vsctl --no-wait set open . external-ids:ovn-encap-ip=${OVNEncapIP}
 # Before starting vswitchd, block it from flushing existing datapath flows.
 ovs-vsctl --no-wait set open_vswitch . other_config:flow-restore-wait=true
 
+# Restore the priority if this was changed during the update
+if [ -f /var/lib/openvswitch/old_priority ]; then
+    priority=$(cat /var/lib/openvswitch/old_priority)
+    echo "Restoring old priority, which was: $priority"
+    chassis_id=$(ovs-vsctl get Open_Vswitch . external_ids:system-id)
+    nb_output=$(ovn-nbctl --no-leader-only $DBOptions $TLSOptions --columns=_uuid,priority find Gateway_Chassis chassis_name=$chassis_id)
+    err=$?
+    if [ $err -ne 0 ]; then
+        echo "Error while getting gateway chassis uuid $err"
+    fi
+    row_uuid=$(echo "$nb_output" | grep "_uuid" | cut -d':' -f2 | xargs)
+    rm /var/lib/openvswitch/old_priority
+    ovn-nbctl --no-leader-only $DBOptions $TLSOptions set Gateway_Chassis $row_uuid priority=$priority
+    err=$?
+    if [ $err -ne 0 ]; then
+        echo "Error while setting gateway chassis priority ($priority), error: $err"
+    fi
+fi
+
 # It's safe to start vswitchd now. Do it.
 # --detach to allow the execution to continue to restoring the flows.
 /usr/sbin/ovs-vswitchd --pidfile --mlockall --detach
@@ -49,6 +91,10 @@ cleanup_flows_backup
 # Now, inform vswitchd that we are done.
 ovs-vsctl remove open_vswitch . other_config flow-restore-wait
 
+# At this point, ovsdb-server and vswitchd are already running, update (if it was the case)
+# is already done. Delete update file
+rm $update_semaphore_file || true
+
 # This is container command script. Block it from exiting, otherwise k8s will
 # restart the container again.
 sleep infinity

templates/ovncontroller/bin/stop-ovsdb-server.sh

@@ -17,6 +17,12 @@
 set -ex
 source $(dirname $0)/functions
 
+# If file is present, skip stop script
+if [ -f $skip_ovsdb_server_stop_file ]; then
+    rm $skip_ovsdb_server_stop_file
+    exit 0
+fi
+
 # The ovs_vswitchd container has to terminate before ovsdb-server because it
 # needs access to db in its preStop script. The preStop script backs up flows
 # for restoration during the next startup. This semaphore ensures the vswitchd

templates/ovncontroller/bin/stop-vswitchd.sh

@@ -17,6 +17,12 @@
 set -ex
 source $(dirname $0)/functions
 
+# If file is present, skip stop script
+if [ -f $skip_vswitchd_stop_file ]; then
+    rm $skip_vswitchd_stop_file
+    exit 0
+fi
+
 # Clean up any previously created flow backups to avoid conflict with newly
 # generated backup.
 cleanup_flows_backup