Merge pull request openstack-k8s-operators#338 from Akrog/revert-lvm-workaround

Support LVM

Author: openshift-merge-bot[bot]

config/samples/backends/bases/lvm/lvm.yaml

@@ -1,3 +1,9 @@
+# This machine config creates a systemd unit that creates a file based LVM VG
+# called cinder-volumes and mounts it on each reboot.
+#
+# Even though the systemd unit will be deployed in ALL OCP master nodes it will
+# only really run on the nodes that have the "openstack.org/cinder-lvm"
+# label.
 apiVersion: machineconfiguration.openshift.io/v1
 kind: MachineConfig
 metadata:
@@ -17,19 +23,23 @@ spec:
         Contents:  |
           [Unit]
           Description=Cinder LVM losetup
-          DefaultDependencies=no
           Conflicts=umount.target
-          Requires=lvm2-monitor.service systemd-udev-settle.service
-          Before=local-fs.target umount.target
-          After=var.mount lvm2-monitor.service systemd-udev-settle.service
+          Requires=lvm2-monitor.service systemd-udev-settle.service kubelet.service
+          After=var.mount lvm2-monitor.service systemd-udev-settle.service kubelet.service
 
           [Service]
           Environment=VG_GB_SIZE=10G
           Environment=LOOPBACK_FILE=/var/home/core/cinder-volumes
-          Type=oneshot
-          ExecStart=bash -c "if [[ ! -e ${LOOPBACK_FILE} ]]; then /bin/truncate -s ${VG_GB_SIZE} ${LOOPBACK_FILE} && /sbin/vgcreate cinder-volumes `/sbin/losetup --show -f ${LOOPBACK_FILE}`; else /sbin/losetup -f ${LOOPBACK_FILE}; fi"
-          ExecStop=bash -c "/sbin/losetup -d `/sbin/losetup -j ${LOOPBACK_FILE} -l -n -O NAME`"
+          Environment=LABEL=openstack.org/cinder-lvm
+
+          # Waits until cluster is available by failing, then Restart kicks in after 5 seconds
+          ExecStartPre=oc --kubeconfig=/var/lib/kubelet/kubeconfig get node
+          ExecStart=bash -c "if [ \"$(oc --kubeconfig=/var/lib/kubelet/kubeconfig get node -l ${LABEL} -o name)\" = \"node/$(hostname)\" ]; then if [[ ! -e ${LOOPBACK_FILE} ]]; then /bin/truncate -s ${VG_GB_SIZE} ${LOOPBACK_FILE} && /sbin/vgcreate cinder-volumes `/sbin/losetup --show -f ${LOOPBACK_FILE}`; else /sbin/losetup -f ${LOOPBACK_FILE}; fi; fi"
+          ExecStop=bash -c "if vgdisplay cinder-volumes; then /sbin/losetup -d `/sbin/losetup -j ${LOOPBACK_FILE} -l -n -O NAME`; fi"
           RemainAfterExit=yes
 
+          Restart=on-failure
+          RestartSec=5
+
           [Install]
-          WantedBy=local-fs-pre.target
+          WantedBy=multi-user.target

config/samples/backends/lvm/README.md

@@ -0,0 +1,47 @@
+# Cinder LVM Backend Samples
+
+Two LVM examples are provided using different transport protocols:
+
+- LVM using iSCSI
+- LVM using NVMe-TCP
+
+The LVM is a special case, because the stored data is actually on the OCP node
+and not an external storage system. This has several implications:
+
+- OCP nodes will reboot after the samples manifests are applied because they
+  are creating `MachineConfig` objects to start daemons and create the LVM
+  backing file.
+
+- Network configuration on the OCP host needs to be different, as it needs a
+  macvlan interface on top of the storage VLAN interface. This can be easily
+  done by modifying the operators deployment step running:
+  `NETWORK_STORAGE_MACVLAN=true make openstack`. Other option is to manually
+  modify the `nncp` of the OCP node.
+
+- We need to mark one of our OCP nodes with a specific label so that:
+
+  - The LVM `MachineConfig` that creates a file based LVM Volume Group and
+    ensures it is mounted on boot only really runs in one node (even if the
+    systemd unit is deployed in all the nodes).
+
+  - The cinder-volume service for the LVM backend can only run on the node that
+    has the LVM Volume Group.
+
+To mark the first ready node as the lvm node we can run:
+
+```bash
+$ oc label node \
+  $(oc get nodes --field-selector spec.unschedulable=false -l node-role.kubernetes.io/worker -o jsonpath="{.items[0].metadata.name}") \
+  openstack.org/cinder-lvm=
+```
+
+If we have a specific node in mind we can just do:
+```bash
+$ oc label node <nodename> openstack.org/cinder-lvm=
+```
+
+To see how this label is being used please look into the following files:
+
+- `./iscsi/backend.yaml`
+- `./nvme-tcp/backend.yaml`
+- `../bases/lvm/lvm.yaml`

config/samples/backends/lvm/iscsi/backend.yaml

@@ -3,7 +3,9 @@
 # The correct way to configure and run the daemons is using `MachineConfig` as shown in `iscsid.yaml` and `multipathd.yaml`.
 # To create the LVM on CRC the sample also uses a `MachineConfig` using a systemd unit with file `lvm.yaml`
 #
-# ATTENTION: After applying this OpenShift nodes will reboot, because of the `MachineConfig` changes and will take a while to recover.
+# ATTENTION:
+# - Node where we want LVM to run needs to be labeled with openstack.org/cinder-lvm
+# - After applying this OpenShift nodes will reboot, because of the `MachineConfig` changes and will take a while to recover.
 
 apiVersion: core.openstack.org/v1beta1
 kind: OpenStackControlPlane
@@ -14,6 +16,8 @@ spec:
     template:
       cinderVolumes:
         lvm-iscsi:
+          nodeSelector:
+            openstack.org/cinder-lvm: ""
           networkAttachments:
           - storage
           customServiceConfig: |

config/samples/backends/lvm/nvme-tcp/backend.yaml

@@ -3,7 +3,9 @@
 # The correct way to configure the load of additional kernel modules is using `MachineConfig` as shown in `nvme-fabrics.yaml`.yaml`.
 # To create the LVM on CRC the sample also uses a `MachineConfig` using a systemd unit with file `lvm.yaml`
 #
-# ATTENTION: After applying this OpenShift nodes will reboot, because of the `MachineConfig` changes and will take a while to recover.
+# ATTENTION:
+# - Node where we want LVM to run needs to be labeled with openstack.org/cinder-lvm
+# - After applying this OpenShift nodes will reboot, because of the `MachineConfig` changes and will take a while to recover.
 apiVersion: core.openstack.org/v1beta1
 kind: OpenStackControlPlane
 metadata:
@@ -13,6 +15,8 @@ spec:
     template:
       cinderVolumes:
         lvm-nvme-tcp:
+          nodeSelector:
+            openstack.org/cinder-lvm: ""
           networkAttachments:
           - storage
           customServiceConfig: |

controllers/cindervolume_controller.go

@@ -699,19 +699,17 @@ func (r *CinderVolumeReconciler) generateServiceConfigs(
 
 	templateParameters := make(map[string]interface{})
 	if usesLVM {
-		// Configure 'target_secondary_ip_addresses' using addresses in the
-		// network attachments. This relies on the fact that the LVM backend
-		// can only have one replica.
-		//
-		// Do not configure the primary 'target_ip_address', so that it will default
-		// to the service's 'my_ip' address. This will be the dynamically assigned
-		// OpenShift SDN address, which allows control plane services like g-api and
-		// c-bak to connect to LVM volumes.
 		networkAttachmentAddrs := cinder.GetNetworkAttachmentAddrs(
 			instance.Namespace, instance.Spec.NetworkAttachments, instance.Status.NetworkAttachments)
 
+		// Configure target IP addresses using all addresses in the network
+		// attachments. This relies on the fact that the LVM backend can only
+		// have one replica.
 		if len(networkAttachmentAddrs) > 0 {
-			templateParameters["TargetSecondaryIpAddresses"] = strings.Join(networkAttachmentAddrs, ",")
+			templateParameters["TargetIpAddress"] = networkAttachmentAddrs[0]
+			if len(networkAttachmentAddrs) > 1 {
+				templateParameters["TargetSecondaryIpAddresses"] = strings.Join(networkAttachmentAddrs[1:], ",")
+			}
 		}
 	}