e2e: add 8-socket 4k-CPU e2e vm and related test for balloons policy

test/e2e/files/Vagrantfile.in

@@ -74,7 +74,7 @@ Vagrant.configure("2") do |config|
     qemu.memory = "QEMU_MEM"
     qemu.arch = "x86_64"
     qemu.machine = "QEMU_MACHINE"
-    qemu.cpu = "host"
+    qemu.cpu = "QEMU_CPU"
     qemu.net_device = "virtio-net-pci"
     qemu.extra_netdev_args = "net=192.168.76.0/24,dhcpstart=192.168.76.9"
     qemu.ssh_port = SSH_PORT

test/e2e/lib/topology2qemuopts.py

@@ -24,6 +24,8 @@
                       The default is 1.
 "packages"            number of packages.
                       The default is 1.
+"cpus-present"        number of logical CPUs present in the system.
+                      The default value 0 means "all".
 
 NUMA node distances are defined with following keys:
 "dist-all": [[from0to0, from0to1, ...], [from1to0, from1to1, ...], ...]
@@ -108,13 +110,15 @@ def validate(numalist):
         raise ValueError('expected list containing dicts, got %s' % (type(numalist,).__name__))
     valid_keys = set(("mem", "nvmem", "dimm",
                       "cores", "threads", "nodes", "dies", "packages",
+                      "cpus-present",
                       "node-dist", "dist-all",
                       "dist-other-package", "dist-same-package", "dist-same-die"))
     int_range_keys = {'cores': ('>= 0', lambda v: v >= 0),
                       'threads': ('> 0', lambda v: v > 0),
                       'nodes': ('> 0', lambda v: v > 0),
                       'dies': ('> 0', lambda v: v > 0),
-                      'packages': ('> 0', lambda v: v > 0)}
+                      'packages': ('> 0', lambda v: v > 0),
+                      'cpus-present': ('>= 0', lambda v: v >=0)}
     for numalistindex, numaspec in enumerate(numalist):
         for key in numaspec:
             if not key in valid_keys:
@@ -210,12 +214,14 @@ def dists(numalist):
     return dist_dict
 
 def qemuopts(numalist):
-    machineparam = "-machine pc"
+    machineparam = "-machine q35,kernel-irqchip=split"
+    cpuparam = "-cpu host,x2apic=on"
     numaparams = []
     objectparams = []
     deviceparams = []
     lastnode = -1
     lastcpu = -1
+    lastcpupresent = -1
     lastdie = -1
     lastsocket = -1
     lastmem = -1
@@ -249,6 +255,7 @@ def qemuopts(numalist):
         cpucount = int(numaspec.get("cores", 0)) * threadcount # logical cpus per numa node (cores * threads)
         diecount = int(numaspec.get("dies", 1))
         packagecount = int(numaspec.get("packages", 1))
+        cpuspresentcount = int(numaspec.get("cpus-present", 0))
         memsize = numaspec.get("mem", "0")
         memdimm = numaspec.get("dimm", "")
         if memsize != "0":
@@ -333,6 +340,10 @@ def qemuopts(numalist):
                             currentnumaparams.append("node,nodeid=%s" % (lastnode,))
                         currentnumaparams[-1] = currentnumaparams[-1] + (",cpus=%s-%s" % (lastcpu + 1, lastcpu + cpucount))
                         lastcpu += cpucount
+                        if cpuspresentcount > 0:
+                            lastcpupresent = cpuspresentcount - 1
+                        else:
+                            lastcpupresent += cpucount
                     numaparams.extend(currentnumaparams)
     node_node_dist = dists(numalist)
     for sourcenode in sorted(node_node_dist.keys()):
@@ -350,7 +361,7 @@ def qemuopts(numalist):
         # Don't give dies parameter unless it is absolutely necessary
         # because it requires Qemu >= 5.0.
         diesparam = ""
-    cpuparam = "-smp cpus=%s,threads=%s%s,sockets=%s" % (lastcpu + 1, threadcount, diesparam, lastsocket + 1)
+    smpparam = "-smp cpus=%s,threads=%s%s,sockets=%s,maxcpus=%s" % (lastcpupresent + 1, threadcount, diesparam, lastsocket + 1, lastcpu + 1)
     maxmem = siadd(totalmem, totalnvmem)
     startmem = sisub(sisub(maxmem, unpluggedmem), pluggedmem)
     memparam = "-m size=%s,slots=%s,maxmem=%s" % (startmem, memslots, maxmem)
@@ -362,6 +373,7 @@ def qemuopts(numalist):
     if separated_output_vars == True:
         return ("MACHINE:" + machineparam + "|" +
                 "CPU:" + cpuparam + "|" +
+                "SMP:" + smpparam + "|" +
                 "MEM:" + memparam + "|" +
                 "EXTRA:" +
                 ", ".join(map(lambda x: "\"" + x + "\"", numaparams)) +
@@ -373,6 +385,7 @@ def qemuopts(numalist):
     else:
         return (machineparam + " " +
             cpuparam + " " +
+            smpparam + " " +
             memparam + " " +
             " ".join(numaparams) +
             " " +

test/e2e/lib/vm.bash

@@ -126,11 +126,12 @@ vm-setup() {
         error "error in topology"
     fi
 
-    MACHINE=$(echo $VM_QEMU_CPUMEM | sed 's/MACHINE:-machine \([^|]*\).*/\1/g')
-    CPU=$(echo $VM_QEMU_CPUMEM | sed 's/MACHINE:.*CPU:-smp \([^|]*\).*/\1/g')
-    MEM=$(echo $VM_QEMU_CPUMEM | sed 's/MACHINE:.*CPU:.*MEM:-m \([^|]*\).*/\1/g')
-    EXTRA_ARGS=$(echo $VM_QEMU_CPUMEM | sed 's/MACHINE:.*CPU:.*MEM:.*EXTRA:\([^|]*\).*/\1/g')
-    EXTRA_ARGS+="${EXTRA_ARGS:+,} \"-monitor\", \"unix:monitor.sock,server,nowait\""
+    local MACHINE=$(echo $VM_QEMU_CPUMEM | sed 's/MACHINE:-machine \([^|]*\).*/\1/g')
+    local CPU=$(echo $VM_QEMU_CPUMEM | sed 's/MACHINE:.*CPU:-cpu \([^|]*\).*/\1/g')
+    local SMP=$(echo $VM_QEMU_CPUMEM | sed 's/MACHINE:.*CPU:.*SMP:-smp \([^|]*\).*/\1/g')
+    local MEM=$(echo $VM_QEMU_CPUMEM | sed 's/MACHINE:.*CPU:.*SMP:.*MEM:-m \([^|]*\).*/\1/g')
+    local EXTRA_ARGS=$(echo $VM_QEMU_CPUMEM | sed 's/MACHINE:.*CPU:.*SMP:.*MEM:.*EXTRA:\([^|]*\).*/\1/g')
+    local EXTRA_ARGS+="${EXTRA_ARGS:+,} \"-monitor\", \"unix:monitor.sock,server,nowait\""
 
     case $efi in
         "") ;;
@@ -167,6 +168,7 @@ vm-setup() {
     if [ "$vagrant_debug" == "1" ]; then
 	echo "MACHINE: $MACHINE"
 	echo "CPU: $CPU"
+	echo "SMP: $SMP"
 	echo "MEM: $MEM"
 	echo "EXTRA: $EXTRA_ARGS"
         echo "image: ${distro_img:-vagrant default}"
@@ -182,8 +184,9 @@ vm-setup() {
 	sed -e "s/SERVER_NAME/$vmname/g" \
 	    -e "s/DISTRO/$distro_name/g" \
 	    -e "s/QEMU_MACHINE/$MACHINE/" \
+	    -e "s/QEMU_CPU/$CPU/" \
+	    -e "s/QEMU_SMP/$SMP/" \
 	    -e "s/QEMU_MEM/$MEM/" \
-	    -e "s/QEMU_SMP/$CPU/" \
 	    -e "s|QEMU_EXTRA_ARGS|$EXTRA_ARGS|" \
             -e "s:QEMU_DIR:$qemu_dir:" \
             -e "s|^.*config.vm.box_url.*$|$CUSTOM_IMAGE|g" \
@@ -342,6 +345,42 @@ vm-reboot() { # script API
     deadline=$_deadline host-wait-vm-ssh-server $_vagrantdir
 }
 
+vm-cpu-hotplug() { # script API
+    # Usage: vm-cpu-hotplug SOCKETID COREID THREADID
+    #
+    # Hotplug currently unplugged CPU to VM.
+    #
+    # Examples:
+    #   vm-cpu-hotplug 0 255 0
+    local socketid=$1
+    local coreid=$2
+    local threadid=$3
+    local deviceid="cpu-s$socketid-c$coreid-t$threadid"
+    if [[ -z "$threadid" ]]; then
+        error "missing one or more IDs: socket core thread"
+        return 1
+    fi
+    vm-monitor "device_add driver=host-x86_64-cpu,id=${deviceid},socket-id=${socketid},core-id=${coreid},thread-id=${threadid}"
+}
+
+vm-cpu-hotremove() { # script API
+    # Usage: vm-cpu-hotremove SOCKETID COREID THREADID
+    #
+    # Hotremove currently plugged CPU from VM.
+    #
+    # Examples:
+    #   vm-cpu-hotremove 0 255 0
+    local socketid=$1
+    local coreid=$2
+    local threadid=$3
+    local deviceid="cpu-s$socketid-c$coreid-t$threadid"
+    if [[ -z "$threadid" ]]; then
+        error "missing one or more IDs: socket core thread"
+        return 1
+    fi
+    vm-monitor "device_del ${deviceid}"
+}
+
 vm-mem-hotplug() { # script API
     # Usage: vm-mem-hotplug MEMORY
     #

test/e2e/policies.test-suite/balloons/n6-hbm-cxl/test04-nrt/code.var.sh

@@ -6,50 +6,25 @@ cleanup() {
 cleanup
 helm_config=${TEST_DIR}/balloons-nrt.cfg helm-launch balloons
 
-export get_nrt="kubectl get noderesourcetopologies.topology.node.k8s.io \$(hostname)"
-
-verify-zone-attribute() {
-    local zone_name=$1
-    local attribute_name=$2
-    local expected_value_re=$3
-    echo ""
-    echo "### Verifying topology zone $zone_name attribute $attribute_name value matches $expected_value_re"
-    vm-command "$get_nrt -o json | jq -r '.zones[] | select (.name == \"$zone_name\").attributes[] | select(.name == \"$attribute_name\").value'"
-    [[ "$COMMAND_OUTPUT" =~ $expected_value_re ]] ||
-        command-error "expected zone $zone_name attribute $attribute_name value $expected_value, got: $COMMAND_OUTPUT"
-}
-
-verify-zone-resource() {
-    local zone_name=$1
-    local resource_name=$2
-    local resource_field=$3
-    local expected_value=$4
-    echo ""
-    echo "### Verifying topology zone $zone_name resource $resouce_name field $resource_field equals $expected_value"
-    vm-command "$get_nrt -o json | jq -r '.zones[] | select (.name == \"$zone_name\").resources[] | select(.name == \"$resource_name\").$resource_field'"
-    [[ "$COMMAND_OUTPUT" == "$expected_value" ]] ||
-        command-error "expected zone $zone_name resource $resource_name.$resource_field $expected_value, got: $COMMAND_OUTPUT"
-}
-
 # Print full NRT yaml for debugging
-vm-command "$get_nrt -o yaml"
+vm-command "$nrt_kubectl_get -o yaml"
 
 # Verify zones when fullsocket balloons do not include containers.
-verify-zone-attribute "fullsocket[0]" "cpuset" "[4567]"
-verify-zone-attribute "fullsocket[0]" "shared cpuset" "5-7|4,6-7|4-5,7|4-6"
-verify-zone-attribute "fullsocket[0]" "excess cpus" "1"
-verify-zone-attribute "fullsocket[1]" "cpuset" "0|1"
-verify-zone-attribute "fullsocket[1]" "shared cpuset" "1-2|0,2"
-verify-zone-attribute "fullsocket[1]" "excess cpus" "1"
-verify-zone-attribute "reserved[0]" "cpuset" "3"
-verify-zone-attribute "reserved[0]" "shared cpuset" "^\$"
-
-verify-zone-resource "reserved[0]" "cpu" "capacity" "8"
-verify-zone-resource "reserved[0]" "cpu" "allocatable" "6"
-verify-zone-resource "fullsocket[0]" "cpu" "allocatable" "4"
-verify-zone-resource "fullsocket[0]" "cpu" "available" "4"
-verify-zone-resource "fullsocket[1]" "cpu" "allocatable" "4"
-verify-zone-resource "fullsocket[1]" "cpu" "available" "4"
+nrt-verify-zone-attribute "fullsocket[0]" "cpuset" "[4567]"
+nrt-verify-zone-attribute "fullsocket[0]" "shared cpuset" "5-7|4,6-7|4-5,7|4-6"
+nrt-verify-zone-attribute "fullsocket[0]" "excess cpus" "1"
+nrt-verify-zone-attribute "fullsocket[1]" "cpuset" "0|1"
+nrt-verify-zone-attribute "fullsocket[1]" "shared cpuset" "1-2|0,2"
+nrt-verify-zone-attribute "fullsocket[1]" "excess cpus" "1"
+nrt-verify-zone-attribute "reserved[0]" "cpuset" "3"
+nrt-verify-zone-attribute "reserved[0]" "shared cpuset" "^\$"
+
+nrt-verify-zone-resource "reserved[0]" "cpu" "capacity" "8"
+nrt-verify-zone-resource "reserved[0]" "cpu" "allocatable" "6"
+nrt-verify-zone-resource "fullsocket[0]" "cpu" "allocatable" "4"
+nrt-verify-zone-resource "fullsocket[0]" "cpu" "available" "4"
+nrt-verify-zone-resource "fullsocket[1]" "cpu" "allocatable" "4"
+nrt-verify-zone-resource "fullsocket[1]" "cpu" "available" "4"
 
 # Create burstable containers without CPU limits
 CPUREQ="750m" MEMREQ="100M" CPULIM="" MEMLIM="500M"
@@ -58,28 +33,28 @@ POD_ANNOTATION='cpu.preserve.resource-policy.nri.io/container.pod0c1: "true"
 CONTCOUNT=3 create balloons-busybox
 
 # Print full NRT yaml for debugging
-vm-command "$get_nrt -o yaml"
+vm-command "$nrt_kubectl_get -o yaml"
 
 # Verify selected zone attributes
-verify-zone-resource "default/pod0/pod0c0" "cpu" "capacity" "4" # balloon's + shared CPUs
-verify-zone-resource "default/pod0/pod0c0" "cpu" "allocatable" "750m" # requested CPUs
-verify-zone-resource "default/pod0/pod0c0" "cpu" "available" "0" # nothing available on the subzone
-verify-zone-attribute "default/pod0/pod0c0" "cpuset" "4-7"
-verify-zone-attribute "default/pod0/pod0c0" "memory set" "1"
+nrt-verify-zone-resource "default/pod0/pod0c0" "cpu" "capacity" "4" # balloon's + shared CPUs
+nrt-verify-zone-resource "default/pod0/pod0c0" "cpu" "allocatable" "750m" # requested CPUs
+nrt-verify-zone-resource "default/pod0/pod0c0" "cpu" "available" "0" # nothing available on the subzone
+nrt-verify-zone-attribute "default/pod0/pod0c0" "cpuset" "4-7"
+nrt-verify-zone-attribute "default/pod0/pod0c0" "memory set" "1"
 
-verify-zone-resource "default/pod0/pod0c1" "cpu" "capacity" "" # preserve => container should not exist, not assigned into any balloon
+nrt-verify-zone-resource "default/pod0/pod0c1" "cpu" "capacity" "" # preserve => container should not exist, not assigned into any balloon
 
-verify-zone-resource "default/pod0/pod0c2" "cpu" "capacity" "4" # expect same balloon as pod0c0
-verify-zone-resource "default/pod0/pod0c2" "cpu" "allocatable" "750m"
-verify-zone-attribute "default/pod0/pod0c2" "memory set" "^\$" # preserve memory
+nrt-verify-zone-resource "default/pod0/pod0c2" "cpu" "capacity" "4" # expect same balloon as pod0c0
+nrt-verify-zone-resource "default/pod0/pod0c2" "cpu" "allocatable" "750m"
+nrt-verify-zone-attribute "default/pod0/pod0c2" "memory set" "^\$" # preserve memory
 
 # Create burstable containers with CPU limits
 CPUREQ="200m" MEMREQ="100M" CPULIM="1500m" MEMLIM="500M"
 POD_ANNOTATION=''
 CONTCOUNT=2 create balloons-busybox
 
 # Print full NRT yaml for debugging
-vm-command "$get_nrt -o yaml"
+vm-command "$nrt_kubectl_get -o yaml"
 
-verify-zone-resource "default/pod1/pod1c0" "cpu" "capacity" "1500m" # limit < allowed cpus
-verify-zone-attribute "default/pod1/pod1c0" "cpuset" "0-2" # expected fullsocket[1]
+nrt-verify-zone-resource "default/pod1/pod1c0" "cpu" "capacity" "1500m" # limit < allowed cpus
+nrt-verify-zone-attribute "default/pod1/pod1c0" "cpuset" "0-2" # expected fullsocket[1]

test/e2e/policies.test-suite/balloons/nrt.source.sh

@@ -0,0 +1,24 @@
+export nrt_kubectl_get="kubectl get noderesourcetopologies.topology.node.k8s.io \$(hostname)"
+
+nrt-verify-zone-attribute() {
+    local zone_name=$1
+    local attribute_name=$2
+    local expected_value_re=$3
+    echo ""
+    echo "### Verifying topology zone $zone_name attribute $attribute_name value matches $expected_value_re"
+    vm-command "$nrt_kubectl_get -o json | jq -r '.zones[] | select (.name == \"$zone_name\").attributes[] | select(.name == \"$attribute_name\").value'"
+    [[ "$COMMAND_OUTPUT" =~ $expected_value_re ]] ||
+        command-error "expected zone $zone_name attribute $attribute_name value $expected_value, got: $COMMAND_OUTPUT"
+}
+
+nrt-verify-zone-resource() {
+    local zone_name=$1
+    local resource_name=$2
+    local resource_field=$3
+    local expected_value=$4
+    echo ""
+    echo "### Verifying topology zone $zone_name resource $resouce_name field $resource_field equals $expected_value"
+    vm-command "$nrt_kubectl_get -o json | jq -r '.zones[] | select (.name == \"$zone_name\").resources[] | select(.name == \"$resource_name\").$resource_field'"
+    [[ "$COMMAND_OUTPUT" == "$expected_value" ]] ||
+        command-error "expected zone $zone_name resource $resource_name.$resource_field $expected_value, got: $COMMAND_OUTPUT"
+}

test/e2e/policies.test-suite/balloons/s8c4k/test01-sparse-4kcpus/balloons-sparse-4kcpus.cfg

@@ -0,0 +1,42 @@
+config:
+  pinCPU: true
+  pinMemory: false
+  balloonTypes:
+    - name: pkg0
+      minCPUs: 1
+      maxCPUs: 1
+      preferCloseToDevices:
+      - /sys/devices/system/node/node0
+      shareIdleCPUsInSame: package
+
+    - name: pkg2
+      minCPUs: 1
+      maxCPUs: 1
+      preferCloseToDevices:
+      - /sys/devices/system/node/node2
+
+    - name: pkg7
+      minCPUs: 1
+      maxCPUs: 1
+      minBalloons: 1
+      preferCloseToDevices:
+      - /sys/devices/system/node/node7
+
+  availableResources:
+    cpu: cpuset:0-4095
+  reservedResources:
+    cpu: "cpuset:1"
+
+  instrumentation:
+    httpEndpoint: ":8891"
+  log:
+    debug:
+      - nri-resource-policy
+      - resource-manager
+      - cache
+      - policy
+      - nri-plugin
+      - sysfs
+    source: true
+    klog:
+      skip_headers: true