e2e: enable custom kernel building, caching and installing (CXL-2)

test/e2e/files/Vagrantfile.in

@@ -15,6 +15,8 @@ HOSTNAME = "SERVER_NAME"
 # How many nodes to create
 N = 1
 
+KERNEL_GETSOURCE = "#{ENV['kernel_getsource']}"
+KERNEL_CONFIG = "#{ENV['kernel_config']}"
 K8S_RELEASE = "#{ENV['k8s_release']}"
 K8S_VERSION = "#{ENV['k8s_version']}"
 HELM_RELEASE = "#{ENV['helm_release']}"
@@ -108,6 +110,8 @@ Vagrant.configure("2") do |config|
       https_proxy: "#{ENV['HTTPS_PROXY']}",
       http_proxy: "#{ENV['HTTP_PROXY']}",
       no_proxy: "#{ENV['NO_PROXY']}",
+      kernel_getsource: KERNEL_GETSOURCE,
+      kernel_config: KERNEL_CONFIG,
       k8s_release: K8S_RELEASE,
       k8s_version: K8S_VERSION,
       helm_release: HELM_RELEASE,

test/e2e/lib/vm.bash

@@ -274,7 +274,7 @@ vm-play() {
 	  -i "${vm}," -u vagrant \
 	  --private-key="$private_key" \
 	  --ssh-common-args "-F $vagrantdir/.ssh-config" \
-	  --extra-vars "cri_runtime=${k8scri} nri_resource_policy_src=${nri_resource_policy_src} cache_dir=$CACHE_DIR"
+	  --extra-vars "cri_runtime=${k8scri} nri_resource_policy_src=${nri_resource_policy_src} cache_dir=$CACHE_DIR kernel_getsource=$kernel_getsource kernel_config=$kernel_config"
     )
 }
 
@@ -756,31 +756,128 @@ vm-set-kernel-cmdline() {
     fi
 }
 
-fedora-set-kernel-cmdline() {
-    local e2e_defaults="$*"
-    vm-command "mkdir -p /etc/default; touch /etc/default/grub; sed -i '/e2e:fedora-set-kernel-cmdline/d' /etc/default/grub"
-    vm-command "echo 'GRUB_CMDLINE_LINUX_DEFAULT=\"\${GRUB_CMDLINE_LINUX_DEFAULT} ${e2e_defaults}\" # by e2e:fedora-set-kernel-cmdline' >> /etc/default/grub" || {
-        command-error "writing new command line parameters failed"
-    }
-    vm-command "grub2-mkconfig -o /boot/grub2/grub.cfg" || {
-        command-error "updating grub failed"
-    }
+vm-kernel-pkgs-install() { # script API
+    # Usage: vm-kernel-pkgs-install
+    #
+    # Install custom kernel packages from a tar file.
+    # Does nothing if requested packages are already installed.
+    #
+    # Environment variables:
+    #   file       contains the name of the tarball on vm.
+    #              The default is ~/kernel-pkgs.tar.
+    #
+    # Saves previous default to ~/.vm-kernel-pkgs.default-kernel.
+    # That kernel can be restored with vm-kernel-pkgs-uninstall.
+    local file="${file:-kernel-pkgs.tar}"
+    local feat="vm-kernel-pkgs"
+
+    vm-command "tar tf $file" || \
+        command-error "cannot list contents of kernel packages file $file"
+    local to_be_installed="$COMMAND_OUTPUT"
+
+    vm-command "cat .$feat.installed_packages 2>/dev/null || echo ''" || \
+        command-error "cannot read previously installed kernel packages list"
+    local already_installed="$COMMAND_OUTPUT"
+
+    if [ "$to_be_installed" == "$already_installed" ]; then
+        echo "vm-kernel-pkgs-install: kernel packages from $file already installed, skipping installation"
+        return 0
+    fi
+
+    vm-command "[ -d $feat ] && rm -rf $feat; mkdir $feat; tar xvf $file -C $feat | tee .$feat.extracted_packages" ||
+        command-error "cannot extract kernel packages from $file"
+
+    if grep -q .rpm <<< "$to_be_installed"; then
+        # Store current kernel / Fedora
+        vm-command "grubby --default-kernel | tee .$feat.default-kernel" || \
+            command-error "cannot save current default kernel"
+        # Install new kernel packages / Fedora
+        vm-command "rpm -Uvh $feat/*.rpm" || \
+            command-error "cannot install kernel rpm packages from $file"
+    elif grep -q .deb <<< "$to_be_installed"; then
+        # Store current kernel / Ubuntu
+        vm-command "uname -r | tee .$feat.default-kernel"
+        # Install new kernel packages / Ubuntu
+        vm-command "dpkg -i $feat/*.deb" || \
+            command-error "cannot install kernel deb packages from $file"
+        # In lack of grubby, set GRUB_DEFAULT in both "original" (for current) and "override" (for new kernel) configs.
+        # They both boot the system with current kernel and new kernel, respectively.
+        # When wish to change back from the new to the original kernel, it suffices to remove the "override" config
+        # and update grub.
+        vm-command "
+            oldkernelversion=\$(uname -r)
+            newkernelversion=\$(dpkg --contents $feat/linux-image-*.deb | awk -Fvmlinuz- /vmlinuz-/'{print \$2}')
+            submenu=\$(grep -i submenu.*advanced /boot/grub/grub.cfg | awk -F\' '{print \$2}')
+            oldmenuchoice=\$(grep \"menuentry.*\$oldkernelversion\" /boot/grub/grub.cfg | grep -v recovery | awk -F\' '{print \$2}')
+            newmenuchoice=\$(grep \"menuentry.*\$newkernelversion\" /boot/grub/grub.cfg | grep -v recovery | awk -F\' '{print \$2}')
+            echo 'Save original grub default'
+            echo \"GRUB_DEFAULT='\$submenu>\$oldmenuchoice'\" | tee /etc/default/grub.d/e2e-00-original-default-kernel.cfg
+            echo 'Override original grub default with:'
+            echo \"GRUB_DEFAULT='\$submenu>\$newmenuchoice'\" | tee /etc/default/grub.d/e2e-01-override-default-kernel.cfg
+            update-grub
+            "
+    else
+        command-error "no kernel packages found in $file"
+    fi
+    vm-command "cat .$feat.extracted_packages >> .$feat.installed_packages && rm .$feat.extracted_packages" || \
+        command-error "cannot save installed kernel packages list"
+
+    echo "Booting to new kernel..."
+    vm-reboot || \
+        error "vm-kernel-pkgs-install: reboot failed after installing new kernel packages"
+
+    vm-command "uname -a"
 }
 
-ubuntu-set-kernel-cmdline() {
-    local e2e_defaults="$*"
-    vm-command "echo 'GRUB_CMDLINE_LINUX_DEFAULT=\"\${GRUB_CMDLINE_LINUX_DEFAULT} ${e2e_defaults}\"' > /etc/default/grub.d/60-e2e-defaults.cfg" || {
-        command-error "writing new command line parameters failed"
-    }
-    vm-command "update-grub" || {
-        command-error "updating grub failed"
+vm-kernel-pkgs-uninstall() { # script API
+    # Usage: vm-kernel-pkgs-uninstall
+    #
+    # Boot to previous kernel before vm-kernel-pkgs-install and
+    # uninstall custom kernel packages installed with vm-kernel-pkgs-install.
+    local feat="vm-kernel-pkgs"
+    local default_kernel
+    local installed_packages
+    vm-command "cat .$feat.default-kernel" || {
+        echo "vm-kernel-pkgs-uninstall: kernel-pkgs not installed"
+        return 0
     }
-}
 
-vm-set-kernel-cmdline() {
-    if [[ "$distro" == *fedora* ]]; then
-        fedora-set-kernel-cmdline "$*"
+    default_kernel="$COMMAND_OUTPUT"
+    if [ -z "$default_kernel" ]; then
+        command-error "cannot restore previous default kernel, file ~/.$feat.default-kernel is missing or empty"
+    fi
+
+    vm-command "cat .$feat.installed_packages" || \
+        command-error "cannot find installed kernel packages list file ~/.$feat.installed_packages"
+    installed_packages="$COMMAND_OUTPUT"
+    if [ -z "$installed_packages" ]; then
+        command-error "cannot uninstall kernel packages, file ~/.$feat.installed_packages is missing or empty"
+    fi
+
+    # Modify grub to select previous kernel on next boot
+    if grep -q rpm <<< "$installed_packages"; then
+        vm-command "grubby --set-default=\"$default_kernel\"" || \
+            command-error "cannot restore previous default kernel to $default_kernel"
     else
-        ubuntu-set-kernel-cmdline "$*"
+        vm-command "rm -f /etc/default/grub.d/e2e-01-override-default-kernel.cfg; update-grub"
     fi
+
+    echo "Booting to previous kernel: $default_kernel."
+    vm-reboot
+
+    # Uninstall previously installed custom kernel packages.
+    if grep -q rpm <<< "$installed_packages"; then
+        vm-command "sed 's/.rpm//g' < .$feat.installed_packages | xargs rpm -e --nodeps" || \
+            command-error "failed to uninstall packages: $installed_packages"
+    elif grep -q deb <<< "$installed_packages"; then
+        vm-command "xargs -a .$feat.installed_packages dpkg -r" || \
+            command-error "failed to uninstall packages: $installed_packages"
+        vm-command "rm -f /etc/default/grub.d/e2e-00-original-default-kernel.cfg; update-grub"
+    else
+        command-error "no kernel packages found in ~/.$feat.installed_packages"
+    fi
+    vm-command "rm -f .$feat.installed_packages .$feat.default-kernel" || \
+        command-error "cannot remove ~/.$feat.installed_packages and ~/.$feat.default-kernel files"
+
+    vm-command "uname -a"
 }

test/e2e/playbook/custom-kernel-fedora.yaml

@@ -0,0 +1,29 @@
+---
+- name: Install kernel build dependencies
+  become: yes
+  ansible.builtin.package:
+    pkg:
+      - fedpkg
+      - fedora-packager
+      - rpmdevtools
+      - ncurses-devel
+      - pesign
+      - grubby
+      - make
+      - gcc
+      - flex
+      - bison
+      - elfutils-devel
+      - elfutils-libelf-devel
+      - dwarves
+      - openssl
+      - openssl-devel
+      - perl
+    state: present
+    update_cache: true
+
+- name: Build kernel RPMs
+  shell: "cd linux && make -j$(nproc) binrpm-pkg"
+
+- name: Create vm:~/kernel-pkgs.tar containing kernel RPMs
+  shell: "cd linux/rpmbuild/RPMS/x86_64; tar cf ~/kernel-pkgs.tar kernel-*.rpm"

test/e2e/playbook/custom-kernel-ubuntu.yaml

@@ -0,0 +1,30 @@
+---
+- name: Install kernel build dependencies
+  become: yes
+  ansible.builtin.package:
+    pkg:
+      - git-core
+      - build-essential
+      - debhelper-compat
+      - flex
+      - bison
+      - bc
+      - kmod
+      - cpio
+      - libncurses5-dev
+      - libelf-dev
+      - libssl-dev
+      - libdw-dev
+      - dwarves
+    state: present
+    update_cache: true
+
+- name: Build kernel DEBs
+  shell: "ls linux-*.deb || (cd linux && make -j$(nproc) bindeb-pkg)"
+
+- name: Create vm:~/kernel-pkgs.tar containing kernel RPMs
+  shell: "tar cf ~/kernel-pkgs.tar linux-*.deb"
+
+- name: Make sure kernel uses serial console for debugging
+  become: yes
+  shell: "grep -sr console=ttyS /etc/default/grub* || echo 'GRUB_CMDLINE_LINUX=\"$GRUB_CMDLINE_LINUX console=ttyS0,115200n8 \"' > /etc/default/grub.d/e2e-cmdline-serialconsole.cfg"

test/e2e/playbook/custom-kernel.yaml

@@ -0,0 +1,92 @@
+---
+- name: Check if a custom kernel is required and available
+  hosts: all
+  vars:
+    cached_kernel_pkgs_tarball: ""
+    kernel_getsource: "{{ kernel_getsource }}"
+    kernel_getsource_cmd: ""
+    kernel_config: "{{ kernel_config }}"
+
+  tasks:
+    - name: Check if we have already built kernel in cache
+      when: kernel_getsource != '' and kernel_config != ''
+      block:
+        - name: Make sure kernel uses serial console for debugging
+          become: yes
+          shell: "grep -sr console=ttyS /etc/default/grub* || echo 'GRUB_CMDLINE_LINUX=\"$GRUB_CMDLINE_LINUX console=ttyS0,115200n8 \"' > /etc/default/grub.d/e2e-cmdline-serialconsole.cfg"
+
+        - name: Expand kernel_getsource shorthands
+          ansible.builtin.set_fact:
+            kernel_getsource_cmd: "{{ shorthands[kernel_getsource] | default(kernel_getsource) }}"
+          vars:
+            shorthands:
+              'vanilla': "git clone --depth 1 https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git linux"
+              'mainline': "git clone --depth 1 https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git linux"
+              'cxl-next': "git clone --depth 1 -b next https://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl.git linux"
+        - name: Generate kernel Fedora packages tarball name... getsource {{ kernel_getsource }} getsource_cmd {{ kernel_getsource_cmd }}
+          ansible.builtin.set_fact:
+            cached_kernel_pkgs_tarball: "{{ cache_dir }}/kernel.getsource_{{ kernel_getsource | regex_replace('[^a-zA-Z0-9-]+', '_') }}.config_{{ lookup('file', kernel_config) | hash('sha256') }}.rpms.tar"
+          when: ansible_facts['distribution'] == "Fedora"
+
+        - name: Generate kernel Ubuntu packages tarball name
+          ansible.builtin.set_fact:
+            cached_kernel_pkgs_tarball: "{{ cache_dir }}/kernel.getsource_{{ kernel_getsource | regex_replace('[^a-zA-Z0-9-]+', '_') }}.config_{{ lookup('file', kernel_config) | hash('sha256') }}.debs.tar"
+          when: ansible_facts['distribution'] == "Ubuntu"
+
+        - name: "Lookup cached kernel: {{ cached_kernel_pkgs_tarball }}"
+          delegate_to: localhost
+          stat:
+            path: "{{ cached_kernel_pkgs_tarball }}"
+          register: cached_kernel_stat
+
+        - name: If we have a cached kernel, push it to the vm
+          when: cached_kernel_stat.stat.exists
+          synchronize:
+            src: "{{ cached_kernel_pkgs_tarball }}"
+            dest: "/home/vagrant/kernel-pkgs.tar"
+            use_ssh_args: true
+
+        - name: If we do not have a cached kernel, fetch and build the kernel on vm
+          when: not cached_kernel_stat.stat.exists
+          block:
+            - name: Install kernel_getsource dependencies
+              become: yes
+              ansible.builtin.package:
+                pkg:
+                  - git
+                  - wget
+                  - rsync
+                state: present
+                update_cache: true
+
+            - name: Get kernel sources to vm:~/linux
+              vars:
+                source_id_file: "linux/.kernel_getsource.{{ kernel_getsource | regex_replace('[^a-zA-Z0-9-]+', '_') }}"
+              shell: |
+                [ -f {{ source_id_file }} ] || {
+                    rm -rf linux;
+                    {{ kernel_getsource_cmd }}
+
+                    touch {{ source_id_file }}
+                }
+
+            - name: Copy kernel config to the vm
+              synchronize:
+                src: "{{ kernel_config }}"
+                dest: "linux/.config"
+                use_ssh_args: true
+
+            - name: Build kernel pkgs on Fedora to vm:~/kernel-pkgs.tar
+              ansible.builtin.include_tasks: custom-kernel-fedora.yaml
+              when: ansible_facts['distribution'] == "Fedora"
+
+            - name: Build kernel pkgs on Ubuntu to vm:~/kernel-pkgs.tar
+              ansible.builtin.include_tasks: custom-kernel-ubuntu.yaml
+              when: ansible_facts['distribution'] == "Ubuntu"
+
+            - name: Cache built kernel packages on the host
+              synchronize:
+                src: "kernel-pkgs.tar"
+                dest: "{{ cached_kernel_pkgs_tarball }}"
+                use_ssh_args: true
+                mode: pull

test/e2e/playbook/deploy-policy-plugin.yaml

@@ -1,4 +1,7 @@
 ---
+- name: check need and availability of a custom kernel
+  ansible.builtin.import_playbook: custom-kernel.yaml
+
 - hosts: all
   become: no
   become_user: root

test/e2e/playbook/nri-memory-policy-plugin-deploy.yaml

@@ -1,4 +1,7 @@
 ---
+- name: check need and availability of a custom kernel
+  ansible.builtin.import_playbook: custom-kernel.yaml
+
 - hosts: all
   become: no
   become_user: root