diff --git a/test/e2e/files/Vagrantfile.in b/test/e2e/files/Vagrantfile.in index 7820b142c..3831e21c1 100644 --- a/test/e2e/files/Vagrantfile.in +++ b/test/e2e/files/Vagrantfile.in @@ -15,6 +15,8 @@ HOSTNAME = "SERVER_NAME" # How many nodes to create N = 1 +KERNEL_GETSOURCE = "#{ENV['kernel_getsource']}" +KERNEL_CONFIG = "#{ENV['kernel_config']}" K8S_RELEASE = "#{ENV['k8s_release']}" K8S_VERSION = "#{ENV['k8s_version']}" HELM_RELEASE = "#{ENV['helm_release']}" @@ -108,6 +110,8 @@ Vagrant.configure("2") do |config| https_proxy: "#{ENV['HTTPS_PROXY']}", http_proxy: "#{ENV['HTTP_PROXY']}", no_proxy: "#{ENV['NO_PROXY']}", + kernel_getsource: KERNEL_GETSOURCE, + kernel_config: KERNEL_CONFIG, k8s_release: K8S_RELEASE, k8s_version: K8S_VERSION, helm_release: HELM_RELEASE, diff --git a/test/e2e/lib/vm.bash b/test/e2e/lib/vm.bash index 01f1665ac..fa50d2c75 100644 --- a/test/e2e/lib/vm.bash +++ b/test/e2e/lib/vm.bash @@ -274,7 +274,7 @@ vm-play() { -i "${vm}," -u vagrant \ --private-key="$private_key" \ --ssh-common-args "-F $vagrantdir/.ssh-config" \ - --extra-vars "cri_runtime=${k8scri} nri_resource_policy_src=${nri_resource_policy_src} cache_dir=$CACHE_DIR" + --extra-vars "cri_runtime=${k8scri} nri_resource_policy_src=${nri_resource_policy_src} cache_dir=$CACHE_DIR kernel_getsource=$kernel_getsource kernel_config=$kernel_config" ) } @@ -756,31 +756,128 @@ vm-set-kernel-cmdline() { fi } -fedora-set-kernel-cmdline() { - local e2e_defaults="$*" - vm-command "mkdir -p /etc/default; touch /etc/default/grub; sed -i '/e2e:fedora-set-kernel-cmdline/d' /etc/default/grub" - vm-command "echo 'GRUB_CMDLINE_LINUX_DEFAULT=\"\${GRUB_CMDLINE_LINUX_DEFAULT} ${e2e_defaults}\" # by e2e:fedora-set-kernel-cmdline' >> /etc/default/grub" || { - command-error "writing new command line parameters failed" - } - vm-command "grub2-mkconfig -o /boot/grub2/grub.cfg" || { - command-error "updating grub failed" - } +vm-kernel-pkgs-install() { # script API + # Usage: vm-kernel-pkgs-install + # + # Install custom kernel packages from a tar file. + # Does nothing if requested packages are already installed. + # + # Environment variables: + # file contains the name of the tarball on vm. + # The default is ~/kernel-pkgs.tar. + # + # Saves previous default to ~/.vm-kernel-pkgs.default-kernel. + # That kernel can be restored with vm-kernel-pkgs-uninstall. + local file="${file:-kernel-pkgs.tar}" + local feat="vm-kernel-pkgs" + + vm-command "tar tf $file" || \ + command-error "cannot list contents of kernel packages file $file" + local to_be_installed="$COMMAND_OUTPUT" + + vm-command "cat .$feat.installed_packages 2>/dev/null || echo ''" || \ + command-error "cannot read previously installed kernel packages list" + local already_installed="$COMMAND_OUTPUT" + + if [ "$to_be_installed" == "$already_installed" ]; then + echo "vm-kernel-pkgs-install: kernel packages from $file already installed, skipping installation" + return 0 + fi + + vm-command "[ -d $feat ] && rm -rf $feat; mkdir $feat; tar xvf $file -C $feat | tee .$feat.extracted_packages" || + command-error "cannot extract kernel packages from $file" + + if grep -q .rpm <<< "$to_be_installed"; then + # Store current kernel / Fedora + vm-command "grubby --default-kernel | tee .$feat.default-kernel" || \ + command-error "cannot save current default kernel" + # Install new kernel packages / Fedora + vm-command "rpm -Uvh $feat/*.rpm" || \ + command-error "cannot install kernel rpm packages from $file" + elif grep -q .deb <<< "$to_be_installed"; then + # Store current kernel / Ubuntu + vm-command "uname -r | tee .$feat.default-kernel" + # Install new kernel packages / Ubuntu + vm-command "dpkg -i $feat/*.deb" || \ + command-error "cannot install kernel deb packages from $file" + # In lack of grubby, set GRUB_DEFAULT in both "original" (for current) and "override" (for new kernel) configs. + # They both boot the system with current kernel and new kernel, respectively. + # When wish to change back from the new to the original kernel, it suffices to remove the "override" config + # and update grub. + vm-command " + oldkernelversion=\$(uname -r) + newkernelversion=\$(dpkg --contents $feat/linux-image-*.deb | awk -Fvmlinuz- /vmlinuz-/'{print \$2}') + submenu=\$(grep -i submenu.*advanced /boot/grub/grub.cfg | awk -F\' '{print \$2}') + oldmenuchoice=\$(grep \"menuentry.*\$oldkernelversion\" /boot/grub/grub.cfg | grep -v recovery | awk -F\' '{print \$2}') + newmenuchoice=\$(grep \"menuentry.*\$newkernelversion\" /boot/grub/grub.cfg | grep -v recovery | awk -F\' '{print \$2}') + echo 'Save original grub default' + echo \"GRUB_DEFAULT='\$submenu>\$oldmenuchoice'\" | tee /etc/default/grub.d/e2e-00-original-default-kernel.cfg + echo 'Override original grub default with:' + echo \"GRUB_DEFAULT='\$submenu>\$newmenuchoice'\" | tee /etc/default/grub.d/e2e-01-override-default-kernel.cfg + update-grub + " + else + command-error "no kernel packages found in $file" + fi + vm-command "cat .$feat.extracted_packages >> .$feat.installed_packages && rm .$feat.extracted_packages" || \ + command-error "cannot save installed kernel packages list" + + echo "Booting to new kernel..." + vm-reboot || \ + error "vm-kernel-pkgs-install: reboot failed after installing new kernel packages" + + vm-command "uname -a" } -ubuntu-set-kernel-cmdline() { - local e2e_defaults="$*" - vm-command "echo 'GRUB_CMDLINE_LINUX_DEFAULT=\"\${GRUB_CMDLINE_LINUX_DEFAULT} ${e2e_defaults}\"' > /etc/default/grub.d/60-e2e-defaults.cfg" || { - command-error "writing new command line parameters failed" - } - vm-command "update-grub" || { - command-error "updating grub failed" +vm-kernel-pkgs-uninstall() { # script API + # Usage: vm-kernel-pkgs-uninstall + # + # Boot to previous kernel before vm-kernel-pkgs-install and + # uninstall custom kernel packages installed with vm-kernel-pkgs-install. + local feat="vm-kernel-pkgs" + local default_kernel + local installed_packages + vm-command "cat .$feat.default-kernel" || { + echo "vm-kernel-pkgs-uninstall: kernel-pkgs not installed" + return 0 } -} -vm-set-kernel-cmdline() { - if [[ "$distro" == *fedora* ]]; then - fedora-set-kernel-cmdline "$*" + default_kernel="$COMMAND_OUTPUT" + if [ -z "$default_kernel" ]; then + command-error "cannot restore previous default kernel, file ~/.$feat.default-kernel is missing or empty" + fi + + vm-command "cat .$feat.installed_packages" || \ + command-error "cannot find installed kernel packages list file ~/.$feat.installed_packages" + installed_packages="$COMMAND_OUTPUT" + if [ -z "$installed_packages" ]; then + command-error "cannot uninstall kernel packages, file ~/.$feat.installed_packages is missing or empty" + fi + + # Modify grub to select previous kernel on next boot + if grep -q rpm <<< "$installed_packages"; then + vm-command "grubby --set-default=\"$default_kernel\"" || \ + command-error "cannot restore previous default kernel to $default_kernel" else - ubuntu-set-kernel-cmdline "$*" + vm-command "rm -f /etc/default/grub.d/e2e-01-override-default-kernel.cfg; update-grub" fi + + echo "Booting to previous kernel: $default_kernel." + vm-reboot + + # Uninstall previously installed custom kernel packages. + if grep -q rpm <<< "$installed_packages"; then + vm-command "sed 's/.rpm//g' < .$feat.installed_packages | xargs rpm -e --nodeps" || \ + command-error "failed to uninstall packages: $installed_packages" + elif grep -q deb <<< "$installed_packages"; then + vm-command "xargs -a .$feat.installed_packages dpkg -r" || \ + command-error "failed to uninstall packages: $installed_packages" + vm-command "rm -f /etc/default/grub.d/e2e-00-original-default-kernel.cfg; update-grub" + else + command-error "no kernel packages found in ~/.$feat.installed_packages" + fi + vm-command "rm -f .$feat.installed_packages .$feat.default-kernel" || \ + command-error "cannot remove ~/.$feat.installed_packages and ~/.$feat.default-kernel files" + + vm-command "uname -a" } diff --git a/test/e2e/playbook/custom-kernel-fedora.yaml b/test/e2e/playbook/custom-kernel-fedora.yaml new file mode 100644 index 000000000..a35d94adc --- /dev/null +++ b/test/e2e/playbook/custom-kernel-fedora.yaml @@ -0,0 +1,29 @@ +--- +- name: Install kernel build dependencies + become: yes + ansible.builtin.package: + pkg: + - fedpkg + - fedora-packager + - rpmdevtools + - ncurses-devel + - pesign + - grubby + - make + - gcc + - flex + - bison + - elfutils-devel + - elfutils-libelf-devel + - dwarves + - openssl + - openssl-devel + - perl + state: present + update_cache: true + +- name: Build kernel RPMs + shell: "cd linux && make -j$(nproc) binrpm-pkg" + +- name: Create vm:~/kernel-pkgs.tar containing kernel RPMs + shell: "cd linux/rpmbuild/RPMS/x86_64; tar cf ~/kernel-pkgs.tar kernel-*.rpm" diff --git a/test/e2e/playbook/custom-kernel-ubuntu.yaml b/test/e2e/playbook/custom-kernel-ubuntu.yaml new file mode 100644 index 000000000..fe19d761b --- /dev/null +++ b/test/e2e/playbook/custom-kernel-ubuntu.yaml @@ -0,0 +1,30 @@ +--- +- name: Install kernel build dependencies + become: yes + ansible.builtin.package: + pkg: + - git-core + - build-essential + - debhelper-compat + - flex + - bison + - bc + - kmod + - cpio + - libncurses5-dev + - libelf-dev + - libssl-dev + - libdw-dev + - dwarves + state: present + update_cache: true + +- name: Build kernel DEBs + shell: "ls linux-*.deb || (cd linux && make -j$(nproc) bindeb-pkg)" + +- name: Create vm:~/kernel-pkgs.tar containing kernel RPMs + shell: "tar cf ~/kernel-pkgs.tar linux-*.deb" + +- name: Make sure kernel uses serial console for debugging + become: yes + shell: "grep -sr console=ttyS /etc/default/grub* || echo 'GRUB_CMDLINE_LINUX=\"$GRUB_CMDLINE_LINUX console=ttyS0,115200n8 \"' > /etc/default/grub.d/e2e-cmdline-serialconsole.cfg" diff --git a/test/e2e/playbook/custom-kernel.yaml b/test/e2e/playbook/custom-kernel.yaml new file mode 100644 index 000000000..34aec83e6 --- /dev/null +++ b/test/e2e/playbook/custom-kernel.yaml @@ -0,0 +1,92 @@ +--- +- name: Check if a custom kernel is required and available + hosts: all + vars: + cached_kernel_pkgs_tarball: "" + kernel_getsource: "{{ kernel_getsource }}" + kernel_getsource_cmd: "" + kernel_config: "{{ kernel_config }}" + + tasks: + - name: Check if we have already built kernel in cache + when: kernel_getsource != '' and kernel_config != '' + block: + - name: Make sure kernel uses serial console for debugging + become: yes + shell: "grep -sr console=ttyS /etc/default/grub* || echo 'GRUB_CMDLINE_LINUX=\"$GRUB_CMDLINE_LINUX console=ttyS0,115200n8 \"' > /etc/default/grub.d/e2e-cmdline-serialconsole.cfg" + + - name: Expand kernel_getsource shorthands + ansible.builtin.set_fact: + kernel_getsource_cmd: "{{ shorthands[kernel_getsource] | default(kernel_getsource) }}" + vars: + shorthands: + 'vanilla': "git clone --depth 1 https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git linux" + 'mainline': "git clone --depth 1 https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git linux" + 'cxl-next': "git clone --depth 1 -b next https://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl.git linux" + - name: Generate kernel Fedora packages tarball name... getsource {{ kernel_getsource }} getsource_cmd {{ kernel_getsource_cmd }} + ansible.builtin.set_fact: + cached_kernel_pkgs_tarball: "{{ cache_dir }}/kernel.getsource_{{ kernel_getsource | regex_replace('[^a-zA-Z0-9-]+', '_') }}.config_{{ lookup('file', kernel_config) | hash('sha256') }}.rpms.tar" + when: ansible_facts['distribution'] == "Fedora" + + - name: Generate kernel Ubuntu packages tarball name + ansible.builtin.set_fact: + cached_kernel_pkgs_tarball: "{{ cache_dir }}/kernel.getsource_{{ kernel_getsource | regex_replace('[^a-zA-Z0-9-]+', '_') }}.config_{{ lookup('file', kernel_config) | hash('sha256') }}.debs.tar" + when: ansible_facts['distribution'] == "Ubuntu" + + - name: "Lookup cached kernel: {{ cached_kernel_pkgs_tarball }}" + delegate_to: localhost + stat: + path: "{{ cached_kernel_pkgs_tarball }}" + register: cached_kernel_stat + + - name: If we have a cached kernel, push it to the vm + when: cached_kernel_stat.stat.exists + synchronize: + src: "{{ cached_kernel_pkgs_tarball }}" + dest: "/home/vagrant/kernel-pkgs.tar" + use_ssh_args: true + + - name: If we do not have a cached kernel, fetch and build the kernel on vm + when: not cached_kernel_stat.stat.exists + block: + - name: Install kernel_getsource dependencies + become: yes + ansible.builtin.package: + pkg: + - git + - wget + - rsync + state: present + update_cache: true + + - name: Get kernel sources to vm:~/linux + vars: + source_id_file: "linux/.kernel_getsource.{{ kernel_getsource | regex_replace('[^a-zA-Z0-9-]+', '_') }}" + shell: | + [ -f {{ source_id_file }} ] || { + rm -rf linux; + {{ kernel_getsource_cmd }} + + touch {{ source_id_file }} + } + + - name: Copy kernel config to the vm + synchronize: + src: "{{ kernel_config }}" + dest: "linux/.config" + use_ssh_args: true + + - name: Build kernel pkgs on Fedora to vm:~/kernel-pkgs.tar + ansible.builtin.include_tasks: custom-kernel-fedora.yaml + when: ansible_facts['distribution'] == "Fedora" + + - name: Build kernel pkgs on Ubuntu to vm:~/kernel-pkgs.tar + ansible.builtin.include_tasks: custom-kernel-ubuntu.yaml + when: ansible_facts['distribution'] == "Ubuntu" + + - name: Cache built kernel packages on the host + synchronize: + src: "kernel-pkgs.tar" + dest: "{{ cached_kernel_pkgs_tarball }}" + use_ssh_args: true + mode: pull diff --git a/test/e2e/playbook/deploy-policy-plugin.yaml b/test/e2e/playbook/deploy-policy-plugin.yaml index e62b9ad2d..c3d604848 100644 --- a/test/e2e/playbook/deploy-policy-plugin.yaml +++ b/test/e2e/playbook/deploy-policy-plugin.yaml @@ -1,4 +1,7 @@ --- +- name: check need and availability of a custom kernel + ansible.builtin.import_playbook: custom-kernel.yaml + - hosts: all become: no become_user: root diff --git a/test/e2e/playbook/nri-memory-policy-plugin-deploy.yaml b/test/e2e/playbook/nri-memory-policy-plugin-deploy.yaml index 7d9afd8d7..5c89d3689 100644 --- a/test/e2e/playbook/nri-memory-policy-plugin-deploy.yaml +++ b/test/e2e/playbook/nri-memory-policy-plugin-deploy.yaml @@ -1,4 +1,7 @@ --- +- name: check need and availability of a custom kernel + ansible.builtin.import_playbook: custom-kernel.yaml + - hosts: all become: no become_user: root