diff --git a/roles/images/performance/defaults/main.yml b/roles/images/performance/defaults/main.yml index a1388b625..953147f6c 100644 --- a/roles/images/performance/defaults/main.yml +++ b/roles/images/performance/defaults/main.yml @@ -1,24 +1,16 @@ # Image Variables -guest_image: 'http://download.eng.tlv.redhat.com/rhel-9/rel-eng/RHEL-9/latest-RHEL-9.2.0/compose/BaseOS/x86_64/images/rhel-guest-image-9.2-20230414.17.x86_64.qcow2' +guest_image: 'http://download.hosts.prod.upshift.rdu2.redhat.com/rhel-9/rel-eng/RHEL-9/latest-RHEL-9.5.0/compose/BaseOS/x86_64/images/rhel-guest-image-9.5-20241009.2.x86_64.qcow2' guest_image_output: '/tmp/' guest_debug: False # Repo Variables repo_urls: - - http://download.eng.tlv.redhat.com/rhel-9/rel-eng/RHEL-9/latest-RHEL-9.2.0/repofile.repo + - http://download.hosts.prod.upshift.rdu2.redhat.com/rhel-9/rel-eng/RHEL-9/latest-RHEL-9.5.0/repofile.repo repo_dir: '/tmp/guest_repos' guest_repos: [] -# DPDK Variables -dpdk_dir: '/root/dpdk' -dpdk_customization_log: "{{ dpdk_dir }}/customization.log" -dpdk_git: 'git://dpdk.org/dpdk' -dpdk_branch: 'tags/v22.11' -dpdk_compile_packages: - - pyelftools - # Trex Variables -trex_version: 'v3.02' +trex_version: 'v3.06' trex_url: 'https://trex-tgn.cisco.com/trex/release' trex_dir: '/opt/trex/' @@ -41,7 +33,3 @@ fio_git: 'git://git.kernel.dk/fio.git' fio_dir: '/root/fio' fio_branch: 'fio-3.9' fio_customization_log: "{{ fio_dir }}/customization.log" - -# mellanox drivers -# mlx_iso: http://file.mad.redhat.com/~mnietoji/MLNX_OFED_LINUX-5.9-0.5.6.0-rhel9.1-x86_64.iso -mlx_iso: '' diff --git a/roles/images/performance/tasks/generic_customizations.yml b/roles/images/performance/tasks/generic_customizations.yml index 8c4f1ea62..eb323e8de 100644 --- a/roles/images/performance/tasks/generic_customizations.yml +++ b/roles/images/performance/tasks/generic_customizations.yml @@ -2,24 +2,14 @@ command: virt-customize -a {{ fetched_image }} --run-command "yum install {{ item }} -y" loop: - git - - gcc - - make - - cmake - - gcc-c++ - - pciutils - - glibc-devel - - glibc-headers - - numactl-devel - vim - tmux - python3 - cloud-init - - meson - pip - iperf3 - mstflint - tuned-profiles-cpu-partitioning - - perl - net-tools - name: create custom users diff --git a/roles/images/performance/tasks/prepare_dpdk.yml b/roles/images/performance/tasks/prepare_dpdk.yml index e5b88166a..4252e7a51 100644 --- a/roles/images/performance/tasks/prepare_dpdk.yml +++ b/roles/images/performance/tasks/prepare_dpdk.yml @@ -1,37 +1,12 @@ -- name: install dpdk compile dependency - command: virt-customize -a {{ fetched_image }} --run-command "pip3 install {{ item }}" - loop: "{{ dpdk_compile_packages }}" - -- name: configure mellanox drivers - block: - - name: create mellanox installation script - lineinfile: - dest: /tmp/mlx_script.sh - line: '{{ item }}' - create: true - mode: 0744 - with_items: - - 'curl -o /tmp/{{ mlx_iso | basename }} {{ mlx_iso }}' - - 'mount -o ro,loop /tmp/{{ mlx_iso | basename }} /mnt' - - 'cat <> /etc/yum.repos.d/mlnx_ofed.repo' - - '[mlnx_ofed]' - - 'name=MLNX_OFED Repository' - - 'baseurl=file:///mnt/RPMS' - - 'enabled=1' - - 'gpgcheck=0' - - 'EOF' - - 'yum install mlnx-ofed-all -y' - - 'rm /etc/yum.repos.d/mlnx_ofed.repo' - - - name: install mellanox drivers - command: virt-customize -v -x -a {{ fetched_image }} --run /tmp/mlx_script.sh - when: mlx_iso | length > 0 +- name: Install dpdk + command: virt-customize -a {{ fetched_image }} --run-command "yum install {{ item }} -y" + loop: + - dpdk + - driverctl + - pciutils -- name: compile dpdk and install trafficgen +- name: install trafficgen command: virt-customize -a {{ fetched_image }} --run-command "{{ item }}" loop: - - "if [ ! -d {{ dpdk_dir }} ];then git clone {{ dpdk_git }} {{ dpdk_dir }};cd {{ dpdk_dir }};git checkout {{ dpdk_branch }};fi" - "if [ ! -d {{ trafficgen_dir }} ];then git clone {{ trafficgen_git }} {{ trafficgen_dir }};cd {{ trafficgen_dir }};git checkout {{ trafficgen_branch }};fi" - "sed -i 's/enforcing/{{ selinux_config }}/g' /etc/selinux/config /etc/selinux/config" - - "cd {{ dpdk_dir }} ; meson setup build; ninja -C build" - - "echo 'DPDK Version: {{ dpdk_branch }}' | tee {{ dpdk_customization_log }}" diff --git a/roles/packet_gen/trex/README.md b/roles/packet_gen/trex/README.md index 6444ff225..4577758fa 100644 --- a/roles/packet_gen/trex/README.md +++ b/roles/packet_gen/trex/README.md @@ -116,9 +116,6 @@ binary_search: True States if config variables should be parsed during run time, if `False`, a user must pass the required trex configuration variables. `True` by default. -``` -gather_trex_conf_live: True -``` Destination out trex config file which will be created on trex instance. @@ -203,25 +200,6 @@ trafficgen_branch: 'master' ``` ### DPDK Variables - -DPDK root directory on instances(cloned from: 'git://dpdk.org/dpdk'). - -`/root/dpdk` by default. -``` -dpdk_root_dir: '/root/dpdk' -``` - -Compiled DPDK binaries directory on instances(compiled as part of NFV perf guest image). - -Info about what we compile can be found in `prepare_performance_images` [role documentation](/roles/images/performance/README.md#description). - -Compilation steps can be found inside `Prepare DPDK binaries inside guest image` [task](/roles/images/performance/tasks/prepare_dpdk.yml). - -`{{ dpdk_root_dir }}/build/app` by default. -``` -dpdk_compiled_dir: '/root/dpdk/build/app' -``` - Binding drivers used by dpdk (vfio-pci, mlx5_core) 'vfio-pci' by default diff --git a/roles/packet_gen/trex/binary_search/defaults/main.yml b/roles/packet_gen/trex/binary_search/defaults/main.yml index a3737fc52..f04349681 100644 --- a/roles/packet_gen/trex/binary_search/defaults/main.yml +++ b/roles/packet_gen/trex/binary_search/defaults/main.yml @@ -2,7 +2,7 @@ binary_perf_log: '/tmp/performance.log' # Trafficgen repo directory on host -trafficgen_dir: '/opt/trafficgen' +trafficgen_dir: '/opt/bench-trafficgen/trafficgen' # binary-search binary binary_search_bin: "{{ trafficgen_dir }}/binary-search.py" @@ -10,6 +10,9 @@ binary_search_bin: "{{ trafficgen_dir }}/binary-search.py" # Frame size in bytes that Trex will generate trex_frame_size: 64 +# Destination ports, one per device +dst_ports: 53000,54000 + # Max packet lost % acceptable trex_max_lost_pct: 0.00 @@ -68,4 +71,4 @@ binary_search_status_delay: 10 # What to do when negative packet loss is encountered # fail,quit,retry-to-fail,retry-to-quit -negative_packet_loss: retry-to-fail \ No newline at end of file +negative_packet_loss: retry-to-fail diff --git a/roles/packet_gen/trex/binary_search/tasks/main.yml b/roles/packet_gen/trex/binary_search/tasks/main.yml index f7efcaf54..93badb9a0 100644 --- a/roles/packet_gen/trex/binary_search/tasks/main.yml +++ b/roles/packet_gen/trex/binary_search/tasks/main.yml @@ -34,6 +34,7 @@ --max-loss-pct {{ trex_max_lost_pct }} --send-teaching-warmup --dst-macs {{ dut_macs }} + --dst-ports {{ dst_ports }} --num-flows {{ trex_flows }} {% if trex_set_vlan %} --vlan-ids {{ trex_vlans }} diff --git a/roles/packet_gen/trex/bind_dpdk_nics/defaults/main.yml b/roles/packet_gen/trex/bind_dpdk_nics/defaults/main.yml index 0d91ca64d..5b6e42f54 100644 --- a/roles/packet_gen/trex/bind_dpdk_nics/defaults/main.yml +++ b/roles/packet_gen/trex/bind_dpdk_nics/defaults/main.yml @@ -1,9 +1,3 @@ -# DPDK directory on host -dpdk_root_dir: '/root/dpdk' - -# Compiled DPDK binaries directory -dpdk_compiled_dir: '{{ dpdk_root_dir }}/build/app' - hugepages_count: 4 # dpdk drivers to use: vfio-pci, mlx5_core diff --git a/roles/packet_gen/trex/bind_dpdk_nics/tasks/main.yml b/roles/packet_gen/trex/bind_dpdk_nics/tasks/main.yml index 39d9199c8..6217f5bc9 100644 --- a/roles/packet_gen/trex/bind_dpdk_nics/tasks/main.yml +++ b/roles/packet_gen/trex/bind_dpdk_nics/tasks/main.yml @@ -1,10 +1,3 @@ -- name: Check if DPDK Compiled Binaries Directory Exists - stat: - path: "{{ dpdk_compiled_dir }}" - register: dpdk_dir - failed_when: not dpdk_dir['stat']['exists'] - become: True - - name: Retrieve default NIC set_fact: nic_def: "{{hostvars[inventory_hostname]['ansible_default_ipv4']['alias']}}" @@ -22,8 +15,11 @@ loop: "{{nic_down}}" become: True -- name: Retrieve Non Active NICs' PCI Slots - shell: "python {{ dpdk_root_dir }}/usertools/dpdk-devbind.py --status | grep -A 5 'Network devices using kernel driver' | grep -v Active | grep 0000 | cut -d ' ' -f 1 | paste -sd ' '" +- name: Retrieve pci slots for nics + shell: | + for nic in {{ nic_down | join(' ') }}; do + ethtool -i $nic | awk '{if ($1 == "bus-info:") print $2}' + done register: instance_nics_pci_slots become: True @@ -55,12 +51,9 @@ state: present params: "{{ item['params'] | default(omit) }}" loop: - - {'name': 'vfio'} + - {'name': 'vfio', 'params': 'enable_unsafe_noiommu_mode=1'} - {'name': 'vfio-pci'} - - name: Ensure enable_unsafe_noiommu_mode - shell: "echo 1 | sudo tee /sys/module/vfio/parameters/enable_unsafe_noiommu_mode" - - name: Load Mellanox Kernel Modules when configured modprobe: name: mlx5_core @@ -68,7 +61,8 @@ when: dpdk_binding_driver == "mlx5_core" - name: Bind DPDK NICs - command: "{{ dpdk_root_dir }}/usertools/dpdk-devbind.py -b {{dpdk_binding_driver}} {{ instance_nics_pci_slots['stdout'] }}" + command: "driverctl set-override {{ item }} {{dpdk_binding_driver}}" + loop: "{{ instance_nics_pci_slots.stdout_lines }}" become: True diff --git a/roles/packet_gen/trex/launch_testpmd/defaults/main.yml b/roles/packet_gen/trex/launch_testpmd/defaults/main.yml index 49a539387..bdb6f5131 100644 --- a/roles/packet_gen/trex/launch_testpmd/defaults/main.yml +++ b/roles/packet_gen/trex/launch_testpmd/defaults/main.yml @@ -1,5 +1,5 @@ # testpmd binary -testpmd_bin: "{{ dpdk_compiled_dir }}/testpmd" +testpmd_bin: "/usr/bin/dpdk-testpmd" # Map list of cores to physical cpu set testpmd_lcores: '3,4,7' @@ -26,10 +26,13 @@ testpmd_rxq: 1 testpmd_txq: 1 # TestPMD forwarding mode -forward_mode: io +forward_mode: mac # TestPMD verbose testpmd_verbose: 0 # TestPMD log file -testpmd_log: "/tmp/testpmd.log" \ No newline at end of file +testpmd_log: "/tmp/testpmd.log" + +# Enable IP forwarding in case L3 needed +packet_gen_ip_forwarding: false diff --git a/roles/packet_gen/trex/trex_instance_config/defaults/main.yaml b/roles/packet_gen/trex/trex_instance_config/defaults/main.yaml index 3d4b063a8..a40d3c3a8 100644 --- a/roles/packet_gen/trex/trex_instance_config/defaults/main.yaml +++ b/roles/packet_gen/trex/trex_instance_config/defaults/main.yaml @@ -1,21 +1,6 @@ -# Parse trex configuration during run time from trex instance -gather_trex_conf_live: True - # Destination of trex config file trex_conf_file: '/etc/trex_cfg.yaml' -# Clone trafficgen repo https://github.com/atheurer/trafficgen -clone_traffic_gen_repo: True - -# Trafficgen repo URI -trafficgen_repo: 'https://github.com/atheurer/trafficgen' - -# Trafficgen branch -trafficgen_branch: 'master' - -# Derecitory where trafficgen repo will be cloned to -trafficgen_dir: '/opt/trafficgen/' - # Platform sepcific Trex configuration trex_platform: master_thread_id: 2 @@ -24,8 +9,5 @@ trex_platform: - socket: 0 threads: [4, 5, 6, 7, 8, 9, 10, 11] -# DPDK root dir -dpdk_root_dir: '/root/dpdk' - # Workaround - can't define this variable inside playbook trex_full_nic_info: [] diff --git a/roles/packet_gen/trex/trex_instance_config/tasks/main.yml b/roles/packet_gen/trex/trex_instance_config/tasks/main.yml index 6f806f80f..9a2303bf0 100644 --- a/roles/packet_gen/trex/trex_instance_config/tasks/main.yml +++ b/roles/packet_gen/trex/trex_instance_config/tasks/main.yml @@ -1,24 +1,8 @@ -- name: Check for trafficgen repo - stat: - path: "{{ trafficgen_dir }}" - register: trafficgen_path +- name: Query Trex Instance If No External Configuration Is Passed + import_tasks: query_trex_instance.yaml -- name: Clone Trafficgen Repo - git: - repo: "{{ trafficgen_repo }}" - dest: "{{ trafficgen_dir }}" - version: "{{ trafficgen_branch }}" - become: True - when: clone_traffic_gen_repo and trafficgen_path.stat.exists == False - -- name: Parse Configuration From Instance - block: - - name: Query Trex Instance If No External Configuration Is Passed - import_tasks: query_trex_instance.yaml - - - name: Generate Variables For Trex Conf From Trex Instance - import_tasks: populate_trex_conf.yaml - when: gather_trex_conf_live +- name: Generate Variables For Trex Conf From Trex Instance + import_tasks: populate_trex_conf.yaml - name: Fail If No Trex Port Info Is Present For Jinja2 Template fail: diff --git a/roles/packet_gen/trex/trex_instance_config/tasks/populate_trex_conf.yaml b/roles/packet_gen/trex/trex_instance_config/tasks/populate_trex_conf.yaml index 676faf4da..a40570772 100644 --- a/roles/packet_gen/trex/trex_instance_config/tasks/populate_trex_conf.yaml +++ b/roles/packet_gen/trex/trex_instance_config/tasks/populate_trex_conf.yaml @@ -20,6 +20,26 @@ {%- endfor -%} {%- endfor -%} +- name: fail if no matches for trex nics + block: + - name: Info from neutron + debug: + var: trex_instance_nics_info + + - name: Ansible Gathered facts + debug: + var: ansible_local["nic_topology"] + + - name: Fail if trex_full_nic_info is empty + ansible.builtin.fail: + msg: >- + trex_full_nic_info is empty this issue might be cause either by + Trex nics already being counded to dpdk or + because of a mismatch between the real mac address of the trex PF and neutron API + please make sure nics in trex are not binded and that the PF mac address matches the + neutron port. + when: trex_full_nic_info | length == 0 + - name: Sort Trex NICs Info Dict Based On PCI Slots set_fact: trex_full_nic_info: "{{ trex_full_nic_info | sort(attribute='pci_slot') }}" diff --git a/roles/packet_gen/trex/trex_instance_config/tasks/query_trex_instance.yaml b/roles/packet_gen/trex/trex_instance_config/tasks/query_trex_instance.yaml index 4bf2d626a..68e28cc44 100644 --- a/roles/packet_gen/trex/trex_instance_config/tasks/query_trex_instance.yaml +++ b/roles/packet_gen/trex/trex_instance_config/tasks/query_trex_instance.yaml @@ -5,6 +5,9 @@ query_instance: "{{ ansible_hostname }}" delegate_to: "{{ groups['undercloud'] | first }}" +- debug: + var: instance_nics | flatten(levels=1) + - name: Parse Trex Server SR-IOV NICs set_fact: trex_instance_sriov_nics: "{{ trex_instance_sriov_nics | default([]) }} + [ {{ item }} ]" @@ -14,29 +17,19 @@ - name: Fail If There Are More/Less Than 2 SR-IOV NICs fail: - msg: "Current automated flow only supports 2 SR-IOV NICs connected to intance, please provide an instance with the allowed number of NICs" + msg: "Current automated flow only supports 2 SR-IOV NICs connected to instance, please provide an instance with the allowed number of NICs" when: trex_instance_sriov_nics | length != 2 -- name: "Check if DPDK Directory Exists on Trex Instance" - stat: - path: "{{ dpdk_root_dir }}" - register: dpdk_dir - failed_when: not dpdk_dir['stat']['exists'] - become: True - -- name: Retrieve info From dpdk-devbind script - shell: "python {{ dpdk_root_dir }}/usertools/dpdk-devbind.py --status" - register: dpdk_devbind_info - become: true - -- name: Check Existence Of Performance Ansible Local Facts - stat: - path: /etc/ansible/facts.d/nic_topology.fact - become: true - register: trex_nics_topology_fact_file +- name: Gather Ansible Facts + setup: - name: Dump Performance Facts To Ansible Local Facts block: + - name: Retrieve info From lshw command + shell: "lshw -C network -json" + register: lshw_info + become: true + - name: Create Local Facts Directory file: path: /etc/ansible/facts.d @@ -46,13 +39,7 @@ - name: Retrieve NICs Gathered By Ansible Facts set_fact: - ans_nics: >- - {{ ans_nics | default([]) }} + [{ - 'name': '{{ item }}', - 'mac_address': '{{ ansible_facts[item]['macaddress'] }}', - 'pci_slot': '{{ dpdk_devbind_info['stdout'] | regex_search('.{4}:.{2}:.{2}\..*if=' + item + '.*', - multiline=True) | regex_search('.{4}:.{2}:.{2}\..') }}' - }] + ans_nics: "{{ ans_nics | default([]) + [{'name': item, 'mac_address': ansible_facts[item]['macaddress'], 'pci_slot': ((lshw_info['stdout'] | from_json | selectattr('serial', 'defined') | selectattr('serial', 'equalto', ansible_facts[item]['macaddress']) | map(attribute='businfo') | list + ['']) | first) | regex_replace('^.*@', '') }] }}" loop: "{{ ansible_interfaces | difference(['lo']) }}" - name: Dump Content To File @@ -63,9 +50,9 @@ - name: Gather Ansible Facts setup: - when: not trex_nics_topology_fact_file['stat']['exists'] -- name: Deactivate Non External Kernel NICs - shell: "ip link set dev {{ item }} down" - loop: "{{ ansible_interfaces | difference(['lo', ansible_default_ipv4['alias']]) }}" - become: True + - name: Deactivate Non External Kernel NICs + shell: "ip link set dev {{ item }} down" + loop: "{{ ansible_interfaces | difference(['lo', ansible_default_ipv4['alias']]) }}" + become: + when: not "nic_topology" in ansible_local.keys() diff --git a/roles/tuning/cpu_pinning_huge_pages/tasks/main.yml b/roles/tuning/cpu_pinning_huge_pages/tasks/main.yml index c3fe9052e..c9e77103b 100644 --- a/roles/tuning/cpu_pinning_huge_pages/tasks/main.yml +++ b/roles/tuning/cpu_pinning_huge_pages/tasks/main.yml @@ -1,39 +1,26 @@ --- +- name: Set grub parameters + set_fact: + grub_params: > + default_hugepagesz={{ hugepages_size }} + hugepagesz={{ hugepages_size }} + hugepages={{ hugepages_count }} + transparent_hugepage=never + nohz=on + isolcpus={{ cpu_pinning_cores }} + nohz_full={{ cpu_pinning_cores }} + rcu_nocbs={{ cpu_pinning_cores }} -- name: Remove parameter from grub if it exists - lineinfile: - dest: /etc/default/grub - regexp: '^GRUB_CMDLINE_LINUX=(.* ){{ item.regex }}([a-zA-Z0-9-,]*)\s*(.*)' - line: 'GRUB_CMDLINE_LINUX=\1\3' - mode: '0644' - state: present - backrefs: yes - loop: - - { regex: 'isolcpus=', context: '{{ cpu_pinning_cores }}' } - - { regex: 'hugepagesz=', context: '{{ hugepages_size }}' } - - { regex: 'default_hugepagesz=', context: '{{ hugepages_size }}' } - - { regex: 'transparent_hugepage=', context: 'never' } - - { regex: 'hugepages=', context: '{{ hugepages_count }}' } - - { regex: 'nohz=on nohz_full=', context: '{{ cpu_pinning_cores }}' } - - { regex: 'rcu_nocbs=', context: '{{ cpu_pinning_cores }}' } - -- name: Add parameter to grub even if it didnt exist before - lineinfile: - dest: /etc/default/grub - regexp: '^GRUB_CMDLINE_LINUX="(.*)"' - line: 'GRUB_CMDLINE_LINUX="\1 {{ item.regex }}{{ item.context }}"' - mode: '0644' - state: present - backrefs: yes - loop: - - { regex: 'isolcpus=', context: '{{ cpu_pinning_cores }}' } - - { regex: 'hugepagesz=', context: '{{ hugepages_size }}' } - - { regex: 'default_hugepagesz=', context: '{{ hugepages_size }}' } - - { regex: 'transparent_hugepage=', context: 'never' } - - { regex: 'hugepages=', context: '{{ hugepages_count }}' } - - { regex: 'nohz=on nohz_full=', context: '{{ cpu_pinning_cores }}' } - - { regex: 'rcu_nocbs=', context: '{{ cpu_pinning_cores }}' } - register: grub +- name: Add parameter to grub + shell: | + pre=$(sudo grubby --info ALL | grep args) + grubby --update-kernel ALL --args '{{ grub_params }}' + after=$(sudo grubby --info ALL | grep args) + if [ "$pre" != "$after" ];then + echo "kernel args updated" + fi + register: makegrub + changed_when: '"kernel args updated" in makegrub.stdout' - name: Set the cores to isolate lineinfile: @@ -48,11 +35,6 @@ command: "/usr/sbin/tuned-adm profile cpu-partitioning" when: isolcpu_set.changed -- name: Make grub - command: "grub2-mkconfig -o /boot/grub2/grub.cfg" - when: grub.changed - register: makegrub - - name: Restart server shell: sleep 2 && shutdown -r now "Ansible updates triggered" async: 1