Skip to content

Commit 4697216

Browse files
Created separate role for sriov
Signed-off-by: Marikkannu, Suresh <[email protected]>
1 parent 1b69b46 commit 4697216

File tree

11 files changed

+150
-71
lines changed

11 files changed

+150
-71
lines changed

Makefile

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,8 @@ export EXTRA_VARS ?= "@$(5GC_ROOT_DIR)/vars/main.yml"
1818
--extra-vars "ROOT_DIR=$(ROOT_DIR)" --extra-vars $(EXTRA_VARS)
1919

2020
#### b. Provision k8s ####
21-
5gc-install: 5gc-router-install 5gc-core-install
22-
5gc-uninstall: 5gc-core-uninstall 5gc-router-uninstall
21+
5gc-install: 5gc-router-install 5gc-sriov-install 5gc-core-install
22+
5gc-uninstall: 5gc-core-uninstall 5gc-sriov-uninstall 5gc-router-uninstall
2323

2424
#### c. Provision router ####
2525
5gc-router-install:
@@ -29,7 +29,15 @@ export EXTRA_VARS ?= "@$(5GC_ROOT_DIR)/vars/main.yml"
2929
ansible-playbook -i $(HOSTS_INI_FILE) $(5GC_ROOT_DIR)/router.yml --tags uninstall \
3030
--extra-vars "ROOT_DIR=$(ROOT_DIR)" --extra-vars $(EXTRA_VARS)
3131

32-
#### d. Provision core ####
32+
#### d. Provision sriov ####
33+
5gc-sriov-install:
34+
ansible-playbook -i $(HOSTS_INI_FILE) $(5GC_ROOT_DIR)/sriov.yml --tags install \
35+
--extra-vars "ROOT_DIR=$(ROOT_DIR)" --extra-vars $(EXTRA_VARS)
36+
5gc-sriov-uninstall:
37+
ansible-playbook -i $(HOSTS_INI_FILE) $(5GC_ROOT_DIR)/sriov.yml --tags uninstall \
38+
--extra-vars "ROOT_DIR=$(ROOT_DIR)" --extra-vars $(EXTRA_VARS)
39+
40+
#### e. Provision core ####
3341
5gc-core-install:
3442
ansible-playbook -i $(HOSTS_INI_FILE) $(5GC_ROOT_DIR)/core.yml --tags install \
3543
--extra-vars "ROOT_DIR=$(ROOT_DIR)" --extra-vars $(EXTRA_VARS)
@@ -39,7 +47,7 @@ export EXTRA_VARS ?= "@$(5GC_ROOT_DIR)/vars/main.yml"
3947
5gc-core-reset: 5gc-core-uninstall 5gc-core-install
4048

4149

42-
#### e. Provision Multiple UPF ###
50+
#### f. Provision Multiple UPF ###
4351
5gc-upf-install:
4452
ansible-playbook -i $(HOSTS_INI_FILE) $(5GC_ROOT_DIR)/upf.yml --tags install \
4553
--extra-vars "ROOT_DIR=$(ROOT_DIR)" --extra-vars $(EXTRA_VARS)

roles/core/templates/sdcore-5g-values.yaml

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -297,8 +297,8 @@ omec-user-plane:
297297
nodeSelectors:
298298
enabled: true
299299
resources:
300-
enabled: {{ core.upf.sriov }}
301-
privileged: {{ core.upf.sriov }}
300+
enabled: {{ core.upf.enable_resources }}
301+
privileged: {{ core.upf.enable_resources }}
302302
images:
303303
repository: "" # defaults to Docker Hub
304304
# following two lines pull busybox from Docker Hub instead of Aether Registry
@@ -311,11 +311,11 @@ omec-user-plane:
311311
config:
312312
upf:
313313
name: "oaisim"
314-
privileged: {{ core.upf.sriov }}
314+
privileged: {{ core.upf.enable_resources }}
315315
sriov:
316-
enabled: {{ core.upf.sriov }} # SRIOV is disabled by default
316+
enabled: {{ core.upf.enable_resources }} # SRIOV is disabled by default
317317
hugepage:
318-
enabled: {{ core.upf.sriov }} # Should be enabled if DPDK is enabled
318+
enabled: {{ core.upf.enable_resources }} # Should be enabled if DPDK is enabled
319319
cniPlugin: {{ core.upf.cniPlugin }} # Can be any other plugin. Dictates how IP address are assigned
320320
ipam: static
321321
routes:
@@ -341,6 +341,10 @@ omec-user-plane:
341341
measure_upf: true
342342
measure_flow: true
343343
gtppsc: true # Extension header enabled in 5G.
344+
# DDP is INTEL NIC's specific feature (supported in E810 series)
345+
# which helps to improve UPF packet processing performance.
346+
# For more details on DDP feature and its usage please refer
347+
# "https://www.intel.com/content/www/us/en/download/19660/intel-ethernet-800-series-dynamic-device-personalization-ddp-for-telecommunication-comms-package.html"
344348
ddp: {{ core.upf.ddp }}
345349
cpiface:
346350
dnn: "internet" # Must match Slice dnn

roles/router/tasks/install.yml

Lines changed: 2 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -160,56 +160,5 @@
160160
when: inventory_hostname in groups['master_nodes']
161161
become: true
162162

163-
- name: deploy sriov plugins
164-
block:
165-
- name: Get number of VF's available for the "{{ core.data_iface }}"
166-
shell: ip link show "{{ core.data_iface }}"|grep -c vf
167-
register: vf_status
168-
changed_when: false
169-
170-
- debug:
171-
var=vf_status.stdout
172-
173-
- name: verify the required VF resourses available in "{{ core.data_iface }}"
174-
ansible.builtin.assert:
175-
that:
176-
- vf_status.stdout | int >= 2
177-
178-
- name: copy sriov-device-plugin.yaml to /tmp/sriov-device-plugin.yaml
179-
template:
180-
src: roles/router/templates/sriov-device-plugin.yaml
181-
dest: "/tmp/sriov-device-plugin.yaml"
182-
183-
- name: copy sriov-device-plugin-config.yaml to /tmp/sriov-device-plugin-config.yaml
184-
template:
185-
src: roles/router/templates/sriov-device-plugin-config.yaml
186-
dest: "/tmp/sriov-device-plugin-config.yaml"
187-
188-
- name: update proper data interface name in /tmp/sriov-device-plugin-config.yaml
189-
shell: sed -i -e "s/PFDEV/{{ core.data_iface }}/g" /tmp/sriov-device-plugin-config.yaml
190-
191-
- name: apply /tmp/sriov-device-plugin.yaml
192-
shell: kubectl apply -f /tmp/sriov-device-plugin.yaml
193-
194-
- name: apply /tmp/sriov-device-plugin-config.yaml
195-
shell: kubectl apply -f /tmp/sriov-device-plugin-config.yaml
196-
197-
- name: Get Sriov Pods Status
198-
shell: kubectl get pods -A |grep sriov
199-
register: pod_status
200-
changed_when: false
201-
202-
- name: restart sriov Pod
203-
shell: |
204-
while IFS= read -r line; do
205-
name=$(echo "$line" | awk '{print $2}')
206-
status=$(echo "$line" | awk '{print $4}')
207-
208-
kubectl delete pod "$name"
209-
done <<< "{{ pod_status.stdout }}"
210-
args:
211-
executable: /bin/bash
212-
when: inventory_hostname in groups['master_nodes'] and core.upf.sriov
213-
always:
214-
- pause:
215-
seconds: 10
163+
- pause:
164+
seconds: 15

roles/router/tasks/uninstall.yml

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -162,15 +162,6 @@
162162
become: true
163163
ignore_errors: yes
164164

165-
- name: remove sriov plugin files in /tmp
166-
file:
167-
path: '{{ item }}'
168-
state: absent
169-
with_items:
170-
- /tmp/sriov-device-plugin.yaml
171-
- /tmp/sriov-device-plugin-config.yaml
172-
when: inventory_hostname in groups['master_nodes'] and core.upf.sriov
173-
174165
# - name: stop aether-ue-nat.service
175166
# systemd:
176167
# name: aether-ue-nat.service

roles/sriov/defaults/main.yml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
core:
2+
data_iface: data
3+
upf:
4+
mode: af_packet
5+
sriov: false
6+
cniPlugin: macvlan
7+
ddp: false

roles/sriov/tasks/install.yml

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
---
2+
3+
# TODO: running on master node for now (fix to run on multiple nodes)
4+
5+
- name: check VF resources and bind them to vfio-pci
6+
block:
7+
- name: Get number of VF's available for the "{{ core.data_iface }}"
8+
shell: ip link show "{{ core.data_iface }}"|grep -c vf
9+
register: vf_status
10+
changed_when: false
11+
12+
- debug:
13+
var=vf_status.stdout
14+
15+
- name: verify the required VF resourses available in "{{ core.data_iface }}"
16+
ansible.builtin.assert:
17+
that:
18+
- vf_status.stdout | int >= 2
19+
20+
- name: check if "{{ core.upf.pci_devices[0] }}" attached to vfio-pci
21+
shell: driverctl -v list-overrides |grep 'vfio-pci'|grep {{ core.upf.pci_devices[0] }}|wc -l
22+
register: vf_status
23+
changed_when: false
24+
25+
- name: Attach "{{ core.upf.pci_devices[0] }}" to vfio-pci
26+
shell: driverctl set-override {{ core.upf.pci_devices[0] }} vfio-pci
27+
changed_when: false
28+
become: true
29+
when: vf_status.stdout | int != 1
30+
31+
- pause:
32+
seconds: 5
33+
34+
- name: check if "{{ core.upf.pci_devices[1] }}" attached to vfio-pci
35+
shell: driverctl -v list-overrides |grep 'vfio-pci'|grep {{ core.upf.pci_devices[1] }}|wc -l
36+
register: vf_status
37+
changed_when: false
38+
39+
- name: Attach "{{ core.upf.pci_devices[1] }}" to vfio-pci
40+
shell: driverctl set-override {{ core.upf.pci_devices[1] }} vfio-pci
41+
changed_when: false
42+
become: true
43+
when: vf_status.stdout | int != 1
44+
45+
- pause:
46+
seconds: 5
47+
48+
when: inventory_hostname in groups['master_nodes'] and core.upf.enable_resources
49+
50+
- name: deploy sriov plugins
51+
block:
52+
- name: copy sriov-device-plugin.yaml to /tmp/sriov-device-plugin.yaml
53+
template:
54+
src: roles/sriov/templates/sriov-device-plugin.yaml
55+
dest: "/tmp/sriov-device-plugin.yaml"
56+
57+
- name: copy sriov-device-plugin-config.yaml to /tmp/sriov-device-plugin-config.yaml
58+
template:
59+
src: roles/sriov/templates/sriov-device-plugin-config.yaml
60+
dest: "/tmp/sriov-device-plugin-config.yaml"
61+
62+
- name: update proper data interface name in /tmp/sriov-device-plugin-config.yaml
63+
shell: sed -i -e "s/PFDEV/{{ core.data_iface }}/g" /tmp/sriov-device-plugin-config.yaml
64+
65+
- name: apply /tmp/sriov-device-plugin.yaml
66+
shell: kubectl apply -f /tmp/sriov-device-plugin.yaml
67+
68+
- name: apply /tmp/sriov-device-plugin-config.yaml
69+
shell: kubectl apply -f /tmp/sriov-device-plugin-config.yaml
70+
71+
- name: Get Sriov Pods Status
72+
shell: kubectl get pods -A |grep sriov
73+
register: pod_status
74+
changed_when: false
75+
76+
- name: restart sriov Pod
77+
shell: |
78+
while IFS= read -r line; do
79+
name=$(echo "$line" | awk '{print $2}')
80+
status=$(echo "$line" | awk '{print $4}')
81+
82+
kubectl delete pod "$name"
83+
done <<< "{{ pod_status.stdout }}"
84+
args:
85+
executable: /bin/bash
86+
- pause:
87+
seconds: 10
88+
when: inventory_hostname in groups['master_nodes'] and core.upf.enable_resources

roles/sriov/tasks/main.yml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
---
2+
3+
- name: install 5gc/sriov
4+
import_tasks: roles/sriov/tasks/install.yml
5+
tags: install
6+
7+
- name: uninstall 5gc/sriov
8+
import_tasks: roles/sriov/tasks/uninstall.yml
9+
tags: uninstall

roles/sriov/tasks/uninstall.yml

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
---
2+
3+
# TODO: running on master node for now (fix to run on multiple nodes)
4+
5+
- name: remove sriov plugin files in /tmp
6+
file:
7+
path: '{{ item }}'
8+
state: absent
9+
with_items:
10+
- /tmp/sriov-device-plugin.yaml
11+
- /tmp/sriov-device-plugin-config.yaml
12+
when: inventory_hostname in groups['master_nodes'] and core.upf.enable_resources
File renamed without changes.
File renamed without changes.

0 commit comments

Comments
 (0)