Skip to content

Commit 7842137

Browse files
sureshmarikkannusureshmarikkannu
committed
Added support to configure/use DPDK/SRIOV mode for UPF
Signed-off-by: Marikkannu, Suresh <[email protected]>
1 parent 5ab83d9 commit 7842137

File tree

5 files changed

+188
-7
lines changed

5 files changed

+188
-7
lines changed

‎roles/core/templates/sdcore-5g-values.yaml‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -297,7 +297,8 @@ omec-user-plane:
297297
nodeSelectors:
298298
enabled: true
299299
resources:
300-
enabled: false
300+
enabled: {{ core.upf.sriov }}
301+
privileged: {{ core.upf.sriov }}
301302
images:
302303
repository: "" # defaults to Docker Hub
303304
# following two lines pull busybox from Docker Hub instead of Aether Registry
@@ -310,11 +311,12 @@ omec-user-plane:
310311
config:
311312
upf:
312313
name: "oaisim"
314+
privileged: {{ core.upf.sriov }}
313315
sriov:
314-
enabled: false # SRIOV is disabled by default
316+
enabled: {{ core.upf.sriov }} # SRIOV is disabled by default
315317
hugepage:
316-
enabled: false # Should be enabled if DPDK is enabled
317-
cniPlugin: macvlan # Can be any other plugin. Dictates how IP address are assigned
318+
enabled: {{ core.upf.sriov }} # Should be enabled if DPDK is enabled
319+
cniPlugin: {{ core.upf.cniPlugin }} # Can be any other plugin. Dictates how IP address are assigned
318320
ipam: static
319321
routes:
320322
- to: {{ ansible_default_ipv4.address }}
@@ -323,20 +325,23 @@ omec-user-plane:
323325
subnet: {{ ran_subnet }} # Subnet for the gNB network
324326
access:
325327
iface: {{ core.data_iface }}
328+
resourceName: "intel.com/intel_sriov_vfio_access"
326329
gateway: 192.168.252.1
327330
ip: 192.168.252.3/24
328331
core:
329332
iface: {{ core.data_iface }}
333+
resourceName: "intel.com/intel_sriov_vfio_core"
330334
gateway: 192.168.250.1
331335
ip: 192.168.250.3/24
332336
cfgFiles:
333337
upf.jsonc:
334-
mode: af_packet # This mode implies no DPDK
338+
mode: {{ core.upf.mode }}
335339
hwcksum: true
336340
log_level: "trace"
337341
measure_upf: true
338342
measure_flow: true
339343
gtppsc: true # Extension header enabled in 5G.
344+
ddp: {{ core.upf.ddp }}
340345
cpiface:
341346
dnn: "internet" # Must match Slice dnn
342347
hostname: "upf"

‎roles/router/tasks/install.yml‎

Lines changed: 53 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -160,5 +160,56 @@
160160
when: inventory_hostname in groups['master_nodes']
161161
become: true
162162

163-
- pause:
164-
seconds: 15
163+
- name: deploy sriov plugins
164+
block:
165+
- name: Get number of VF's available for the "{{ core.data_iface }}"
166+
shell: ip link show "{{ core.data_iface }}"|grep -c vf
167+
register: vf_status
168+
changed_when: false
169+
170+
- debug:
171+
var=vf_status.stdout
172+
173+
- name: verify the required VF resourses available in "{{ core.data_iface }}"
174+
ansible.builtin.assert:
175+
that:
176+
- vf_status.stdout | int >= 2
177+
178+
- name: copy sriov-device-plugin.yaml to /tmp/sriov-device-plugin.yaml
179+
template:
180+
src: roles/router/templates/sriov-device-plugin.yaml
181+
dest: "/tmp/sriov-device-plugin.yaml"
182+
183+
- name: copy sriov-device-plugin-config.yaml to /tmp/sriov-device-plugin-config.yaml
184+
template:
185+
src: roles/router/templates/sriov-device-plugin-config.yaml
186+
dest: "/tmp/sriov-device-plugin-config.yaml"
187+
188+
- name: update proper data interface name in /tmp/sriov-device-plugin-config.yaml
189+
shell: sed -i -e "s/PFDEV/{{ core.data_iface }}/g" /tmp/sriov-device-plugin-config.yaml
190+
191+
- name: apply /tmp/sriov-device-plugin.yaml
192+
shell: kubectl apply -f /tmp/sriov-device-plugin.yaml
193+
194+
- name: apply /tmp/sriov-device-plugin-config.yaml
195+
shell: kubectl apply -f /tmp/sriov-device-plugin-config.yaml
196+
197+
- name: Get Sriov Pods Status
198+
shell: kubectl get pods -A |grep sriov
199+
register: pod_status
200+
changed_when: false
201+
202+
- name: restart sriov Pod
203+
shell: |
204+
while IFS= read -r line; do
205+
name=$(echo "$line" | awk '{print $2}')
206+
status=$(echo "$line" | awk '{print $4}')
207+
208+
kubectl delete pod "$name"
209+
done <<< "{{ pod_status.stdout }}"
210+
args:
211+
executable: /bin/bash
212+
when: inventory_hostname in groups['master_nodes'] and core.upf.sriov
213+
always:
214+
- pause:
215+
seconds: 10

‎roles/router/tasks/uninstall.yml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,15 @@
162162
become: true
163163
ignore_errors: yes
164164

165+
- name: remove sriov plugin files in /tmp
166+
file:
167+
path: '{{ item }}'
168+
state: absent
169+
with_items:
170+
- /tmp/sriov-device-plugin.yaml
171+
- /tmp/sriov-device-plugin-config.yaml
172+
when: inventory_hostname in groups['master_nodes'] and core.upf.sriov
173+
165174
# - name: stop aether-ue-nat.service
166175
# systemd:
167176
# name: aether-ue-nat.service

‎roles/router/templates/sriov-device-plugin-config.yaml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
# Copyright 2022-present Open Networking Foundation
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
apiVersion: v1
5+
kind: ConfigMap
6+
metadata:
7+
name: sriovdp-config
8+
data:
9+
config.json: |
10+
{
11+
"resourceList": [
12+
{
13+
"resourcePrefix": "intel.com",
14+
"resourceName": "intel_sriov_vfio_access",
15+
"selectors": {
16+
"pfNames": ["PFDEV#0"],
17+
"vendors": ["8086"],
18+
"drivers": ["vfio-pci"]
19+
}
20+
},
21+
{
22+
"resourcePrefix": "intel.com",
23+
"resourceName": "intel_sriov_vfio_core",
24+
"selectors": {
25+
"pfNames": ["PFDEV#1"],
26+
"vendors": ["8086"],
27+
"drivers": ["vfio-pci"]
28+
}
29+
}
30+
]
31+
}
32+

‎roles/router/templates/sriov-device-plugin.yaml‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
# Copyright 2022-present Open Networking Foundation
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
apiVersion: v1
5+
kind: ServiceAccount
6+
metadata:
7+
name: sriov-device-plugin
8+
---
9+
apiVersion: apps/v1
10+
kind: DaemonSet
11+
metadata:
12+
name: kube-sriov-device-plugin-amd64
13+
labels:
14+
tier: node
15+
app: sriovdp
16+
spec:
17+
selector:
18+
matchLabels:
19+
name: sriov-device-plugin
20+
template:
21+
metadata:
22+
labels:
23+
name: sriov-device-plugin
24+
tier: node
25+
app: sriovdp
26+
spec:
27+
hostNetwork: true
28+
nodeSelector:
29+
beta.kubernetes.io/arch: amd64
30+
tolerations:
31+
- key: node-role.kubernetes.io/master
32+
operator: Exists
33+
effect: NoSchedule
34+
initContainers:
35+
- name: init-sriov-plugin
36+
image: docker.io/omecproject/omec-cni:1.1.6
37+
imagePullPolicy: IfNotPresent
38+
command: ["bash", "-c"]
39+
args:
40+
- cp /tmp/cni/bin/{sriov,vfioveth,jq,static,dhcp} /host/opt/cni/bin/;
41+
chmod +x /host/opt/cni/bin/vfioveth
42+
volumeMounts:
43+
- name: cni-bin
44+
mountPath: /host/opt/cni/bin
45+
serviceAccountName: sriov-device-plugin
46+
containers:
47+
- name: kube-sriovdp
48+
image: docker.io/nfvpe/sriov-device-plugin:v3.3
49+
imagePullPolicy: IfNotPresent
50+
args:
51+
- --log-dir=sriovdp
52+
- --log-level=10
53+
securityContext:
54+
privileged: true
55+
volumeMounts:
56+
- name: devicesock
57+
mountPath: /var/lib/kubelet/
58+
readOnly: false
59+
- name: log
60+
mountPath: /var/log
61+
- name: config-volume
62+
mountPath: /etc/pcidp
63+
- name: device-info
64+
mountPath: /var/run/k8s.cni.cncf.io/devinfo/dp
65+
volumes:
66+
- name: devicesock
67+
hostPath:
68+
path: /var/lib/kubelet/
69+
- name: log
70+
hostPath:
71+
path: /var/log
72+
- name: device-info
73+
hostPath:
74+
path: /var/run/k8s.cni.cncf.io/devinfo/dp
75+
type: DirectoryOrCreate
76+
- name: config-volume
77+
configMap:
78+
name: sriovdp-config
79+
items:
80+
- key: config.json
81+
path: config.json
82+
- name: cni-bin
83+
hostPath:
84+
path: /opt/cni/bin

0 commit comments

Comments
 (0)