Add Python-Based DASH Pipeline as a drop-in replacement for BMv2 model

Signed-off-by: Farhan Tariq <[email protected]>

Author: f4ni

.github/workflows/dash-bmv2-ci.yml

@@ -12,6 +12,7 @@ on:
       - 'test/**.yml'
       - 'dash-pipeline/**'
       - '!dash-pipeline/dockerfiles/Dockerfile.*'
+      - '!dash-pipeline/py_model*'
       - 'dash-pipeline/dockerfiles/*.env'
       - '!dash-pipeline/.dockerignore'
       - '!dash-pipeline/**.md'

.github/workflows/dash-pymodel-ci.yml

@@ -0,0 +1,103 @@
+name: DASH-PYMODEL-CI
+
+on:
+  push:
+    branches: [ "**" ]
+    paths:
+      - '.gitmodules'
+      - '.github/workflows/dash-pymodel-ci.yml'
+      - 'test/**.py'
+      - 'test/**requirements.txt'
+      - 'test/**.sh'
+      - 'test/**.yml'
+      - 'dash-pipeline/**'
+      - '!dash-pipeline/dockerfiles/Dockerfile.*'
+      - '!dash-pipeline/bmv2*'
+      - 'dash-pipeline/dockerfiles/*.env'
+      - '!dash-pipeline/.dockerignore'
+      - '!dash-pipeline/**.md'
+      - '!dash-pipeline/**.svg'
+      - '!dash-pipeline/**.png'
+      - '!dash-pipeline/**.txt'
+  pull_request:
+    branches: [ "**" ]
+    paths:
+      - '.gitmodules'
+      - '.github/workflows/dash-pymodel-ci.yml'
+      - 'test/**.py'
+      - 'test/**requirements.txt'
+      - 'test/**.sh'
+      - 'test/**.yml'
+      - 'dash-pipeline/**'
+      - '!dash-pipeline/dockerfiles/Dockerfile.*'
+      - '!dash-pipeline/bmv2*'
+      - 'dash-pipeline/dockerfiles/*.env'
+      - '!dash-pipeline/.dockerignore'
+      - '!dash-pipeline/**.md'
+      - '!dash-pipeline/**.svg'
+      - '!dash-pipeline/**.png'
+      - '!dash-pipeline/**.txt'
+  workflow_dispatch:
+
+jobs:
+  build:
+    name: Build and Test Python DASH Pipeline
+    runs-on: ubuntu-22.04
+    env:
+      docker_fg_flags: --privileged
+      docker_fg_root_flags: --privileged -u root
+      docker_bg_root_flags: -d --privileged -u root
+      docker_bg_flags: -d --privileged
+    defaults:
+      run:
+        working-directory: ./dash-pipeline
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.12'
+    - name: Upgrade pip tooling
+      run: python -m pip install --upgrade pip setuptools wheel
+    - name: Install pymodel Python dependencies
+      run: python -m pip install -r py_model/requirements.txt
+    - name: Build python based pymodel artifacts
+      run:  make py-artifacts
+    - name: Update SAI submodule
+      run:  git submodule update --init
+    - name: Pull/Build docker saithrift-bldr image
+      run:  make docker-saithrift-bldr
+    - name: Pull/Build docker pymodel-bldr image
+      run:  make docker-pymodel-bldr
+    - name: Generate SAI API
+      run:  DOCKER_FLAGS=$docker_fg_flags make sai TARGET=pymodel
+    - name: Run pymodel
+      run:  DOCKER_FLAGS=$docker_bg_root_flags make run-pymodel HAVE_DPAPP=y
+    - name: Pull/Build docker dpapp image
+      run:  make docker-dash-dpapp
+    - name: Build dpapp
+      run:  DOCKER_FLAGS=$docker_fg_flags make dpapp TARGET=pymodel
+    - name: Check if SAI spec is updated
+      run:  DOCKER_FLAGS=$docker_fg_flags make check-sai-spec
+    - name: Prepare network
+      run:  DOCKER_FLAGS=$docker_fg_flags make network HAVE_DPAPP=y
+    - name: Run dpapp
+      run:  DOCKER_FLAGS=$docker_bg_flags make run-dpapp TARGET=pymodel
+    - name: Generate saithrift-server
+      run:  DOCKER_FLAGS=$docker_fg_flags make saithrift-server
+    - name: Generate saithrift-client local docker
+      run:  DOCKER_FLAGS=$docker_fg_flags make docker-saithrift-client
+    - name: Run saithrift server
+      run:  DOCKER_FLAGS=$docker_bg_flags make run-saithrift-server TARGET=pymodel
+    - name: Run PTF Tests
+      run:  DOCKER_FLAGS=$docker_fg_root_flags make run-saithrift-ptftests
+
+    - uses: azure/docker-login@v1
+      if: ${{ github.event_name != 'pull_request' && github.repository == 'sonic-net/DASH' }}
+      with:
+        login-server: sonicdash.azurecr.io
+        username: ${{ secrets.DASH_ACR_USERNAME }}
+        password: ${{ secrets.DASH_ACR_PASSWORD }}
+    - name: Publish DASH pymodel builder base docker image
+      run:  make docker-publish-pymodel-bldr
+      if: ${{ github.event_name != 'pull_request' && github.repository == 'sonic-net/DASH' }}

.gitignore

@@ -10,6 +10,9 @@ __pycache__/
 .pytest_cache/
 dash-pipeline/bmv2/dash_pipeline.bmv2/
 dash-pipeline/dpdk-pna/dash_pipeline.dpdk
+dash-pipeline/py_model/dash_pipeline.py_model/
+dash-pipeline/py_model/p4_helper/v1/__pycache__/
+dash-pipeline/py_model/p4_helper/config/__pycache__/
 dash-pipeline/SAI/lib/
 dash-pipeline/SAI/rpc/
 dash-pipeline/dpapp/build

.wordlist.txt

@@ -96,6 +96,7 @@ CloudStorm
 cloudstorm
 CNIP
 codebase
+codegen
 collaterals
 compat
 Compat
@@ -105,7 +106,9 @@ confgen
 config
 configs
 configurated
-Conntrack
+conntrack
+ConntrackIn
+ConntrackOut
 Containerlab
 CP
 CPUs
@@ -236,6 +239,7 @@ ENIs
 ENI's
 enqueue
 enqueues
+entrypoint
 entrypoints
 enum
 EPUs
@@ -504,6 +508,8 @@ PTF
 ptf
 ptftests
 py
+py_model
+pymodel
 PyPi
 pytest
 PyTest
@@ -545,6 +551,7 @@ RJ
 Roadmap
 roadmap
 routable
+rpc
 RPC
 RPCs
 RPF
@@ -665,6 +672,7 @@ TEP
 testability
 testbed
 testbeds
+textproto
 TGen
 Tgen
 Tgens

README.md

@@ -1,4 +1,5 @@
 [![DASH-bmv2-CI](https://github.com/sonic-net/DASH/workflows/DASH-BMV2-CI/badge.svg?branch=main)](https://github.com/sonic-net/DASH/actions/workflows/dash-bmv2-ci.yml)
+[![DASH-pymodel-CI](https://github.com/sonic-net/DASH/workflows/DASH-pymodel-CI/badge.svg?branch=main)](https://github.com/sonic-net/DASH/actions/workflows/dash-pymodel-ci.yml)
 [![Spellcheck](https://github.com/sonic-net/DASH/actions/workflows/dash-md-spellcheck.yml/badge.svg)](https://github.com/sonic-net/DASH/actions/workflows/dash-md-spellcheck.yml) 
 
 # SONiC-DASH - Disaggregated API for SONiC Hosts - extending functionality to stateful workloads!