azure-pipelines.yml

# Starter pipeline
# Start with a minimal pipeline that you can customize to build and deploy your code.
# Add steps that build, run tests, deploy, and more:
# https://aka.ms/yaml

trigger:
  branches:
    include:
      - master
      - 202???
  paths:
    exclude:
      - .github

pr:
  branches:
    include:
      - master
      - 202???
  paths:
    exclude:
      - .github

name: $(TeamProject)_$(Build.DefinitionName)_$(SourceBranchName)_$(Date:yyyyMMdd)$(Rev:.r)

resources:
  repositories:
  - repository: sonic-mgmt
    type: github
    name: sonic-net/sonic-mgmt
    ref: master
    endpoint: sonic-net
  - repository: buildimage
    type: github
    name: sonic-net/sonic-buildimage
    endpoint: sonic-net
    ref: master

parameters:
- name: TIMEOUT_IN_MINUTES_PR_TEST
  type: number
  default: 480

variables:
- template: .azure-pipelines/azure-pipelines-repd-build-variables.yml@buildimage
- template: .azure-pipelines/template-variables.yml@buildimage
- name: CACHE_MODE
  value: rcache
- name: ENABLE_FIPS
  value: y
- name: BUILD_BRANCH
  ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
    value: $(System.PullRequest.TargetBranch)
  ${{ else }}:
    value: $(Build.SourceBranchName)

stages:
- stage: BuildVS
  pool: sonicbld-1es
  jobs:
  - template: .azure-pipelines/azure-pipelines-build.yml
    parameters:
      buildOptions: 'USERNAME=admin SONIC_BUILD_JOBS=$(nproc) BUILD_MULTIASIC_KVM=y INCLUDE_DHCP_SERVER=y ${{ variables.VERSION_CONTROL_OPTIONS }}'
      jobGroups:
      - name: vs

- stage: Build
  pool: sonicbld-1es
  dependsOn: []
  jobs:
  - template: .azure-pipelines/azure-pipelines-build.yml
    parameters:
      buildOptions: 'USERNAME=admin SONIC_BUILD_JOBS=$(nproc) ${{ variables.VERSION_CONTROL_OPTIONS }}'
      jobGroups:
      - name: broadcom
        variables:
          swi_image: yes
          docker_syncd_rpc_image: yes
          platform_rpc: brcm
          INCLUDE_RESTAPI: y
      - name: mellanox
        variables:
          dbg_image: yes
          docker_syncd_rpc_image: yes
          platform_rpc: mlnx
      - name: marvell-prestera-arm64
        pool: sonicbld-arm64
        variables:
           PLATFORM_NAME: marvell-prestera
           PLATFORM_ARCH: arm64
      - name: marvell-prestera-armhf
        pool: sonicbld-armhf
        timeoutInMinutes: 1200
        variables:
          PLATFORM_NAME: marvell-prestera
          PLATFORM_ARCH: armhf
          INCLUDE_RESTAPI: y

- stage: Test
  dependsOn: BuildVS
  condition: and(succeeded(), and(ne(stageDependencies.BuildVS.outputs['vs.SetVar.SKIP_VSTEST'], 'YES'), in(dependencies.BuildVS.result, 'Succeeded', 'SucceededWithIssues')))
  variables:
  - group: SONiC-Elastictest
  - name: inventory
    value: veos_vtb
  - name: testbed_file
    value: vtestbed.csv

# For every test job:
# continueOnError: false means it's a required test job and will block merge if it fails
# continueOnError: true means it's an optional test job and will not block merge even though it fails(unless a required test job depends on its result)

  jobs:
  - job:
    pool: sonictest
    displayName: "vstest"
    condition: false # agent pool don't work. disable this job to unblock PRs.
    timeoutInMinutes: 60
    steps:
    - template: .azure-pipelines/cleanup.yml
    - checkout: self
      clean: true
      submodules: recursive
      displayName: 'Checkout code'

    - script: |
        sudo rm -rf ../target
        sudo rm -rf ../*.deb
      displayName: "Cleanup"

    - task: DownloadPipelineArtifact@2
      inputs:
        artifact: sonic-buildimage.vs
      displayName: "Download sonic-buildimage.vs artifact"

    - script: |
        sudo src/sonic-swss-common/.azure-pipelines/build_and_install_module.sh
      displayName: "Install kernel modules"

    - script: |
        sudo apt-get update
        sudo apt-get install -y make libtool m4 autoconf dh-exec debhelper cmake pkg-config \
                         libhiredis-dev libnl-3-dev libnl-genl-3-dev libnl-route-3-dev libnl-nf-3-dev swig \
                         libpython2.7-dev libboost-dev libboost-serialization-dev uuid-dev libzmq5 libzmq3-dev  python3-pip \
                         cmake libgtest-dev libgmock-dev libyang-dev nlohmann-json3-dev
        sudo pip3 install pytest
        cd src/sonic-swss-common
        ./autogen.sh
        dpkg-buildpackage -rfakeroot -us -uc -b -j$(nproc)
        sudo dpkg -i --force-confask,confnew ../libswsscommon_*.deb
        sudo dpkg -i ../python3-swsscommon_*.deb
      displayName: "Compile sonic swss common"

    - script: |
        sudo docker load -i ../target/docker-sonic-vs.gz
        docker tag docker-sonic-vs:latest docker-sonic-vs:$(Build.BuildNumber)
        username=$(id -un)

        trap "docker ps; docker images; ip netns list; \
              docker rmi docker-sonic-vs:$(Build.BuildNumber); \
              ip netns list | grep -E [-]srv[0-9]+ | awk '{print $1}' | xargs -I {} sudo ip netns delete {}; \
              sudo chown -R ${username}.${username} .; \
              sudo chown -R ${username}.${username} $(System.DefaultWorkingDirectory)" EXIT
        pushd platform/vs/tests
        sudo py.test -v --junitxml=tr.xml --imgname=docker-sonic-vs:$(Build.BuildNumber)
      displayName: "Run vs tests"

    - task: PublishTestResults@2
      inputs:
        testResultsFiles: '**/tr.xml'
        testRunTitle: vstest

  - job: get_impacted_area
    displayName: "Get impacted area"
    timeoutInMinutes: 10
    continueOnError: false
    pool: sonic-ubuntu-1c
    steps:
      - checkout: sonic-mgmt
      - template: .azure-pipelines/impacted_area_testing/get-impacted-area.yml@sonic-mgmt
        parameters:
          BUILD_BRANCH: $(BUILD_BRANCH)

  - job: impacted_area_t0_elastictest
    displayName: "impacted-area-kvmtest-t0 by Elastictest"
    dependsOn:
      - get_impacted_area
    condition: contains(dependencies.get_impacted_area.outputs['SetVariableTask.PR_CHECKERS'], 't0_checker')
    variables:
      TEST_SCRIPTS: $[ dependencies.get_impacted_area.outputs['SetVariableTask.TEST_SCRIPTS'] ]
    timeoutInMinutes: ${{ parameters.TIMEOUT_IN_MINUTES_PR_TEST }}
    continueOnError: false
    pool: sonic-ubuntu-1c
    steps:
      - checkout: sonic-mgmt
      - template: .azure-pipelines/impacted_area_testing/calculate-instance-numbers.yml@sonic-mgmt
        parameters:
          TOPOLOGY: t0
          BUILD_BRANCH: $(BUILD_BRANCH)

      - template: .azure-pipelines/run-test-elastictest-template.yml@sonic-mgmt
        parameters:
          TOPOLOGY: t0
          SCRIPTS: $(SCRIPTS)
          MIN_WORKER: $(INSTANCE_NUMBER)
          MAX_WORKER: $(INSTANCE_NUMBER)
          KVM_IMAGE_BRANCH: $(BUILD_BRANCH)
          MGMT_BRANCH: $(BUILD_BRANCH)

  - job: impacted_area_t0_2vlans_elastictest
    displayName: "impacted-area-kvmtest-t0-2vlans by Elastictest"
    dependsOn:
      - get_impacted_area
    condition: contains(dependencies.get_impacted_area.outputs['SetVariableTask.PR_CHECKERS'], 't0-2vlans_checker')
    variables:
      TEST_SCRIPTS: $[ dependencies.get_impacted_area.outputs['SetVariableTask.TEST_SCRIPTS'] ]
    timeoutInMinutes: ${{ parameters.TIMEOUT_IN_MINUTES_PR_TEST }}
    continueOnError: false
    pool: sonic-ubuntu-1c
    steps:
      - checkout: sonic-mgmt
      - template: .azure-pipelines/impacted_area_testing/calculate-instance-numbers.yml@sonic-mgmt
        parameters:
          TOPOLOGY: t0-2vlans
          BUILD_BRANCH: $(BUILD_BRANCH)

      - template: .azure-pipelines/run-test-elastictest-template.yml@sonic-mgmt
        parameters:
          TOPOLOGY: t0
          SCRIPTS: $(SCRIPTS)
          MIN_WORKER: $(INSTANCE_NUMBER)
          MAX_WORKER: $(INSTANCE_NUMBER)
          DEPLOY_MG_EXTRA_PARAMS: "-e vlan_config=two_vlan_a"
          KVM_IMAGE_BRANCH: $(BUILD_BRANCH)
          MGMT_BRANCH: $(BUILD_BRANCH)

  - job: impacted_area_t1_lag_elastictest
    displayName: "impacted-area-kvmtest-t1-lag by Elastictest"
    dependsOn:
      - get_impacted_area
    condition: contains(dependencies.get_impacted_area.outputs['SetVariableTask.PR_CHECKERS'], 't1_checker')
    variables:
      TEST_SCRIPTS: $[ dependencies.get_impacted_area.outputs['SetVariableTask.TEST_SCRIPTS'] ]
    timeoutInMinutes: ${{ parameters.TIMEOUT_IN_MINUTES_PR_TEST }}
    continueOnError: false
    pool: sonic-ubuntu-1c
    steps:
      - checkout: sonic-mgmt
      - template: .azure-pipelines/impacted_area_testing/calculate-instance-numbers.yml@sonic-mgmt
        parameters:
          TOPOLOGY: t1
          BUILD_BRANCH: $(BUILD_BRANCH)
          # 40 mins for preparing testbed, 10 mins for pre-test and post-test
          PREPARE_TIME: 50

      - template: .azure-pipelines/run-test-elastictest-template.yml@sonic-mgmt
        parameters:
          TOPOLOGY: t1-lag
          SCRIPTS: $(SCRIPTS)
          MIN_WORKER: $(INSTANCE_NUMBER)
          MAX_WORKER: $(INSTANCE_NUMBER)
          KVM_IMAGE_BRANCH: $(BUILD_BRANCH)
          MGMT_BRANCH: $(BUILD_BRANCH)

  - job: impacted_area_dualtor_elastictest
    displayName: "impacted-area-kvmtest-dualtor by Elastictest"
    dependsOn:
      - get_impacted_area
    condition: contains(dependencies.get_impacted_area.outputs['SetVariableTask.PR_CHECKERS'], 'dualtor_checker')
    variables:
      TEST_SCRIPTS: $[ dependencies.get_impacted_area.outputs['SetVariableTask.TEST_SCRIPTS'] ]
    timeoutInMinutes: ${{ parameters.TIMEOUT_IN_MINUTES_PR_TEST }}
    continueOnError: false
    pool: sonic-ubuntu-1c
    steps:
      - checkout: sonic-mgmt
      - template: .azure-pipelines/impacted_area_testing/calculate-instance-numbers.yml@sonic-mgmt
        parameters:
          TOPOLOGY: dualtor
          BUILD_BRANCH: $(BUILD_BRANCH)
          # 30 mins for preparing testbed, 30 mins for pre-test and 20 mins for post-test
          PREPARE_TIME: 80

      - template: .azure-pipelines/run-test-elastictest-template.yml@sonic-mgmt
        parameters:
          TOPOLOGY: dualtor
          SCRIPTS: $(SCRIPTS)
          MIN_WORKER: $(INSTANCE_NUMBER)
          MAX_WORKER: $(INSTANCE_NUMBER)
          COMMON_EXTRA_PARAMS: "--disable_loganalyzer "
          KVM_IMAGE_BRANCH: $(BUILD_BRANCH)
          MGMT_BRANCH: $(BUILD_BRANCH)

  - job: impacted_area_multi_asic_elastictest
    displayName: "impacted-area-kvmtest-multi-asic-t1 by Elastictest"
    dependsOn:
      - get_impacted_area
    condition: contains(dependencies.get_impacted_area.outputs['SetVariableTask.PR_CHECKERS'], 't1-multi-asic_checker')
    variables:
      TEST_SCRIPTS: $[ dependencies.get_impacted_area.outputs['SetVariableTask.TEST_SCRIPTS'] ]
    timeoutInMinutes: ${{ parameters.TIMEOUT_IN_MINUTES_PR_TEST }}
    continueOnError: true
    pool: sonic-ubuntu-1c
    steps:
      - checkout: sonic-mgmt
      - template: .azure-pipelines/impacted_area_testing/calculate-instance-numbers.yml@sonic-mgmt
        parameters:
          TOPOLOGY: t1-multi-asic
          BUILD_BRANCH: $(BUILD_BRANCH)

      - template: .azure-pipelines/run-test-elastictest-template.yml@sonic-mgmt
        parameters:
          TOPOLOGY: t1-8-lag
          SCRIPTS: $(SCRIPTS)
          MIN_WORKER: $(INSTANCE_NUMBER)
          MAX_WORKER: $(INSTANCE_NUMBER)
          NUM_ASIC: 4
          KVM_IMAGE_BRANCH: $(BUILD_BRANCH)
          MGMT_BRANCH: $(BUILD_BRANCH)

  - job: impacted_area_t0_sonic_elastictest
    displayName: "impacted-area-kvmtest-t0-sonic by Elastictest"
    dependsOn:
      - get_impacted_area
    condition: contains(dependencies.get_impacted_area.outputs['SetVariableTask.PR_CHECKERS'], 't0-sonic_checker')
    variables:
      TEST_SCRIPTS: $[ dependencies.get_impacted_area.outputs['SetVariableTask.TEST_SCRIPTS'] ]
    timeoutInMinutes: ${{ parameters.TIMEOUT_IN_MINUTES_PR_TEST }}
    continueOnError: false
    pool: sonic-ubuntu-1c
    steps:
      - checkout: sonic-mgmt
      - template: .azure-pipelines/impacted_area_testing/calculate-instance-numbers.yml@sonic-mgmt
        parameters:
          TOPOLOGY: t0-sonic
          BUILD_BRANCH: $(BUILD_BRANCH)
          PREPARE_TIME: 40

      - template: .azure-pipelines/run-test-elastictest-template.yml@sonic-mgmt
        parameters:
          TOPOLOGY: t0-64-32
          SCRIPTS: $(SCRIPTS)
          MIN_WORKER: $(INSTANCE_NUMBER)
          MAX_WORKER: $(INSTANCE_NUMBER)
          KVM_IMAGE_BRANCH: $(BUILD_BRANCH)
          COMMON_EXTRA_PARAMS: "--neighbor_type=sonic "
          VM_TYPE: vsonic
          MGMT_BRANCH: $(BUILD_BRANCH)
          SPECIFIC_PARAM: '[
            {"name": "bgp/test_bgp_fact.py", "param": "--neighbor_type=sonic --enable_macsec --macsec_profile=128_SCI,256_XPN_SCI"},
            {"name": "macsec", "param": "--neighbor_type=sonic --enable_macsec --macsec_profile=128_SCI,256_XPN_SCI"}
            ]'

  - job: impacted_area_dpu_elastictest
    displayName: "impacted-area-kvmtest-dpu by Elastictest"
    dependsOn:
      - get_impacted_area
    condition: contains(dependencies.get_impacted_area.outputs['SetVariableTask.PR_CHECKERS'], 'dpu_checker')
    variables:
      TEST_SCRIPTS: $[ dependencies.get_impacted_area.outputs['SetVariableTask.TEST_SCRIPTS'] ]
    timeoutInMinutes: ${{ parameters.TIMEOUT_IN_MINUTES_PR_TEST }}
    continueOnError: false
    pool: sonic-ubuntu-1c
    steps:
      - checkout: sonic-mgmt
      - template: .azure-pipelines/impacted_area_testing/calculate-instance-numbers.yml@sonic-mgmt
        parameters:
          TOPOLOGY: dpu
          BUILD_BRANCH: $(BUILD_BRANCH)

      - template: .azure-pipelines/run-test-elastictest-template.yml@sonic-mgmt
        parameters:
          TOPOLOGY: dpu
          SCRIPTS: $(SCRIPTS)
          MIN_WORKER: $(INSTANCE_NUMBER)
          MAX_WORKER: $(INSTANCE_NUMBER)
          KVM_IMAGE_BRANCH: $(BUILD_BRANCH)
          MGMT_BRANCH: $(BUILD_BRANCH)
          SPECIFIC_PARAM: '[
            {"name": "dash/test_dash_vnet.py", "param": "--skip_dataplane_checking"}
            ]'