[ci] Test and build packages using Azure Pipelines (#164)

Configure Azure Pipelines to run unit tests, build Python wheels and publish the test results, test coverage and resulting wheels.

Also fix existing unit tests.

Author: jleveque

.artifactignore

@@ -0,0 +1,2 @@
+**/*
+!dist/*.whl

azure-pipelines.yml

@@ -4,16 +4,101 @@
 # https://aka.ms/yaml
 
 trigger:
-- master
+  branches:
+    include:
+      - '*'
 
 pool:
-  vmImage: ubuntu-20.04
+  vmImage: 'ubuntu-20.04'
+
+container:
+  image: sonicdev-microsoft.azurecr.io:443/sonic-slave-buster:latest
 
 steps:
-- script: echo Hello, world!
-  displayName: 'Run a one-line script'
+- task: DownloadPipelineArtifact@2
+  inputs:
+    source: specific
+    project: build
+    pipeline: 1
+    artifact: sonic-buildimage.kvm
+    runVersion: 'latestFromBranch'
+    runBranch: 'refs/heads/master'
+  displayName: "Download artifacts from latest sonic-buildimage build"
+
+- script: |
+    set -xe
+    sudo dpkg -i libnl-3-200_*.deb
+    sudo dpkg -i libnl-genl-3-200_*.deb
+    sudo dpkg -i libnl-route-3-200_*.deb
+    sudo dpkg -i libnl-nf-3-200_*.deb
+    sudo dpkg -i libhiredis0.14_*.deb
+    sudo dpkg -i libswsscommon_1.0.0_amd64.deb
+    sudo dpkg -i python-swsscommon_1.0.0_amd64.deb
+    sudo dpkg -i python3-swsscommon_1.0.0_amd64.deb
+  workingDirectory: $(Pipeline.Workspace)/target/debs/buster/
+  displayName: 'Install Debian dependencies'
+
+- script: |
+    set -xe
+    pip2 install swsssdk-2.0.1-py2-none-any.whl
+    pip2 install sonic_py_common-1.0-py2-none-any.whl
+    pip2 install sonic_config_engine-1.0-py2-none-any.whl
+    pip3 install swsssdk-2.0.1-py3-none-any.whl
+    pip3 install sonic_py_common-1.0-py3-none-any.whl
+    pip3 install sonic_config_engine-1.0-py3-none-any.whl
+  workingDirectory: $(Pipeline.Workspace)/target/python-wheels/
+  displayName: 'Install Python dependencies'
+
+# Python 2
+- script: |
+    python2 setup.py test
+  displayName: 'Test Python 2'
+
+- task: PublishTestResults@2
+  inputs:
+    testResultsFiles: '$(System.DefaultWorkingDirectory)/test-results.xml'
+    testRunTitle: Python 2
+    failTaskOnFailedTests: true
+  condition: succeededOrFailed()
+  displayName: 'Publish Python 2 test results'
+
+- task: PublishCodeCoverageResults@1
+  inputs:
+    codeCoverageTool: Cobertura
+    summaryFileLocation: '$(System.DefaultWorkingDirectory)/coverage.xml'
+    reportDirectory: '$(System.DefaultWorkingDirectory)/htmlcov/'
+  displayName: 'Publish Python 2 test coverage'
+
+- script: |
+    set -e
+    python2 setup.py bdist_wheel
+  displayName: 'Build Python 2 wheel'
+
+# Python 3
+- script: |
+    python3 setup.py test
+  displayName: 'Test Python 3'
+
+- task: PublishTestResults@2
+  inputs:
+    testResultsFiles: '$(System.DefaultWorkingDirectory)/test-results.xml'
+    testRunTitle: Python 3
+    failTaskOnFailedTests: true
+  condition: succeededOrFailed()
+  displayName: 'Publish Python 3 test results'
+
+- task: PublishCodeCoverageResults@1
+  inputs:
+    codeCoverageTool: Cobertura
+    summaryFileLocation: '$(System.DefaultWorkingDirectory)/coverage.xml'
+    reportDirectory: '$(System.DefaultWorkingDirectory)/htmlcov/'
+  displayName: 'Publish Python 3 test coverage'
 
 - script: |
-    echo Add other tasks to build, test, and deploy your project.
-    echo See https://aka.ms/yaml
-  displayName: 'Run a multi-line script'
+    set -e
+    python3 setup.py bdist_wheel
+  displayName: 'Build Python 3 wheel'
+
+- publish: '$(System.DefaultWorkingDirectory)/dist/'
+  artifact: wheels
+  displayName: "Publish Python wheels"

pytest.ini

@@ -1,2 +1,2 @@
 [pytest]
-addopts = --cov=sonic_platform_base --cov-report html
+addopts = --cov=sonic_platform_base --cov-report html --cov-report term --cov-report xml --junitxml=test-results.xml -v

setup.py

@@ -25,13 +25,14 @@
         'sonic_thermal',
         'sonic_y_cable',
     ],
-    # NOTE: Install also depends on sonic-config-engine for portconfig.py, but we are not yet
-    # building a Python 3 version of sonic-config-engine. Also, this dependency should be
-    # eliminated by moving portconfig.py functionality into sonic-py-common
+    # NOTE: Install also depends on sonic-config-engine for portconfig.py
+    # This dependency should be eliminated by moving portconfig.py
+    # functionality into sonic-py-common
     install_requires=[
         'natsort==6.2.1', # 6.2.1 is the last version which supports Python 2
         'PyYAML',
         'redis',
+        'sonic-config-engine',
         'sonic-py-common'
     ],
     setup_requires = [

tests/t0-sample-port-config.ini renamed to tests/port_config.ini

@@ -2,53 +2,44 @@
 import sys
 
 import pytest
-try:
-    import sonic_platform_base.sonic_sfp.sfputilhelper
-except Exception as e:
-    print("Failed to load sonic_platform_base.sonic_sfp.sfputilhelper  due to {}".format(repr(e)))
+
+from sonic_platform_base.sonic_sfp import sfputilhelper
 
 
 @pytest.fixture(scope="class")
 def setup_class(request):
     # Configure the setup
     test_dir = os.path.dirname(os.path.realpath(__file__))
-    request.cls.port_config = os.path.join(
-        test_dir, 't0-sample-port-config.ini')
-
-    request.cls.port_config = sonic_platform_base.sonic_sfp.sfputilhelper.SfpUtilHelper()
+    request.cls.port_config_file = os.path.join(test_dir, 'port_config.ini')
 
 
 @pytest.mark.usefixtures("setup_class")
 class TestSfpUtilHelper(object):
 
-    platform_sfputil = None
-    port_config = None
+    port_config_file = None
 
     def test_read_port_mappings(self):
-
-        try:
-            platform_sfputil.read_porttab_mappings(self.port_config, 0)
-        except Exception as e:
-            print("Failed to read port tab mappings to {}".format(repr(e)))
-
-        PORT_LIST = ["Ethernet0",
-                     "Ethernet4",
-                     "Ethernet8",
-                     "Ethernet12",
-                     "Ethernet16",
-                     "Ethernet20",
-                     "Ethernet24",
-                     "Ethernet28",
-                     "Ethernet32",
-                     "Ethernet36",
-                     "Ethernet40",
-                     "Ethernet44",
-                     "Ethernet48"]
-
-        if self.platform_sfputil is not None:
-            logical_port_list = self.platform_sfputil.logical
-            assert len(logical_port_name) == len(self.port_list)
-            for logical_port_name in logical_port_list:
-                assert logical_port_name in PORT_LIST
-        else:
-            print("platform_sfputil is None, cannot read Ports")
+        PORT_LIST = [
+            "Ethernet0",
+            "Ethernet4",
+            "Ethernet8",
+            "Ethernet12",
+            "Ethernet16",
+            "Ethernet20",
+            "Ethernet24",
+            "Ethernet28",
+            "Ethernet32",
+            "Ethernet36",
+            "Ethernet40",
+            "Ethernet44",
+            "Ethernet48"
+        ]
+
+        sfputil_helper = sfputilhelper.SfpUtilHelper()
+        sfputil_helper.read_porttab_mappings(self.port_config_file, 0)
+
+        logical_port_list = sfputil_helper.logical
+        assert len(logical_port_list) == len(PORT_LIST)
+
+        for logical_port_name in logical_port_list:
+            assert logical_port_name in PORT_LIST