Merge branch 'master' of https://github.com/Azure/sonic-buildimage into internal

Author: sonic-build

dockers/docker-sonic-mgmt/Dockerfile.j2

@@ -1,5 +1,5 @@
 {% set prefix = DEFAULT_CONTAINER_REGISTRY %}
-FROM {{ prefix }}ubuntu:18.04
+FROM {{ prefix }}ubuntu:20.04
 
 ENV DEBIAN_FRONTEND=noninteractive
 
@@ -21,8 +21,6 @@ RUN apt-get update && apt-get install -y build-essential \
                                          psmisc \
                                          python \
                                          python-dev \
-                                         python-scapy \
-                                         python-pip \
                                          python3-pip \
                                          python3-venv \
                                          rsyslog \
@@ -31,10 +29,20 @@ RUN apt-get update && apt-get install -y build-essential \
                                          sudo \
                                          tcpdump \
                                          telnet \
-                                         vim
+                                         vim \
+                                         python-is-python2 \
+                                         software-properties-common
+
+RUN add-apt-repository -y universe
+RUN curl https://bootstrap.pypa.io/pip/2.7/get-pip.py --output get-pip.py \
+    && python2 get-pip.py
+
+RUN curl -L http://archive.ubuntu.com/ubuntu/pool/universe/s/scapy/python-scapy_2.3.3-3_all.deb \
+    --output python-scapy_2.3.3-3_all.deb \
+    && dpkg -i python-scapy_2.3.3-3_all.deb
 
 RUN pip install setuptools==44.1.1
-RUN pip install cffi==1.10.0 \
+RUN pip install cffi==1.12.0 \
                 contextlib2==0.6.0.post1 \
                 cryptography==3.3.2 \
                 "future>=0.16.0" \
@@ -96,7 +104,7 @@ RUN pip install cffi==1.10.0 \
     && rm -f 1.0.0.tar.gz  \
     && pip install nnpy    \
     && pip install dpkt    \
-    && pip install scapy==2.4.5 --upgrade
+    && pip install scapy==2.4.5 --upgrade --ignore-installed
 
 # Install docker-ce-cli
 RUN apt-get update                  \
@@ -114,10 +122,6 @@ RUN apt-get update                  \
 # Install Azure CLI
 RUN curl -sL https://aka.ms/InstallAzureCLIDeb | bash
 
-# Install Microsoft Azure Kusto Library for Python
-RUN pip install azure-kusto-data==0.0.13 \
-                azure-kusto-ingest==0.0.13
-
 RUN pip install wheel==0.33.6
 
 ## Copy and install sonic-mgmt docker dependencies
@@ -193,8 +197,7 @@ ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 
 ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 PYTHONIOENCODING=UTF-8
 
-RUN python3 -m pip install --upgrade --ignore-installed pip setuptools==58.4.0
-
+RUN python3 -m pip install --upgrade --ignore-installed pip setuptools==58.4.0 wheel==0.33.6
 RUN python3 -m pip install setuptools-rust \
                             aiohttp \
                             defusedxml \
@@ -210,6 +213,7 @@ RUN python3 -m pip install setuptools-rust \
                             ixnetwork-restpy==1.0.64 \
                             ixnetwork-open-traffic-generator==0.0.79 \
                             snappi[ixnetwork,convergence]==0.7.44 \
+                            markupsafe==2.0.1 \
                             jinja2==2.7.2 \
                             jsonpatch \
                             lxml \
@@ -237,7 +241,6 @@ RUN python3 -m pip install setuptools-rust \
                             tabulate \
                             textfsm==1.1.2 \
                             virtualenv \
-                            wheel==0.33.6 \
                             pysubnettree \
                             nnpy \
                             dpkt \
@@ -252,5 +255,5 @@ RUN python3 -m pip install setuptools-rust \
                             celery[redis]==4.4.7 \
                             msrest==0.6.21
 
-# Deactivating a virtualenv.
+# Deactivating a virtualenv
 ENV PATH="$BACKUP_OF_PATH"

files/build_templates/docker_image_ctl.j2

@@ -87,7 +87,7 @@ function preStartAction()
 {%- if docker_container_name == "database" %}
     WARM_DIR=/host/warmboot
     if [ "$DATABASE_TYPE" != "chassisdb" ]; then
-        if [[ ("$BOOT_TYPE" == "warm" || "$BOOT_TYPE" == "fastfast")  && -f $WARM_DIR/dump.rdb ]]; then
+        if [[ ("$BOOT_TYPE" == "warm" || "$BOOT_TYPE" == "fastfast" || "$BOOT_TYPE" == "fast")  && -f $WARM_DIR/dump.rdb ]]; then
             # Load redis content from /host/warmboot/dump.rdb
             docker cp $WARM_DIR/dump.rdb database$DEV:/var/lib/redis/dump.rdb
         else
@@ -218,7 +218,7 @@ function postStartAction()
                  ($(docker exec -i database$DEV sonic-db-cli PING | grep -c PONG) -gt 0) ]]; do
           sleep 1;
         done
-        if [[ ("$BOOT_TYPE" == "warm" || "$BOOT_TYPE" == "fastfast") && -f $WARM_DIR/dump.rdb ]]; then
+        if [[ ("$BOOT_TYPE" == "warm" || "$BOOT_TYPE" == "fastfast" || "$BOOT_TYPE" == "fast") && -f $WARM_DIR/dump.rdb ]]; then
             # retain the dump file from last boot for debugging purposes
             mv $WARM_DIR/dump.rdb $WARM_DIR/dump.rdb.old
         else

scripts/build_kvm_image.sh

@@ -49,6 +49,19 @@ prepare_installer_disk()
     umount $tmpdir
 }
 
+wait_kvm_ready()
+{
+    local count=30
+    local waiting_in_seconds=2.0
+    for ((i=1; i<=$count; i++)); do
+        sleep $waiting_in_seconds
+        echo "$(date) [$i/$count] waiting for the port $KVM_PORT ready"
+        if netstat -l | grep -q ":$KVM_PORT"; then
+          break
+        fi
+    done
+}
+
 apt-get install -y net-tools
 create_disk
 prepare_installer_disk
@@ -86,7 +99,7 @@ echo "Installing SONiC"
 
 kvm_pid=$!
 
-sleep 2.0
+wait_kvm_ready
 
 [ -d "/proc/$kvm_pid" ] || {
         echo "ERROR: kvm died."
@@ -114,7 +127,7 @@ echo "Booting up SONiC"
 
 kvm_pid=$!
 
-sleep 2.0
+wait_kvm_ready
 
 [ -d "/proc/$kvm_pid" ] || {
         echo "ERROR: kvm died."

src/ifupdown2/patch/0003-Fix-the-return-value-of-utils._execute_subprocess-me.patch

@@ -0,0 +1,21 @@
+Fix the return value of utils._execute_subprocess method for empty strings
+---
+ ifupdown2/ifupdown/utils.py | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/ifupdown2/ifupdown/utils.py b/ifupdown2/ifupdown/utils.py
+index d638fe9..0c5d8ce 100644
+--- a/ifupdown2/ifupdown/utils.py
++++ b/ifupdown2/ifupdown/utils.py
+@@ -380,7 +380,7 @@ class utils():
+         finally:
+             utils.disable_subprocess_signal_forwarding(signal.SIGINT)
+ 
+-        cmd_output_string = cmd_output.decode() if cmd_output else cmd_output
++        cmd_output_string = cmd_output.decode() if cmd_output is not None else cmd_output
+ 
+         if cmd_returncode != 0:
+             raise Exception(cls._format_error(cmd,
+-- 
+2.14.1
+

src/ifupdown2/patch/series

@@ -1,2 +1,3 @@
 0001-fix-broadcast-addr-encoding.patch
 0002-disable-checks-when-using-no-wait.patch
+0003-Fix-the-return-value-of-utils._execute_subprocess-me.patch