diff --git a/resources/docker_files/README.md b/resources/docker_files/README.md index dc67d259a..317fb1604 100644 --- a/resources/docker_files/README.md +++ b/resources/docker_files/README.md @@ -36,6 +36,8 @@ sudo docker run --network=host --rm -i -t ubuntu-18.04 ``` Above, ```-i``` is for interactive mode and ```-t``` is for emulating a tty. ```--rm``` tells docker to cleanup the container after exit. (See note above regarding `--network=host`.) All images launch ```bash``` on startup. Hence, user is on a ```bash``` shell when image is started in the interactive mode. +Note that the additional parameter `--security-opt seccomp=unconfined` which is disabling the ASLR for the zeroize test, is now set by default. + Use ```run.sh``` for enabling ```git``` and mounting a host workspace inside docker. Example: ```sh $ ./run.sh /home/mazimkhan/github/mazimkhan ubuntu-18.04 diff --git a/resources/docker_files/run.sh b/resources/docker_files/run.sh index 996969246..2db4f99ef 100755 --- a/resources/docker_files/run.sh +++ b/resources/docker_files/run.sh @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (c) 2017-2021, ARM Limited, All Rights Reserved +# Copyright (c) 2017-2022, ARM Limited, All Rights Reserved # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -56,5 +56,5 @@ echo " Mounting $SSH_CFG_PATH --> /home/user/.ssh" echo " Mounting $MOUNT_DIR --> /var/lib/ws" echo "****************************************************" -sudo docker run --network=host --rm -i -t -u $USR_ID:$USR_GRP -w /var/lib/ws -v $MOUNT_DIR:/var/lib/ws -v $SSH_CFG_PATH:/home/user/.ssh --cap-add SYS_PTRACE ${IMAGE} +sudo docker run --network=host --rm -i -t -u $USR_ID:$USR_GRP -w /var/lib/ws -v $MOUNT_DIR:/var/lib/ws -v $SSH_CFG_PATH:/home/user/.ssh --cap-add SYS_PTRACE --security-opt seccomp=unconfined ${IMAGE} diff --git a/resources/docker_files/ubuntu-16.04/Dockerfile b/resources/docker_files/ubuntu-16.04/Dockerfile index 043e6ccf7..313335263 100644 --- a/resources/docker_files/ubuntu-16.04/Dockerfile +++ b/resources/docker_files/ubuntu-16.04/Dockerfile @@ -1,6 +1,6 @@ # ubuntu-16.04/Dockerfile # -# Copyright (c) 2018-2021, ARM Limited, All Rights Reserved +# Copyright (c) 2018-2022, ARM Limited, All Rights Reserved # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -117,7 +117,7 @@ RUN wget -q https://developer.arm.com/-/media/Files/downloads/compiler/DS500-PA- ENV ARMC5_BIN_DIR=/usr/local/ARM_Compiler_5.06u3/bin/ ENV PATH=$PATH:/usr/local/ARM_Compiler_5.06u3/bin -ENV ARMLMD_LICENSE_FILE=8225@armcc.licenses.mbedcloudtesting.com +ENV ARMLMD_LICENSE_FILE=7010@10.6.26.52:7010@10.6.26.53:7010@10.6.26.54:7010@10.6.26.56 # Install ARM Compiler 6.6 RUN mkdir temp && cd temp && \ diff --git a/resources/docker_files/ubuntu-18.04/Dockerfile b/resources/docker_files/ubuntu-18.04/Dockerfile index c060a690a..b44ffd70b 100644 --- a/resources/docker_files/ubuntu-18.04/Dockerfile +++ b/resources/docker_files/ubuntu-18.04/Dockerfile @@ -1,6 +1,6 @@ # ubuntu-18.04/Dockerfile # -# Copyright (c) 2018-2021, ARM Limited, All Rights Reserved +# Copyright (c) 2018-2022, ARM Limited, All Rights Reserved # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -17,83 +17,102 @@ # # This file is part of Mbed TLS (https://www.trustedfirmware.org/projects/mbed-tls/) - - -################################################################ -#### Documentation -################################################################ - # Purpose # ------- # -# This docker file is for creating a ubuntu-18.04 platform container. It -# contains setup and installation of tools for executing same set of Mbed TLS -# tests as there are in the CI. This conatiner can be used for reproducing and -# testing failures found in the CI. - +# This docker file is for creating the ubuntu-18.04 image that is used in the +# CI. It can also be used for reproducing and testing CI failures. -#Start with basic Ubuntu 18.04 image FROM ubuntu:18.04 -ENV DEBIAN_FRONTEND=noninteractive - -# Install necessary apt tools -RUN apt-get update > /dev/null && \ - apt-get install -y apt-transport-https \ - apt-utils \ - software-properties-common > /dev/null && \ - apt-get clean && rm -rf /var/lib/apt/lists/ - -# set the working directory to /opt/slave -WORKDIR /opt/slave - -# Pre-approve Oracle Java license -RUN echo debconf shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \ - echo debconf shared/accepted-oracle-license-v1-1 seen true | debconf-set-selections - -# Install source control tools -RUN apt-get update > /dev/null && \ - apt-get install -y git > /dev/null && \ - apt-get clean && rm -rf /var/lib/apt/lists/ - -# Install Python 2.7 -RUN apt-get update > /dev/null && \ - apt-get install -y python2.7 \ - libffi-dev \ - python-dev \ - python-pip \ - python-setuptools \ - python-distutils-extra > /dev/null && \ - apt-get clean && rm -rf /var/lib/apt/lists/ - -# Install Python 3 pip -RUN apt-get update > /dev/null && \ - apt-get install -y python3-pip > /dev/null && \ - apt-get clean && rm -rf /var/lib/apt/lists/ - -# Install build tools -RUN apt-get update > /dev/null && \ - apt-get install -y cmake \ - make \ - valgrind \ - doxygen \ - graphviz \ - lcov \ - abi-dumper \ - gcc-mingw-w64-i686 \ - clang \ - wget \ - lsof > /dev/null && \ - apt-get clean && rm -rf /var/lib/apt/lists/ +ARG DEBIAN_FRONTEND=noninteractive +WORKDIR /opt/src + +# Support for i386: +# - for 32-bit builds+tests of Mbed TLS +# - required to install Arm Compiler 5.06 (armcc) +RUN dpkg --add-architecture i386 + +# Main apt-get call with all packages except those that have conflicts, +# handled below. One big alphabetised list, in order to avoid duplicates, with +# comments explaining why each package is needed. +RUN apt-get update -q && apt-get install -yq \ + # installed from source, but this installs the dependencies + abi-dumper \ + # to build Mbed TLS: gcc, binutils, make, etc. + build-essential \ + # to build Mbed TLS + clang \ + # to build Mbed TLS + cmake \ + # to build Mbed TLS's documentation + doxygen \ + # to cross-build Mbed TLS + gcc-mingw-w64-i686 \ + # to check out Mbed TLS and others + git \ + # to build Mbed TLS's documentation + graphviz \ + # to measure code coverage of Mbed TLS + lcov \ + # for 32-bit Mbed TLS testing and armcc + libc6-i386 \ + # for 32-bit Mbed TLS testing and armcc + libc6:i386 \ + # to build GnuTLS (nettle with public key support aka hogweed) + libgmp-dev \ + # to build GnuTLS >= 3.6 (could also use --with-included-unistring) + libunistring-dev \ + # for armcc + libstdc++6:i386 \ + # to build GnuTLS + libtasn1-6-dev \ + # needed for armcc (see locale-gen below) + locales \ + # used by compat.sh and ssl-opt.sh + lsof \ + # to build GnuTLS (nettle) + m4 \ + # to build Mbed TLS and others + make \ + # to build GnuTLS with locally-compiled nettle + pkg-config \ + # to install the preferred version of pylint + python3-pip \ + # for Mbed TLS tests + valgrind \ + # to download things installed from other places + wget \ + # to build Mbed TLS with MBEDTLS_ZILIB_SUPPORT (removed in 3.0) + zlib1g \ + # to build Mbed TLS with MBEDTLS_ZILIB_SUPPORT (removed in 3.0) + zlib1g-dev \ + && rm -rf /var/lib/apt/lists/ + +# Install all the parts of gcc-multilib, which is necessary for 32-bit builds. +# gcc-multilib conflicts with cross-compiler packages that we'll install later, +# so don't keep it around. Just let it install its dependencies +# (gcc--multilib and libc support), then remove it. Manually create +# one crucial symlink that's otherwise provided by the gcc-multilib package +# (without that symlink, 32-bit builds won't find system headers). Note that +# just installing the dependencies of gcc-multilib also brings in gcc-multilib +# as a Recommends dependency. +RUN apt-get update -q && apt-get install -yq \ + gcc-multilib \ + && rm -rf /var/lib/apt/lists/ && \ + dpkg -r gcc-multilib && \ + ln -s x86_64-linux-gnu/asm /usr/include/asm + +# Install arm-linux-gnueabi-gcc - to cross-build Mbed TLS +RUN apt-get update -q && apt-get install -yq \ + gcc-arm-linux-gnueabi \ + libc6-dev-armel-cross \ + && rm -rf /var/lib/apt/lists/ + +# Install abi-compliance-checker & abi-dumper +RUN apt-get update -q && apt-get install -yq abi-compliance-checker abi-dumper # Install ARM Compiler 5.06 -RUN dpkg --add-architecture i386 && \ - apt-get update > /dev/null && \ - apt-get install -y libc6-i386 \ - libc6:i386 \ - libstdc++6:i386 > /dev/null && \ - apt-get clean && rm -rf /var/lib/apt/lists/ - RUN wget -q https://developer.arm.com/-/media/Files/downloads/compiler/DS500-PA-00003-r5p0-22rel0.tgz && \ tar -zxf DS500-PA-00003-r5p0-22rel0.tgz && \ ./Installer/setup.sh --i-agree-to-the-contained-eula --no-interactive -d /usr/local/ARM_Compiler_5.06u3 --quiet && \ @@ -101,7 +120,7 @@ RUN wget -q https://developer.arm.com/-/media/Files/downloads/compiler/DS500-PA- ENV ARMC5_BIN_DIR=/usr/local/ARM_Compiler_5.06u3/bin/ ENV PATH=$PATH:/usr/local/ARM_Compiler_5.06u3/bin -ENV ARMLMD_LICENSE_FILE=8225@armcc.licenses.mbedcloudtesting.com +ENV ARMLMD_LICENSE_FILE=7010@10.6.26.52:7010@10.6.26.53:7010@10.6.26.54:7010@10.6.26.56 # Install ARM Compiler 6.6 RUN mkdir temp && cd temp && \ @@ -114,31 +133,36 @@ ENV ARMC6_BIN_DIR=/usr/local/ARM_Compiler_6.6/bin/ # Install arm-none-eabi-gcc RUN wget -q https://developer.arm.com/-/media/Files/downloads/gnu-rm/5_4-2016q3/gcc-arm-none-eabi-5_4-2016q3-20160926-linux.tar.bz2 -O gcc-arm-none-eabi-5_4-2016q3-20160926-linux.tar.bz2 && \ - tar -xjf gcc-arm-none-eabi-5_4-2016q3-20160926-linux.tar.bz2 -C /opt + tar -xjf gcc-arm-none-eabi-5_4-2016q3-20160926-linux.tar.bz2 -C /opt && \ + rm gcc-arm-none-eabi-5_4-2016q3-20160926-linux.tar.bz2 ENV PATH=/opt/gcc-arm-none-eabi-5_4-2016q3/bin:$PATH -# Install openssl 1.0.2g -RUN apt-get update > /dev/null && \ - apt-get install -y gcc-multilib \ - p11-kit \ - libgmp10 \ - libgmp-dev \ - pkg-config \ - m4 \ - libp11-kit-dev > /dev/null && \ - apt-get clean && rm -rf /var/lib/apt/lists/ - +# Install exact upstream versions of OpenSSL and GnuTLS +# +# Distro packages tend to include patches that disrupt our testing scripts, +# and such patches may be added at any time. Avoid surprises by using fixed +# versions. +# +# GnuTLS has a number of (optional) dependencies: +# - nettle (crypto library): quite tighly coupled, so build one for each +# version of GnuTLS that we want. +# - libtasn1: can use the Ubuntu version +# - p11-kit: optional, for smart-card support - configure it out +# - libunistring: since 3.6 - the Ubuntu package works; if it didn't a config +# option --with-included-libunistring is available. + +# Install openssl 1.0.2g - main version, in the PATH RUN wget -q https://www.openssl.org/source/old/1.0.2/openssl-1.0.2g.tar.gz && \ tar -zxf openssl-1.0.2g.tar.gz && cd openssl-1.0.2g && \ - ./config --openssldir=/usr/local/openssl-1.0.2g && \ + ./config --openssldir=/usr/local/openssl-1.0.2g enable-ssl-trace && \ make clean && make && make install && cd .. && \ rm -rf openssl-1.0.2g* ENV OPENSSL=/usr/local/openssl-1.0.2g/bin/openssl ENV PATH=/usr/local/openssl-1.0.2g/bin:$PATH -# Install openssl 1.0.1j for legacy testing +# Install openssl 1.0.1j - "legacy" version RUN wget -q https://www.openssl.org/source/old/1.0.1/openssl-1.0.1j.tar.gz && \ tar -zxf openssl-1.0.1j.tar.gz && cd openssl-1.0.1j && \ ./config --openssldir=/usr/local/openssl-1.0.1j && \ @@ -147,38 +171,24 @@ RUN wget -q https://www.openssl.org/source/old/1.0.1/openssl-1.0.1j.tar.gz && \ ENV OPENSSL_LEGACY=/usr/local/openssl-1.0.1j/bin/openssl -# Install openssl 1.1.1a for ARIA cipher testing +# Install openssl 1.1.1a - "next" version RUN wget -q https://www.openssl.org/source/openssl-1.1.1a.tar.gz && \ tar -zxf openssl-1.1.1a.tar.gz && cd openssl-1.1.1a && \ - ./config --prefix=/usr/local/openssl-1.1.1a -Wl,--enable-new-dtags,-rpath,'$(LIBRPATH)' && \ + ./config --prefix=/usr/local/openssl-1.1.1a -Wl,--enable-new-dtags,-rpath,'$(LIBRPATH)' enable-ssl-trace && \ make clean && make && make install && cd .. && \ rm -rf openssl-1.1.1a* ENV OPENSSL_NEXT=/usr/local/openssl-1.1.1a/bin/openssl -# Install Gnu TLS 3.4.10 +# Install Gnu TLS 3.4.10 (nettle 3.1) - main version, in the PATH RUN wget -q https://ftp.gnu.org/gnu/nettle/nettle-3.1.tar.gz && \ tar -zxf nettle-3.1.tar.gz && cd nettle-3.1 && \ - ./configure --prefix=/usr/local/libnettle-3.1 --exec_prefix=/usr/local/libnettle-3.1 --disable-shared && \ - make && make install && cd .. && rm -rf nettle-3.1* - -ENV PKG_CONFIG_PATH=/usr/local/libnettle-3.1/lib/pkgconfig:/usr/local/libnettle-3.1/lib64/pkgconfig:/usr/local/lib/pkgconfig - -RUN wget -q https://ftp.gnu.org/gnu/libtasn1/libtasn1-4.13.tar.gz && \ - tar -zxf libtasn1-4.13.tar.gz && cd libtasn1-4.13 && \ - ./configure && make && make install && \ - cd .. && rm -rf libtasn1-4.13* - -RUN wget -q https://github.com/p11-glue/p11-kit/releases/download/0.23.10/p11-kit-0.23.10.tar.gz && \ - tar -zxf p11-kit-0.23.10.tar.gz && cd p11-kit-0.23.10 && \ - ./configure --prefix=/usr/local/libp11-kit-0.23.10 && make && make install && \ - cd .. && rm -rf p11-kit-0.23.10* - -ENV PKG_CONFIG_PATH=/usr/local/lib/libp11-kit-0.23.10/lib/pkgconfig:/usr/local/lib/libp11-kit-0.23.10/lib64/pkgconfig:$PKG_CONFIG_PATH - -RUN wget -q https://www.gnupg.org/ftp/gcrypt/gnutls/v3.4/gnutls-3.4.10.tar.xz && \ + ./configure --prefix=/usr/local/libnettle-3.1 --exec_prefix=/usr/local/libnettle-3.1 --disable-shared --disable-openssl && \ + make && make install && cd .. && rm -rf nettle-3.1* && \ + export PKG_CONFIG_PATH=/usr/local/libnettle-3.1/lib/pkgconfig:/usr/local/libnettle-3.1/lib64/pkgconfig:/usr/local/lib/pkgconfig && \ + wget -q https://www.gnupg.org/ftp/gcrypt/gnutls/v3.4/gnutls-3.4.10.tar.xz && \ tar -xJf gnutls-3.4.10.tar.xz && cd gnutls-3.4.10 && \ - ./configure --prefix=/usr/local/gnutls-3.4.10 --exec_prefix=/usr/local/gnutls-3.4.10 --disable-shared && \ + ./configure --prefix=/usr/local/gnutls-3.4.10 --exec_prefix=/usr/local/gnutls-3.4.10 --disable-shared --without-p11-kit && \ make && make install && cat config.log && cd .. && \ rm -rf gnutls-3.4.10* @@ -186,68 +196,46 @@ ENV GNUTLS_CLI=/usr/local/gnutls-3.4.10/bin/gnutls-cli ENV GNUTLS_SERV=/usr/local/gnutls-3.4.10/bin/gnutls-serv ENV PATH=/usr/local/gnutls-3.4.10/bin:$PATH -# Install Gnu TLS 3.3.8 for legacy testing +# Install Gnu TLS 3.3.8 (nettle 2.7) - "legacy" version RUN wget -q https://ftp.gnu.org/gnu/nettle/nettle-2.7.1.tar.gz && \ tar -zxf nettle-2.7.1.tar.gz && cd nettle-2.7.1 && \ - ./configure --prefix=/usr/local/libnettle-2.7.1 --exec_prefix=/usr/local/libnettle-2.7.1 --disable-shared && \ - make && make install && cd .. && rm -rf nettle-2.7.1* - -ENV PKG_CONFIG_PATH=/usr/local/libnettle-2.7.1/lib/pkgconfig:/usr/local/libnettle-2.7.1/lib64/pkgconfig:/usr/local/lib/pkgconfig - -RUN wget -q https://www.gnupg.org/ftp/gcrypt/gnutls/v3.3/gnutls-3.3.8.tar.xz && \ + ./configure --prefix=/usr/local/libnettle-2.7.1 --exec_prefix=/usr/local/libnettle-2.7.1 --disable-shared --disable-openssl && \ + make && make install && cd .. && rm -rf nettle-2.7.1* && \ + export PKG_CONFIG_PATH=/usr/local/libnettle-2.7.1/lib/pkgconfig:/usr/local/libnettle-2.7.1/lib64/pkgconfig:/usr/local/lib/pkgconfig && \ + wget -q https://www.gnupg.org/ftp/gcrypt/gnutls/v3.3/gnutls-3.3.8.tar.xz && \ tar -xJf gnutls-3.3.8.tar.xz && cd gnutls-3.3.8 && \ - ./configure --prefix=/usr/local/gnutls-3.3.8 --exec_prefix=/usr/local/gnutls-3.3.8 --disable-shared && \ + ./configure --prefix=/usr/local/gnutls-3.3.8 --exec_prefix=/usr/local/gnutls-3.3.8 --disable-shared --without-p11-kit && \ make && make install && cat config.log && cd .. && \ rm -rf gnutls-3.3.8* ENV GNUTLS_LEGACY_CLI=/usr/local/gnutls-3.3.8/bin/gnutls-cli ENV GNUTLS_LEGACY_SERV=/usr/local/gnutls-3.3.8/bin/gnutls-serv -# Instal GNU TLS 3.6.5 for broader interoperability testing -RUN wget -q https://ftp.gnu.org/gnu/nettle/nettle-3.4.1.tar.gz && \ - tar -zxf nettle-3.4.1.tar.gz && cd nettle-3.4.1 && \ - ./configure --prefix=/usr/local/libnettle-3.4.1 --exec_prefix=/usr/local/libnettle-3.4.1 --disable-shared && \ - make && make install && cd .. && rm -rf nettle-3.4.1* - -ENV PKG_CONFIG_PATH=/usr/local/libnettle-3.4.1/lib/pkgconfig:/usr/local/libnettle-3.4.1/lib64/pkgconfig:/usr/local/lib/pkgconfig - -RUN apt-get update > /dev/null && \ - apt-get install -y libunistring-dev > /dev/null && \ - apt-get clean && rm -rf /var/lib/apt/lists/ - -RUN wget -q https://www.gnupg.org/ftp/gcrypt/gnutls/v3.6/gnutls-3.6.5.tar.xz && \ - tar -xJf gnutls-3.6.5.tar.xz && cd gnutls-3.6.5 && \ - ./configure --prefix=/usr/local/gnutls-3.6.5 --exec_prefix=/usr/local/gnutls-3.6.5 --disable-shared && \ +# Instal GNU TLS 3.7.2 (nettle 3.7) - "next" version +RUN wget -q https://ftp.gnu.org/gnu/nettle/nettle-3.7.3.tar.gz && \ + tar -zxf nettle-3.7.3.tar.gz && cd nettle-3.7.3 && \ + ./configure --prefix=/usr/local/libnettle-3.7.3 --exec_prefix=/usr/local/libnettle-3.7.3 --disable-shared --disable-openssl && \ + make && make install && cd .. && rm -rf nettle-3.7.3* && \ + export PKG_CONFIG_PATH=/usr/local/libnettle-3.7.3/lib/pkgconfig:/usr/local/libnettle-3.7.3/lib64/pkgconfig:/usr/local/lib/pkgconfig && \ + wget -q https://www.gnupg.org/ftp/gcrypt/gnutls/v3.7/gnutls-3.7.2.tar.xz && \ + tar -xJf gnutls-3.7.2.tar.xz && cd gnutls-3.7.2 && \ + ./configure --prefix=/usr/local/gnutls-3.7.2 --exec_prefix=/usr/local/gnutls-3.7.2 --disable-shared --without-p11-kit && \ make && make install && cat config.log && cd .. && \ - rm -rf gnutls-3.6.5* - -ENV GNUTLS_NEXT_CLI=/usr/local/gnutls-3.6.5/bin/gnutls-cli -ENV GNUTLS_NEXT_SERV=/usr/local/gnutls-3.6.5/bin/gnutls-serv + rm -rf gnutls-3.7.2* -# Remove Ubuntu unattended-upgrades to prevent unwanted changes to system while it is running -RUN apt-get purge -y unattended-upgrades - - -# Install Python 2 pip packages -# The pip wrapper scripts can get out of sync with pip due to upgrading it outside the package manager, so invoke the module directly -RUN python2 -m pip install pip setuptools --upgrade > /dev/null - -RUN python2 -m pip install yotta matplotlib > /dev/null +ENV GNUTLS_NEXT_CLI=/usr/local/gnutls-3.7.2/bin/gnutls-cli +ENV GNUTLS_NEXT_SERV=/usr/local/gnutls-3.7.2/bin/gnutls-serv # Install Python pip packages # # The pip wrapper scripts can get out of sync with pip due to upgrading it # outside the package manager, so invoke the module directly. # -# Ubuntu 18.04's pip (9.0.1) doesn't support suppressing the progress bar, -# which is annoying in CI logs. Install pip<21, same as on Ubuntu 16.04 -# (although we could use a later version if we wanted). -# # Piping to cat suppresses the progress bar, but means that a failure # won't be caught (`stuff | cat` succeeds if cat succeeds, even if `stuff` # fails). The subsequent use of "pip config" (which requires pip >=10) # will however fail if the installation of a more recent pip failed. -RUN python3 -m pip install 'pip<21' --upgrade | cat && \ +RUN python3 -m pip install pip --upgrade | cat && \ python3 -m pip config set global.progress_bar off && \ python3 -m pip install setuptools --upgrade && \ # For pylint we want a known version, as later versions may add checks at @@ -258,12 +246,10 @@ RUN python3 -m pip install 'pip<21' --upgrade | cat && \ python3 -m pip install mypy==0.780 && \ true -# Set the locale -RUN apt-get clean && apt-get update && apt-get install -y locales zip python-tk -RUN locale-gen en_US.UTF-8 - # Set locale for ARMCC to work -RUN locale && locale-gen "en_US.UTF-8" && dpkg-reconfigure locales +RUN locale && \ + locale-gen "en_US.UTF-8" && \ + dpkg-reconfigure locales # Add user RUN useradd -m user @@ -277,3 +263,4 @@ ENV AGENT_WORKDIR=${AGENT_WORKDIR} WORKDIR ${AGENT_WORKDIR} ENTRYPOINT ["bash"] + diff --git a/resources/docker_files/ubuntu-20.04/Dockerfile b/resources/docker_files/ubuntu-20.04/Dockerfile new file mode 100644 index 000000000..7b020e6d3 --- /dev/null +++ b/resources/docker_files/ubuntu-20.04/Dockerfile @@ -0,0 +1,269 @@ +# ubuntu-20.04/Dockerfile +# +# Copyright (c) 2018-2022, ARM Limited, All Rights Reserved +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file is part of Mbed TLS (https://www.trustedfirmware.org/projects/mbed-tls/) + +# Purpose +# ------- +# +# This docker file is for creating the ubuntu-20.04 image that is used in the +# CI. It can also be used for reproducing and testing CI failures. + +FROM ubuntu:20.04 + +ARG DEBIAN_FRONTEND=noninteractive +WORKDIR /opt/src + +# Support for i386: +# - for 32-bit builds+tests of Mbed TLS +# - required to install Arm Compiler 5.06 (armcc) +RUN dpkg --add-architecture i386 + +# Main apt-get call with all packages except those that have conflicts, +# handled below. One big alphabetised list, in order to avoid duplicates, with +# comments explaining why each package is needed. +RUN apt-get update -q && apt-get install -yq \ + # installed from source, but this installs the dependencies + abi-dumper \ + # to build Mbed TLS: gcc, binutils, make, etc. + build-essential \ + # to build Mbed TLS + clang \ + # to build Mbed TLS + cmake \ + # to build Mbed TLS's documentation + doxygen \ + # to cross-build Mbed TLS + gcc-mingw-w64-i686 \ + # to check out Mbed TLS and others + git \ + # to build Mbed TLS's documentation + graphviz \ + # to measure code coverage of Mbed TLS + lcov \ + # for 32-bit Mbed TLS testing and armcc + libc6-i386 \ + # for 32-bit Mbed TLS testing and armcc + libc6:i386 \ + # to build GnuTLS (nettle with public key support aka hogweed) + libgmp-dev \ + # to build GnuTLS >= 3.6 (could also use --with-included-unistring) + libunistring-dev \ + # for armcc + libstdc++6:i386 \ + # to build GnuTLS + libtasn1-6-dev \ + # needed for armcc (see locale-gen below) + locales \ + # used by compat.sh and ssl-opt.sh + lsof \ + # to build GnuTLS (nettle) + m4 \ + # to build Mbed TLS and others + make \ + # to build GnuTLS with locally-compiled nettle + pkg-config \ + # to install the preferred version of pylint + python3-pip \ + # for Mbed TLS tests + valgrind \ + # to download things installed from other places + wget \ + # to build Mbed TLS with MBEDTLS_ZILIB_SUPPORT (removed in 3.0) + zlib1g \ + # to build Mbed TLS with MBEDTLS_ZILIB_SUPPORT (removed in 3.0) + zlib1g-dev \ + && rm -rf /var/lib/apt/lists/ + +# Install all the parts of gcc-multilib, which is necessary for 32-bit builds. +# gcc-multilib conflicts with cross-compiler packages that we'll install later, +# so don't keep it around. Just let it install its dependencies +# (gcc--multilib and libc support), then remove it. Manually create +# one crucial symlink that's otherwise provided by the gcc-multilib package +# (without that symlink, 32-bit builds won't find system headers). Note that +# just installing the dependencies of gcc-multilib also brings in gcc-multilib +# as a Recommends dependency. +RUN apt-get update -q && apt-get install -yq \ + gcc-multilib \ + && rm -rf /var/lib/apt/lists/ && \ + dpkg -r gcc-multilib && \ + ln -s x86_64-linux-gnu/asm /usr/include/asm + +# Install arm-linux-gnueabi-gcc - to cross-build Mbed TLS +RUN apt-get update -q && apt-get install -yq \ + gcc-arm-linux-gnueabi \ + libc6-dev-armel-cross \ + && rm -rf /var/lib/apt/lists/ + +# Install ARM Compiler 5.06 +RUN wget -q https://developer.arm.com/-/media/Files/downloads/compiler/DS500-PA-00003-r5p0-22rel0.tgz && \ + tar -zxf DS500-PA-00003-r5p0-22rel0.tgz && \ + ./Installer/setup.sh --i-agree-to-the-contained-eula --no-interactive -d /usr/local/ARM_Compiler_5.06u3 --quiet && \ + rm -rf DS500-PA-00003-r5p0-22rel0.tgz releasenotes.html Installer/ + +ENV ARMC5_BIN_DIR=/usr/local/ARM_Compiler_5.06u3/bin/ +ENV PATH=$PATH:/usr/local/ARM_Compiler_5.06u3/bin +ENV ARMLMD_LICENSE_FILE=7010@10.6.26.52:7010@10.6.26.53:7010@10.6.26.54:7010@10.6.26.56 + +# Install ARM Compiler 6.6 +RUN mkdir temp && cd temp && \ + wget -q --no-check-certificate https://developer.arm.com/-/media/Files/downloads/compiler/DS500-BN-00026-r5p0-07rel0.tgz?revision=8f0d9fb0-9616-458c-b2f5-d0dac83ea93c?product=Downloads,64-bit,,Linux,6.6 -O arm6.tgz && \ + tar -zxf arm6.tgz && ls -ltr && \ + ./install_x86_64.sh --i-agree-to-the-contained-eula --no-interactive -d /usr/local/ARM_Compiler_6.6 --quiet && \ + cd .. && rm -rf temp/ + +ENV ARMC6_BIN_DIR=/usr/local/ARM_Compiler_6.6/bin/ + +# Install arm-none-eabi-gcc +RUN wget -q https://developer.arm.com/-/media/Files/downloads/gnu-rm/5_4-2016q3/gcc-arm-none-eabi-5_4-2016q3-20160926-linux.tar.bz2 -O gcc-arm-none-eabi-5_4-2016q3-20160926-linux.tar.bz2 && \ + tar -xjf gcc-arm-none-eabi-5_4-2016q3-20160926-linux.tar.bz2 -C /opt && \ + rm gcc-arm-none-eabi-5_4-2016q3-20160926-linux.tar.bz2 + +ENV PATH=/opt/gcc-arm-none-eabi-5_4-2016q3/bin:$PATH + +# Install abi-compliance-checker +RUN apt-get update -q && apt-get install -yq abi-compliance-checker + +# Install abi-dumper +RUN apt-get install -yq abi-dumper + +# Install exact upstream versions of OpenSSL and GnuTLS +# +# Distro packages tend to include patches that disrupt our testing scripts, +# and such patches may be added at any time. Avoid surprises by using fixed +# versions. +# +# GnuTLS has a number of (optional) dependencies: +# - nettle (crypto library): quite tighly coupled, so build one for each +# version of GnuTLS that we want. +# - libtasn1: can use the Ubuntu version +# - p11-kit: optional, for smart-card support - configure it out +# - libunistring: since 3.6 - the Ubuntu package works; if it didn't a config +# option --with-included-libunistring is available. + +# Install openssl 1.0.2g - main version, in the PATH +RUN wget -q https://www.openssl.org/source/old/1.0.2/openssl-1.0.2g.tar.gz && \ + tar -zxf openssl-1.0.2g.tar.gz && cd openssl-1.0.2g && \ + ./config --openssldir=/usr/local/openssl-1.0.2g enable-ssl-trace && \ + make clean && make && make install && cd .. && \ + rm -rf openssl-1.0.2g* + +ENV OPENSSL=/usr/local/openssl-1.0.2g/bin/openssl +ENV PATH=/usr/local/openssl-1.0.2g/bin:$PATH + +# Install openssl 1.0.1j - "legacy" version +RUN wget -q https://www.openssl.org/source/old/1.0.1/openssl-1.0.1j.tar.gz && \ + tar -zxf openssl-1.0.1j.tar.gz && cd openssl-1.0.1j && \ + ./config --openssldir=/usr/local/openssl-1.0.1j && \ + make clean && make && make install && cd .. && \ + rm -rf openssl-1.0.1j* + +ENV OPENSSL_LEGACY=/usr/local/openssl-1.0.1j/bin/openssl + +# Install openssl 1.1.1a - "next" version +RUN wget -q https://www.openssl.org/source/openssl-1.1.1a.tar.gz && \ + tar -zxf openssl-1.1.1a.tar.gz && cd openssl-1.1.1a && \ + ./config --prefix=/usr/local/openssl-1.1.1a -Wl,--enable-new-dtags,-rpath,'$(LIBRPATH)' enable-ssl-trace && \ + make clean && make && make install && cd .. && \ + rm -rf openssl-1.1.1a* + +ENV OPENSSL_NEXT=/usr/local/openssl-1.1.1a/bin/openssl + +# Install Gnu TLS 3.4.10 (nettle 3.1) - main version, in the PATH +RUN wget -q https://ftp.gnu.org/gnu/nettle/nettle-3.1.tar.gz && \ + tar -zxf nettle-3.1.tar.gz && cd nettle-3.1 && \ + ./configure --prefix=/usr/local/libnettle-3.1 --exec_prefix=/usr/local/libnettle-3.1 --disable-shared --disable-openssl && \ + make && make install && cd .. && rm -rf nettle-3.1* && \ + export PKG_CONFIG_PATH=/usr/local/libnettle-3.1/lib/pkgconfig:/usr/local/libnettle-3.1/lib64/pkgconfig:/usr/local/lib/pkgconfig && \ + wget -q https://www.gnupg.org/ftp/gcrypt/gnutls/v3.4/gnutls-3.4.10.tar.xz && \ + tar -xJf gnutls-3.4.10.tar.xz && cd gnutls-3.4.10 && \ + ./configure --prefix=/usr/local/gnutls-3.4.10 --exec_prefix=/usr/local/gnutls-3.4.10 --disable-shared --without-p11-kit && \ + make && make install && cat config.log && cd .. && \ + rm -rf gnutls-3.4.10* + +ENV GNUTLS_CLI=/usr/local/gnutls-3.4.10/bin/gnutls-cli +ENV GNUTLS_SERV=/usr/local/gnutls-3.4.10/bin/gnutls-serv +ENV PATH=/usr/local/gnutls-3.4.10/bin:$PATH + +# Install Gnu TLS 3.3.8 (nettle 2.7) - "legacy" version +RUN wget -q https://ftp.gnu.org/gnu/nettle/nettle-2.7.1.tar.gz && \ + tar -zxf nettle-2.7.1.tar.gz && cd nettle-2.7.1 && \ + ./configure --prefix=/usr/local/libnettle-2.7.1 --exec_prefix=/usr/local/libnettle-2.7.1 --disable-shared --disable-openssl && \ + make && make install && cd .. && rm -rf nettle-2.7.1* && \ + export PKG_CONFIG_PATH=/usr/local/libnettle-2.7.1/lib/pkgconfig:/usr/local/libnettle-2.7.1/lib64/pkgconfig:/usr/local/lib/pkgconfig && \ + wget -q https://www.gnupg.org/ftp/gcrypt/gnutls/v3.3/gnutls-3.3.8.tar.xz && \ + tar -xJf gnutls-3.3.8.tar.xz && cd gnutls-3.3.8 && \ + ./configure --prefix=/usr/local/gnutls-3.3.8 --exec_prefix=/usr/local/gnutls-3.3.8 --disable-shared --without-p11-kit && \ + make && make install && cat config.log && cd .. && \ + rm -rf gnutls-3.3.8* + +ENV GNUTLS_LEGACY_CLI=/usr/local/gnutls-3.3.8/bin/gnutls-cli +ENV GNUTLS_LEGACY_SERV=/usr/local/gnutls-3.3.8/bin/gnutls-serv + +# Instal GNU TLS 3.7.2 (nettle 3.7) - "next" version +RUN wget -q https://ftp.gnu.org/gnu/nettle/nettle-3.7.3.tar.gz && \ + tar -zxf nettle-3.7.3.tar.gz && cd nettle-3.7.3 && \ + ./configure --prefix=/usr/local/libnettle-3.7.3 --exec_prefix=/usr/local/libnettle-3.7.3 --disable-shared --disable-openssl && \ + make && make install && cd .. && rm -rf nettle-3.7.3* && \ + export PKG_CONFIG_PATH=/usr/local/libnettle-3.7.3/lib/pkgconfig:/usr/local/libnettle-3.7.3/lib64/pkgconfig:/usr/local/lib/pkgconfig && \ + wget -q https://www.gnupg.org/ftp/gcrypt/gnutls/v3.7/gnutls-3.7.2.tar.xz && \ + tar -xJf gnutls-3.7.2.tar.xz && cd gnutls-3.7.2 && \ + ./configure --prefix=/usr/local/gnutls-3.7.2 --exec_prefix=/usr/local/gnutls-3.7.2 --disable-shared --without-p11-kit && \ + make && make install && cat config.log && cd .. && \ + rm -rf gnutls-3.7.2* + +ENV GNUTLS_NEXT_CLI=/usr/local/gnutls-3.7.2/bin/gnutls-cli +ENV GNUTLS_NEXT_SERV=/usr/local/gnutls-3.7.2/bin/gnutls-serv + +# Install Python pip packages +# +# The pip wrapper scripts can get out of sync with pip due to upgrading it +# outside the package manager, so invoke the module directly. +# +# Piping to cat suppresses the progress bar, but means that a failure +# won't be caught (`stuff | cat` succeeds if cat succeeds, even if `stuff` +# fails). The subsequent use of "pip config" (which requires pip >=10) +# will however fail if the installation of a more recent pip failed. +RUN python3 -m pip install pip --upgrade | cat && \ + python3 -m pip config set global.progress_bar off && \ + python3 -m pip install setuptools --upgrade && \ + # For pylint we want a known version, as later versions may add checks at + # any time, making CI results unpredictable. + python3 -m pip install pylint==2.4.4 && \ + # For mypy, use the earliest version that works with our code base. + # See https://github.com/ARMmbed/mbedtls/pull/3953 . + python3 -m pip install mypy==0.780 && \ + true + +# Set locale for ARMCC to work +RUN locale && \ + locale-gen "en_US.UTF-8" && \ + dpkg-reconfigure locales + +# Add user +RUN useradd -m user + +# Create workspace +ARG AGENT_WORKDIR=/var/lib/builds +RUN mkdir -p ${AGENT_WORKDIR} && chown user:user ${AGENT_WORKDIR} +USER user +ENV AGENT_WORKDIR=${AGENT_WORKDIR} + +WORKDIR ${AGENT_WORKDIR} + +ENTRYPOINT ["bash"] + diff --git a/vars/common.groovy b/vars/common.groovy index 8dd4a34fa..2e78ce21d 100644 --- a/vars/common.groovy +++ b/vars/common.groovy @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021, Arm Limited, All Rights Reserved + * Copyright (c) 2019-2022, Arm Limited, All Rights Reserved * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -41,7 +41,7 @@ import groovy.transform.Field @Field docker_ecr = "666618195821.dkr.ecr.eu-west-1.amazonaws.com" @Field docker_repo = "$docker_ecr/$docker_repo_name" -@Field linux_platforms = ["ubuntu-16.04", "ubuntu-18.04"] +@Field linux_platforms = ["ubuntu-16.04", "ubuntu-18.04", "ubuntu-20.04"] @Field bsd_platforms = ["freebsd"] @Field bsd_compilers = ["clang"] @Field all_compilers = ['gcc', 'clang'] diff --git a/vars/dockerfile-builder-Jenkinsfile b/vars/dockerfile-builder-Jenkinsfile index 3eb689ae8..af310c66d 100644 --- a/vars/dockerfile-builder-Jenkinsfile +++ b/vars/dockerfile-builder-Jenkinsfile @@ -9,6 +9,7 @@ * Build options * - DOCKER_IMAGE_16_04_ACTION * - DOCKER_IMAGE_18_04_ACTION + * - DOCKER_IMAGE_20_04_ACTION * * Environment variables: * - GIT_CREDENTIALS_ID diff --git a/vars/dockerfile_builder.groovy b/vars/dockerfile_builder.groovy index 7b2392697..0f8cd8018 100644 --- a/vars/dockerfile_builder.groovy +++ b/vars/dockerfile_builder.groovy @@ -14,6 +14,7 @@ def run_job() { stage('dockerfile-builder') { def jobs = gen_job_for_action(DOCKER_IMAGE_16_04_ACTION, 'ubuntu-16.04') jobs += gen_job_for_action(DOCKER_IMAGE_18_04_ACTION, 'ubuntu-18.04') + jobs += gen_job_for_action(DOCKER_IMAGE_20_04_ACTION, 'ubuntu-20.04') jobs.failFast = false parallel jobs } diff --git a/vars/gen_jobs.groovy b/vars/gen_jobs.groovy index e7deade43..5738bf258 100644 --- a/vars/gen_jobs.groovy +++ b/vars/gen_jobs.groovy @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021, Arm Limited, All Rights Reserved + * Copyright (c) 2019-2022, Arm Limited, All Rights Reserved * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -220,16 +220,10 @@ scripts/min_requirements.py --user } dir('src') { checkout_repo.checkout_repo() - /* ARMLMD_LICENSE_FILE is supposed to be set in the - * Dockerfile, but the value there is out-of-date and we - * have trouble re-building the images (due to the death - * of Python 2), so we override it here as a temporary - * work-around. */ writeFile file: 'steps.sh', text: """\ #!/bin/sh set -eux ulimit -f 20971520 -export ARMLMD_LICENSE_FILE="7010@10.6.26.52:7010@10.6.26.53:7010@10.6.26.54:7010@10.6.26.56" export MBEDTLS_TEST_OUTCOME_FILE='${job_name}-outcome.csv' ${extra_setup_code} ./tests/scripts/all.sh --seed 4 --keep-going $component @@ -528,12 +522,9 @@ pip install -r requirements.txt if (example == 'atecc608a') { tag_filter = "--tag-filters HAS_CRYPTOKIT" } - /* See gen_all_sh_jobs() regarding setting of - * ARMLMD_LICENSE_FILE here. */ sh """\ ulimit -f 20971520 . $WORKSPACE/mbed-venv/bin/activate -export ARMLMD_LICENSE_FILE="7010@10.6.26.52:7010@10.6.26.53:7010@10.6.26.54:7010@10.6.26.56" mbed compile -m ${platform} -t ${compiler} """ for (int attempt = 1; attempt <= 3; attempt++) { @@ -585,13 +576,20 @@ def gen_release_jobs(label_prefix='', run_examples=true) { } if (env.RUN_ALL_SH == "true") { - for (component in common.available_all_sh_components['ubuntu-16.04']) { - jobs = jobs + gen_all_sh_jobs('ubuntu-16.04', component, label_prefix) + for (component in (common.available_all_sh_components['ubuntu-20.04'])) { + jobs = jobs + gen_all_sh_jobs('ubuntu-20.04', component, label_prefix) } + + for (component in (common.available_all_sh_components['ubuntu-16.04'] - + common.available_all_sh_components['ubuntu-20.04'])) { + jobs = jobs + gen_all_sh_jobs('ubuntu-16.04', component, label_prefix) + } + for (component in (common.available_all_sh_components['ubuntu-18.04'] - + common.available_all_sh_components['ubuntu-20.04'] - common.available_all_sh_components['ubuntu-16.04'])) { jobs = jobs + gen_all_sh_jobs('ubuntu-18.04', component, label_prefix) - } + } } /* FreeBSD all.sh jobs */