Merge pull request #1709 from habibayassin/scripted-pipeline

vvbandeira · web-flow · commit 05ac51182f9b · 2024-06-27T14:52:11.000-07:00
jenkins: scripted pipelines and shared funcs
diff --git a/.github/workflows/github-actions-cron-test-installer.yml b/.github/workflows/github-actions-cron-test-installer.yml
@@ -26,7 +26,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: ["ubuntu20.04", "ubuntu22.04", "centos7"]
+        os: ["ubuntu20.04", "ubuntu22.04"]
     runs-on: ubuntu-latest
     steps:
       - name: Check out repository code
@@ -52,9 +52,4 @@ jobs:
       - name: Test build
         run: |
           cmd="source ./env.sh ; yosys -help ; openroad -help ; make -C flow ;"
-          if [[ ${{ matrix.os }} == "centos7" ]]; then
-            cmd="source /opt/rh/devtoolset-8/enable; ${cmd}"
-            cmd="source /opt/rh/llvm-toolset-7.0/enable; ${cmd}"
-            cmd="source /opt/rh/rh-python38/enable; ${cmd}"
-          fi
           docker run openroad/flow-${{ matrix.os }}-builder /bin/bash -c "${cmd}"
diff --git a/.gitignore b/.gitignore
@@ -89,3 +89,4 @@ docs/main
 docs/build
 GitGuide.md
 mainREADME.md
+build
diff --git a/docker/Dockerfile.builder b/docker/Dockerfile.builder
@@ -2,7 +2,7 @@
 # i.e., "./build_openroad.sh --local" from inside a docker container
 # NOTE: don't use this file directly unless you know what you are doing,
 # instead use etc/DockerHelper.sh
-ARG fromImage=openroad/flow-centos7-dev:latest
+ARG fromImage=openroad/flow-ubuntu22.04-dev:latest
 
 FROM $fromImage
 
diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev
@@ -1,7 +1,7 @@
 # create image with dependencies needed to compile openroad app and run openroad flow scripts
 # NOTE: don't use this file directly unless you know what you are doing,
 # instead use etc/DockerHelper.sh
-ARG fromImage=centos:centos7
+ARG fromImage=ubuntu:22.04
 
 FROM $fromImage
 
@@ -12,11 +12,13 @@ COPY DependencyInstaller.sh .
 COPY InstallerOpenROAD.sh \
        /tmp/installer/tools/OpenROAD/etc/DependencyInstaller.sh
 
-RUN ./DependencyInstaller.sh \
+ARG options=""
+
+RUN ./DependencyInstaller.sh $options \
       && rm -rf /tmp/installer
 
 ARG fromImage
 
 RUN echo "$fromImage" | grep -q "ubuntu" && \
     strip --remove-section=.note.ABI-tag /usr/lib/x86_64-linux-gnu/libQt5Core.so || \
-    echo "Skipping strip command as fromImage does not contain 'ubuntu'"
+    echo "Skipping strip command as fromImage does not contain 'ubuntu'"
diff --git a/etc/DependencyInstaller.sh b/etc/DependencyInstaller.sh
@@ -111,6 +111,29 @@ _installDarwinPackages() {
     brew install --cask klayout
 }
 
+_installCI() {
+    apt-get -y update
+
+    #docker
+    apt install -y \
+        apt-transport-https \
+        ca-certificates \
+        curl \
+        software-properties-common
+    curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
+    echo \
+    "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \
+    $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
+    apt-get -y update
+    apt-get install -y docker-ce docker-ce-cli containerd.io
+
+    # python
+    apt-get install -y python3
+
+    # stdbuf
+    apt-get install -y coreutils
+}
+
 _help() {
     cat <<EOF
 
@@ -138,6 +161,8 @@ Usage: $0
                                 #    "$HOME/.local". Only used with
                                 #    -common. This flag cannot be used with
                                 #    sudo or with root access.
+       $0 -ci
+                                # Installs CI tools
 EOF
     exit "${1:-1}"
 }
@@ -150,6 +175,7 @@ PREFIX=""
 option="all"
 # default isLocal
 isLocal="false"
+CI="no"
 
 # default values, can be overwritten by cmdline args
 while [ "$#" -gt 0 ]; do
@@ -174,6 +200,9 @@ while [ "$#" -gt 0 ]; do
         -local)
             OR_INSTALLER_ARGS="${OR_INSTALLER_ARGS} -local"
             ;;
+        -ci)
+            CI="yes"
+            ;;
         -prefix=*)
             OR_INSTALLER_ARGS="${OR_INSTALLER_ARGS} $1"
             ;;
@@ -209,6 +238,9 @@ esac
 
 case "${os}" in
     "CentOS Linux" )
+        if [[ ${CI} == "yes" ]]; then
+            echo "WARNING: Installing CI dependencies is only supported on Ubuntu 22.04" >&2
+        fi
         _installORDependencies
         if [[ "${option}" == "base" || "${option}" == "all" ]]; then
             _installCentosPackages
@@ -220,6 +252,10 @@ case "${os}" in
         ;;
     "Ubuntu" )
         version=$(awk -F= '/^VERSION_ID/{print $2}' /etc/os-release | sed 's/"//g')
+        if [[ ${CI} == "yes" ]]; then
+            echo "Installing CI Tools"
+            _installCI
+        fi
         _installORDependencies
         if [[ "${option}" == "base" || "${option}" == "all" ]]; then
             _installUbuntuPackages "${version}"
@@ -232,6 +268,9 @@ case "${os}" in
         fi
         ;;
     "Darwin" )
+        if [[ ${CI} == "yes" ]]; then
+            echo "WARNING: Installing CI dependencies is only supported on Ubuntu 22.04" >&2
+        fi
         _installORDependencies
         if [[ "${option}" == "base" || "${option}" == "all" ]]; then
             _installDarwinPackages
diff --git a/etc/DockerHelper.sh b/etc/DockerHelper.sh
@@ -18,26 +18,26 @@ usage: $0 [CMD] [OPTIONS]
   push                          Push the docker image to Docker Hub
 
   OPTIONS:
-  -os=OS_NAME                   Choose beween centos7 (default), ubuntu20.04 and ubuntu22.04.
+  -os=OS_NAME                   Choose beween ubuntu20.04 and ubuntu22.04 (default).
   -target=TARGET                Choose target fo the Docker image:
                                   'dev': os + packages to compile app
                                   'builder': os + packages to compile app +
                                              copy source code and build app
   -threads                      Max number of threads to use if compiling.
   -sha                          Use git commit sha as the tag image. Default is
                                   'latest'.
+  -ci                           Install CI tools in image
   -h -help                      Show this message and exits
+  -username                     Docker Username
+  -password                     Docker Password
 
 EOF
     exit "${1:-1}"
 }
 
 _setup() {
-    commitSha="$(git rev-parse HEAD)"
+    commitSha="$(git rev-parse HEAD | tr -cd 'a-zA-Z0-9-')"
     case "${os}" in
-        "centos7")
-            osBaseImage="centos:centos7"
-            ;;
         "ubuntu20.04")
             osBaseImage="ubuntu:20.04"
             ;;
@@ -50,13 +50,12 @@ _setup() {
             ;;
     esac
     imageName="${IMAGE_NAME_OVERRIDE:-"${org}/flow-${os}-${target}"}"
-    if [[ "${useCommitSha}" == "yes" ]]; then
-        imageTag="${commitSha}"
-    else
-        imageTag="latest"
+    imageTag="${commitSha}"
+    if [[ "${tag}" != "NONE" ]]; then
+        imageTag="${tag}"
     fi
     case "${target}" in
-        "builder" )
+        "builder" | "master")
             fromImage="${FROM_IMAGE_OVERRIDE:-"${org}/flow-${os}-dev"}:${imageTag}"
             context="."
             buildArgs="--build-arg numThreads=${numThreads}"
@@ -65,7 +64,7 @@ _setup() {
             fromImage="${FROM_IMAGE_OVERRIDE:-$osBaseImage}"
             cp tools/OpenROAD/etc/DependencyInstaller.sh etc/InstallerOpenROAD.sh
             context="etc"
-            buildArgs=""
+            buildArgs="--build-arg options=${options}"
             ;;
         *)
             echo "Target ${target} not found" >&2
@@ -79,38 +78,51 @@ _setup() {
 
 _create() {
     echo "Create docker image ${imagePath} using ${file}"
-    docker build --file "${file}" --tag "${imagePath}" ${buildArgs} "${context}"
+    docker build \
+        --file "${file}" \
+        --tag "${imagePath}" \
+        ${buildArgs} \
+        "${context}" \
+        --progress plain
     rm -f etc/InstallerOpenROAD.sh
 }
 
 _push() {
+    if [[ -z ${username+x} ]]; then
+        echo "Missing required -username=<USER> argument"
+        _help
+    fi
+    if [[ -z ${password+x} ]]; then
+        echo "Missing required -password=<PASS> argument"
+        _help
+    fi
+    docker login --username ${username} --password ${password}
+    if [[ "${tag}" == "NONE" ]]; then
+        tag="latest"
+    fi
+    mkdir -p build
     case "${target}" in
         "dev" )
-            read -p "Will push docker image ${imagePath} to DockerHub [y/N]" -n 1 -r
-            echo
-            if [[ $REPLY =~ ^[Yy]$  ]]; then
-                mkdir -p build
-
-                OS_LIST="centos7 ubuntu20.04 ubuntu22.04"
-                # create image with sha and latest tag for all os
-                for os in ${OS_LIST}; do
-                    ./etc/DockerHelper.sh create -target=dev \
-                        2>&1 | tee build/create-${os}-latest.log
-                    ./etc/DockerHelper.sh create -target=dev -sha \
-                        2>&1 | tee build/create-${os}-${commitSha}.log
-                done
-
-                for os in ${OS_LIST}; do
-                    echo [DRY-RUN] docker push openroad/flow-${os}-dev:latest
-                    echo [DRY-RUN] docker push openroad/flow-${os}-dev:${commitSha}
-                done
-
-            else
-                echo "Will not push."
-            fi
+            ./etc/DockerHelper.sh create -os=${os} -ci -target=${target} \
+                2>&1 | tee build/create-${os}-${target}-${tag}.log
+            docker push ${imagePath}
             ;;
+
+        "master" )
+            # Create dev image needed as a base for builder image
+            ./etc/DockerHelper.sh create -os=${os} -target=dev \
+                2>&1 | tee build/create-${os}-dev-${target}-${tag}.log
+            # Create builder image
+            ./etc/DockerHelper.sh create -os=${os} -target=builder \
+                2>&1 | tee build/create-${os}-${target}-${tag}.log
+            docker push ${org}/flow-${os}-dev:${commitSha}
+            docker push ${org}/flow-${os}-dev:${commitSha} ${org}/flow-${os}-dev:latest
+            docker push ${org}/flow-${os}-builder:${commitSha} ${org}/orfs:${commitSha}
+            docker push ${org}/flow-${os}-builder:${commitSha} ${org}/orfs:${tag}
+            ;;
+
         *)
-            echo "Target ${target} is not valid candidate for push to DockerHub." >&2
+            echo "Target ${target} is not valid candidate for push to Docker Hub." >&2
             _help
             ;;
     esac
@@ -139,17 +151,21 @@ if [[ -z $(command -v "${_rule}") ]]; then
     _help
 fi
 
-# default values, can be overwritten by cmdline args
-os="centos7"
+# default values, can be overwritten by command line arguments
+os="ubuntu22.04"
 target="dev"
-useCommitSha="no"
 numThreads="-1"
+tag="NONE"
+options=""
 
 while [ "$#" -gt 0 ]; do
     case "${1}" in
         -h|-help)
             _help 0
             ;;
+        -ci )
+            options="-ci"
+            ;;
         -os=* )
             os="${1#*=}"
             ;;
@@ -159,10 +175,16 @@ while [ "$#" -gt 0 ]; do
         -threads=* )
             numThreads="${1#*=}"
             ;;
-        -sha )
-            useCommitSha=yes
+        -username=* )
+            username="${1#*=}"
+            ;;
+        -password=* )
+            password="${1#*=}"
+            ;;
+        -tag=* )
+            tag="${1#*=}"
             ;;
-        -os | -target )
+        -os | -target | -threads | -username | -password | -tag )
             echo "${1} requires an argument" >&2
             _help
             ;;
diff --git a/flow/test/test_delta_debug.sh b/flow/test/test_delta_debug.sh
@@ -6,6 +6,7 @@
 # variable names such as $(false), unused variables, etc.
 set -ue -o pipefail
 
+cd flow
 testname=uart
 
 make DESIGN_CONFIG=designs/asap7/$testname/config.mk place
diff --git a/jenkins/public_tests_all.Jenkinsfile b/jenkins/public_tests_all.Jenkinsfile
diff --git a/jenkins/public_tests_small.Jenkinsfile b/jenkins/public_tests_small.Jenkinsfile
