Merge remote-tracking branch 'origin/master' into secure-pin-swap-fix-flow

precisionmoon · precisionmoon · commit 2914ac9c33fe · 2024-07-04T01:47:40.000Z
diff --git a/build_openroad.sh b/build_openroad.sh
@@ -206,6 +206,8 @@ EOF
         fi
 fi
 
+echo "[INFO FLW-0028] Compiling with ${PROC} threads."
+
 # Only add install prefix variables after parsing arguments.
 YOSYS_ARGS+=" PREFIX=${INSTALL_PATH}/yosys"
 OPENROAD_APP_ARGS+=" -D CMAKE_INSTALL_PREFIX=${INSTALL_PATH}/OpenROAD"
diff --git a/etc/DockerHelper.sh b/etc/DockerHelper.sh
@@ -24,8 +24,7 @@ usage: $0 [CMD] [OPTIONS]
                                   'builder': os + packages to compile app +
                                              copy source code and build app
   -threads                      Max number of threads to use if compiling.
-  -sha                          Use git commit sha as the tag image. Default is
-                                  'latest'.
+                                  Default = \$(nproc)
   -ci                           Install CI tools in image
   -h -help                      Show this message and exits
   -username                     Docker Username
@@ -36,7 +35,6 @@ EOF
 }
 
 _setup() {
-    commitSha="$(git rev-parse HEAD | tr -cd 'a-zA-Z0-9-')"
     case "${os}" in
         "ubuntu20.04")
             osBaseImage="ubuntu:20.04"
@@ -50,9 +48,10 @@ _setup() {
             ;;
     esac
     imageName="${IMAGE_NAME_OVERRIDE:-"${org}/flow-${os}-${target}"}"
-    imageTag="${commitSha}"
-    if [[ "${tag}" != "NONE" ]]; then
+    if [[ "${tag}" != "" ]]; then
         imageTag="${tag}"
+    else
+        imageTag=$(./etc/DockerTag.sh -dev)
     fi
     case "${target}" in
         "builder" | "master")
@@ -95,43 +94,34 @@ _push() {
         echo "Missing required -password=<PASS> argument"
         _help
     fi
-    docker login --username ${username} --password ${password}
-    if [[ "${tag}" == "NONE" ]]; then
-        tag="latest"
+    if [[ "${target}" != "dev" ]] && [[ "${target}" != "master" ]]; then
+        echo "Target ${target} is not valid candidate for push to Docker Hub." >&2
+        _help
     fi
-    mkdir -p build
-    case "${target}" in
-        "dev" )
-            ./etc/DockerHelper.sh create -os=${os} -ci -target=${target} \
-                2>&1 | tee build/create-${os}-${target}-${tag}.log
-            docker push ${imagePath}
-            ;;
 
-        "master" )
-            # Create dev image needed as a base for builder image
-            ./etc/DockerHelper.sh create -os=${os} -target=dev \
-                2>&1 | tee build/create-${os}-dev-${target}-${tag}.log
-            # Create builder image
-            ./etc/DockerHelper.sh create -os=${os} -target=builder \
-                2>&1 | tee build/create-${os}-${target}-${tag}.log
+    docker login --username "${username}" --password "${password}"
 
-            docker push ${org}/flow-${os}-dev:${commitSha}
+    if [[ "${tag}" == "" ]]; then
+        tag=$(./etc/DockerTag.sh -dev)
+    fi
 
-            docker tag ${org}/flow-${os}-dev:${commitSha} ${org}/flow-${os}-dev:latest
-            docker push ${org}/flow-${os}-dev:latest
+    mkdir -p build
+    ./etc/DockerHelper.sh create -os=${os} -target=dev -tag=${tag} -ci \
+        2>&1 | tee build/create-${os}-dev-${tag}.log
 
-            docker tag ${org}/flow-${os}-builder:${commitSha} ${org}/orfs:${commitSha}
-            docker push ${org}/orfs:${commitSha}
+    docker push "${imageName}:${tag}"
 
-            docker tag ${org}/flow-${os}-builder:${commitSha} ${org}/orfs:${tag}
-            docker push ${org}/orfs:${tag}
-            ;;
+    if [[ "${target}" == "master" ]]; then
+        tag=$(./etc/DockerTag.sh -master)
+        # Create builder image
+        ./etc/DockerHelper.sh create -os=${os} -target=builder -tag=${tag} \
+            2>&1 | tee build/create-${os}-${target}-${tag}.log
 
-        *)
-            echo "Target ${target} is not valid candidate for push to Docker Hub." >&2
-            _help
-            ;;
-    esac
+        docker tag ${org}/flow-${os}-builder:${tag} ${org}/orfs:${tag}
+        docker push ${org}/orfs:${tag}
+        docker tag ${org}/flow-${os}-builder:${tag} ${org}/orfs:${tag}
+        docker push ${org}/orfs:${tag}
+    fi
 }
 
 #
@@ -161,7 +151,7 @@ fi
 os="ubuntu22.04"
 target="dev"
 numThreads="-1"
-tag="NONE"
+tag=""
 options=""
 
 while [ "$#" -gt 0 ]; do
diff --git a/etc/DockerTag.sh b/etc/DockerTag.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+cd $(dirname $(realpath $0))/../
+
+if [[ "$@" == "-dev" ]]; then
+    file_list=(
+        "./build_openroad.sh"
+        "./docker/Dockerfile.builder"
+        "./docker/Dockerfile.dev"
+        "./etc/DependencyInstaller.sh"
+        "./etc/DockerHelper.sh"
+        "./tools/OpenROAD/docker/Dockerfile.builder"
+        "./tools/OpenROAD/docker/Dockerfile.dev"
+        "./tools/OpenROAD/docker/Dockerfile.binary"
+        "./tools/OpenROAD/etc/DependencyInstaller.sh"
+    )
+    cat "${file_list[@]}" | sha256sum | awk '{print substr($1, 1, 6)}'
+elif [[ "$@" == "-master" ]]; then
+    git describe
+else
+    echo "Usage: $0 {-dev|-master}"
+    exit 1
+fi
diff --git a/flow/util/genReport.py b/flow/util/genReport.py
@@ -194,25 +194,30 @@ def get_summary(status, text):
     '''
     TODO: docs
     '''
-    text += '-' * 30 + '\n'
-    text += ' ' * 7 + f"{status} designs\n"
-    text += '-' * 30 + '\n'
+    tmp = ''
     for name, data in design_list.items():
         if data['status'] == status:
             content = gen_report(name, data)
             if content != '':
-                text += content + '\n'
+                tmp += content + '\n'
                 # add empty line for readability
                 if args.verbose >= 2 or status == STATUS_RED:
-                    text += '\n'
+                    tmp += '\n'
+    if tmp != '':
+        text += '-' * 30 + '\n'
+        text += ' ' * 7 + f"{status} designs\n"
+        text += '-' * 30 + '\n'
+        text += tmp
+
     return text
 
 
 def write_summary():
     '''
     Write summary and individual report files with data on design_list
     '''
-    summary = '=' * 60 + '\n'
+    summary = '\n'
+    summary += '=' * 60 + '\n'
     summary += ' ' * 26 + 'SUMMARY\n'
     summary += '=' * 60 + '\n'
     summary += f"\nNumber of designs: {len(design_list.keys())}\n\n"
diff --git a/jenkins/public_tests_all.Jenkinsfile b/jenkins/public_tests_all.Jenkinsfile
@@ -1,68 +1,29 @@
-@Library('utils@main') _
+@Library('utils@orfs-v2.0.1') _
 
-node {
+node('ubuntu22') {
 
-    properties([
-            copyArtifactPermission('${JOB_NAME},'+env.BRANCH_NAME),
-    ]);
+    properties([copyArtifactPermission('${JOB_NAME},'+env.BRANCH_NAME)]);
 
     stage('Checkout') {
         checkout scm;
     }
 
-    def commitHash = "none";
-    def DOCKER_IMAGE_TAG = "latest";
+    def DOCKER_IMAGE;
     stage('Build and Push Docker Image') {
-        if (isDependencyInstallerChanged(env.BRANCH_NAME)) {
-            commitHash = sh(script: 'git rev-parse HEAD', returnStdout: true);
-            commitHash = commitHash.replaceAll(/[^a-zA-Z0-9-]/, '');
-            DOCKER_IMAGE_TAG = pushCIImage(env.BRANCH_NAME, commitHash);
-        }
+        DOCKER_IMAGE = dockerPush('ubuntu22.04', 'orfs');
+        echo "Docker image is $DOCKER_IMAGE";
     }
-    def DOCKER_IMAGE = "openroad/flow-ubuntu22.04-dev:${DOCKER_IMAGE_TAG}";
 
-    docker.image("${DOCKER_IMAGE}").inside('--user=root --privileged -v /var/run/docker.sock:/var/run/docker.sock') {
-        stage('Build ORFS and Stash bins') {
-            sh "git config --system --add safe.directory '*'";
-            localBuild();
-        }
+    stage('Build ORFS and Stash bins') {
+        buildBins(DOCKER_IMAGE);
     }
 
     stage('Run Tests') {
-        Map tasks = [failFast: false];
-        def test_slugs = getTestSlugs("all");
-        for (test in test_slugs) {
-            def currentSlug = test; // copy needed to correctly pass args to runTests
-            tasks["${test}"] = {
-                node {
-                    catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
-                        docker.image("${DOCKER_IMAGE}").inside('--user=root --privileged -v /var/run/docker.sock:/var/run/docker.sock') {
-                            sh "git config --system --add safe.directory '*'";
-                            checkout scm;
-                            runTests(currentSlug);
-                        }
-                    }
-                }
-            }
-        }
-        parallel(tasks);
+        runTests(DOCKER_IMAGE, 'pr');
     }
 
-    docker.image("${DOCKER_IMAGE}").inside('--user=root --privileged -v /var/run/docker.sock:/var/run/docker.sock') {
-        sh "git config --system --add safe.directory '*'";
-        stage('Report Summary') {
-            generateReportShortSummary();
-        }
-        stage("Report HTML Table") {
-            generateReportHtmlTable();
-        }
-        stage('Upload Metadata') {
-            uploadMetadata(env.BRANCH_NAME, commitHash);
-        }
-        stage('Send Report') {
-            def COMMIT_AUTHOR_EMAIL = sh(script: "git --no-pager show -s --format='%ae'", returnStdout: true).trim();
-            sendEmail(env.BRANCH_NAME, COMMIT_AUTHOR_EMAIL);
-        }
+    stage ('Cleanup and Reporting') {
+        finalReport(DOCKER_IMAGE);
     }
 
 }
diff --git a/jenkins/public_tests_small.Jenkinsfile b/jenkins/public_tests_small.Jenkinsfile
