Merge remote-tracking branch 'upstream/dev' into deps/pyyaml

Author: auphelia

docker/jenkins/Jenkinsfile

@@ -93,7 +93,7 @@ pipeline {
                 cleanPreviousBuildFiles(env.FINN_HOST_BUILD_DIR)
 
                 // Pass in the marker to run with pytest and the XML test results filename
-                runDockerPytestWithMarker("fpgadataflow", "${env.TEST_NAME}", "--cov --cov-report=html:coverage_fpgadataflow")
+                runDockerPytestWithMarker("fpgadataflow", "${env.TEST_NAME}", "--cov --cov-report=html:coverage_fpgadataflow -n ${env.NUM_PYTEST_WORKERS} --dist worksteal")
 
                 // Stash the test results file(s)
                 stash name: env.TEST_NAME, includes: "${env.TEST_NAME}.xml,${env.TEST_NAME}.html"
@@ -324,21 +324,17 @@ void runDockerPytestWithMarker(String marker, String testResultsFilename, String
   sh """./run-docker.sh python -m pytest -m ${marker} --junitxml=${testResultsFilename}.xml --html=${testResultsFilename}.html --self-contained-html ${additionalOptions}"""
 }
 
-def findBoardBuildFiles(String searchDir, String dirToFind) {
-  def result = sh(script: "find $searchDir -type d -name \"$dirToFind*\"", returnStdout: true).trim()
-  if (result.empty) {
-      error "Directory containing '$dirToFind' not found."
-  }
-	return result
-}
-
 void findCopyZip(String board, String findDir, String copyDir) {
-  def buildDir = findBoardBuildFiles(findDir, "hw_deployment_${board}")
-  sh "cp -r ${buildDir}/${board} ${copyDir}/"
-  dir(copyDir) {
-    sh "zip -r ${board}.zip ${board}/"
-    sh "mkdir -p ${env.ARTIFACT_DIR}/${copyDir}/"
-    sh "cp ${board}.zip ${env.ARTIFACT_DIR}/${copyDir}/"
+  sh "mkdir -p ${copyDir}"
+  try {
+    sh "cp -r ${findDir}/hw_deployment_*/${board} ${copyDir}/"
+    dir(copyDir) {
+      sh "zip -r ${board}.zip ${board}/"
+      sh "mkdir -p ${env.ARTIFACT_DIR}/${copyDir}/"
+      sh "cp ${board}.zip ${env.ARTIFACT_DIR}/${copyDir}/"
+    }
+  } catch (err) {
+    error "No ${board} hw_deployment_* build artifacts found in ${findDir}"
   }
 }
 

notebooks/end2end_example/bnn-pynq/cnv_end2end_example.ipynb

@@ -484,8 +484,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from shutil import copy\n",
-    "from distutils.dir_util import copy_tree\n",
+    "from shutil import copy, copytree\n",
     "\n",
     "# create directory for deployment files\n",
     "deployment_dir = make_build_dir(prefix=\"pynq_deployment_\")\n",
@@ -503,7 +502,7 @@
     "\n",
     "# driver.py and python libraries\n",
     "pynq_driver_dir = model.get_metadata_prop(\"pynq_driver_dir\")\n",
-    "copy_tree(pynq_driver_dir, deployment_dir)"
+    "copytree(pynq_driver_dir, deployment_dir, dirs_exist_ok=True)"
    ]
   },
   {

notebooks/end2end_example/bnn-pynq/tfc_end2end_example.ipynb

@@ -895,8 +895,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from shutil import copy\n",
-    "from distutils.dir_util import copy_tree\n",
+    "from shutil import copy, copytree\n",
     "\n",
     "# create directory for deployment files\n",
     "deployment_dir = make_build_dir(prefix=\"pynq_deployment_\")\n",
@@ -914,7 +913,7 @@
     "\n",
     "# driver.py and python libraries\n",
     "pynq_driver_dir = model.get_metadata_prop(\"pynq_driver_dir\")\n",
-    "copy_tree(pynq_driver_dir, deployment_dir)"
+    "copytree(pynq_driver_dir, deployment_dir, dirs_exist_ok=True)"
    ]
   },
   {

run-docker.sh

@@ -88,7 +88,7 @@ SCRIPTPATH=$(dirname "$SCRIPT")
 : ${PLATFORM_REPO_PATHS="/opt/xilinx/platforms"}
 : ${XRT_DEB_VERSION="xrt_202220.2.14.354_22.04-amd64-xrt"}
 : ${FINN_HOST_BUILD_DIR="/tmp/$DOCKER_INST_NAME"}
-: ${FINN_DOCKER_TAG="xilinx/finn:$(git describe --always --tags --dirty).$XRT_DEB_VERSION"}
+: ${FINN_DOCKER_TAG="xilinx/finn:$(OLD_PWD=$(pwd); cd $SCRIPTPATH; git describe --always --tags --dirty; cd $OLD_PWD).$XRT_DEB_VERSION"}
 : ${FINN_DOCKER_PREBUILT="0"}
 : ${FINN_DOCKER_RUN_AS_ROOT="0"}
 : ${FINN_DOCKER_GPU="$(docker info | grep nvidia | wc -m)"}

src/finn/builder/build_dataflow_steps.py

@@ -33,7 +33,6 @@
 import shutil
 import warnings
 from copy import deepcopy
-from distutils.dir_util import copy_tree
 from functools import partial
 from qonnx.core.modelwrapper import ModelWrapper
 from qonnx.custom_op.registry import getCustomOp
@@ -656,7 +655,9 @@ def step_create_stitched_ip(model: ModelWrapper, cfg: DataflowBuildConfig):
             )
         )
         # TODO copy all ip sources into output dir? as zip?
-        copy_tree(model.get_metadata_prop("vivado_stitch_proj"), stitched_ip_dir)
+        shutil.copytree(
+            model.get_metadata_prop("vivado_stitch_proj"), stitched_ip_dir, dirs_exist_ok=True
+        )
         print("Vivado stitched IP written into " + stitched_ip_dir)
     if VerificationStepType.STITCHED_IP_RTLSIM in cfg._resolve_verification_steps():
         # prepare ip-stitched rtlsim
@@ -761,7 +762,7 @@ def step_make_pynq_driver(model: ModelWrapper, cfg: DataflowBuildConfig):
     if DataflowOutputType.PYNQ_DRIVER in cfg.generate_outputs:
         driver_dir = cfg.output_dir + "/driver"
         model = model.transform(MakePYNQDriver(cfg._resolve_driver_platform()))
-        copy_tree(model.get_metadata_prop("pynq_driver_dir"), driver_dir)
+        shutil.copytree(model.get_metadata_prop("pynq_driver_dir"), driver_dir, dirs_exist_ok=True)
         print("PYNQ Python driver written into " + driver_dir)
     return model
 
@@ -862,8 +863,8 @@ def step_deployment_package(model: ModelWrapper, cfg: DataflowBuildConfig):
         bitfile_dir = cfg.output_dir + "/bitfile"
         driver_dir = cfg.output_dir + "/driver"
         os.makedirs(deploy_dir, exist_ok=True)
-        copy_tree(bitfile_dir, deploy_dir + "/bitfile")
-        copy_tree(driver_dir, deploy_dir + "/driver")
+        shutil.copytree(bitfile_dir, deploy_dir + "/bitfile", dirs_exist_ok=True)
+        shutil.copytree(driver_dir, deploy_dir + "/driver", dirs_exist_ok=True)
     return model
 
 

src/finn/custom_op/fpgadataflow/concat.py

@@ -29,7 +29,6 @@
 
 import numpy as np
 from qonnx.core.datatype import DataType
-from qonnx.util.basic import roundup_to_integer_multiple
 
 from finn.custom_op.fpgadataflow.hwcustomop import HWCustomOp
 
@@ -134,10 +133,6 @@ def execute_node(self, context, graph):
         result = np.concatenate(inp_values, axis=-1)
         context[node.output[0]] = result
 
-    def get_instream_width_padded(self, ind=0):
-        in_width = self.get_instream_width(ind)
-        return roundup_to_integer_multiple(in_width, 8)
-
     def get_verilog_top_module_intf_names(self):
         intf_names = super().get_verilog_top_module_intf_names()
         n_inputs = self.get_n_inputs()

tests/end2end/test_end2end_bnn_pynq.py

@@ -40,7 +40,6 @@
 import warnings
 from brevitas.export import export_qonnx
 from dataset_loading import cifar, mnist
-from distutils.dir_util import copy_tree
 from qonnx.core.datatype import DataType
 from qonnx.core.modelwrapper import ModelWrapper
 from qonnx.custom_op.registry import getCustomOp
@@ -59,7 +58,7 @@
 from qonnx.transformation.lower_convs_to_matmul import LowerConvsToMatMul
 from qonnx.transformation.merge_onnx_models import MergeONNXModels
 from qonnx.util.cleanup import cleanup as qonnx_cleanup
-from shutil import copy
+from shutil import copy, copytree
 
 import finn.transformation.fpgadataflow.convert_to_hw_layers as to_hw
 import finn.transformation.streamline.absorb as absorb
@@ -357,7 +356,7 @@ def deploy_based_on_board(model, model_title, topology, wbits, abits, board):
 
     # driver.py and python libraries
     pynq_driver_dir = model.get_metadata_prop("pynq_driver_dir")
-    copy_tree(pynq_driver_dir, deployment_dir)
+    copytree(pynq_driver_dir, deployment_dir, dirs_exist_ok=True)
     model.set_metadata_prop("pynq_deploy_dir", deployment_dir)