diff --git a/docker/jenkins/Jenkinsfile b/docker/jenkins/Jenkinsfile index 6d51fffd64..cca3436363 100644 --- a/docker/jenkins/Jenkinsfile +++ b/docker/jenkins/Jenkinsfile @@ -93,7 +93,7 @@ pipeline { cleanPreviousBuildFiles(env.FINN_HOST_BUILD_DIR) // Pass in the marker to run with pytest and the XML test results filename - runDockerPytestWithMarker("fpgadataflow", "${env.TEST_NAME}", "--cov --cov-report=html:coverage_fpgadataflow") + runDockerPytestWithMarker("fpgadataflow", "${env.TEST_NAME}", "--cov --cov-report=html:coverage_fpgadataflow -n ${env.NUM_PYTEST_WORKERS} --dist worksteal") // Stash the test results file(s) stash name: env.TEST_NAME, includes: "${env.TEST_NAME}.xml,${env.TEST_NAME}.html" @@ -324,21 +324,17 @@ void runDockerPytestWithMarker(String marker, String testResultsFilename, String sh """./run-docker.sh python -m pytest -m ${marker} --junitxml=${testResultsFilename}.xml --html=${testResultsFilename}.html --self-contained-html ${additionalOptions}""" } -def findBoardBuildFiles(String searchDir, String dirToFind) { - def result = sh(script: "find $searchDir -type d -name \"$dirToFind*\"", returnStdout: true).trim() - if (result.empty) { - error "Directory containing '$dirToFind' not found." - } - return result -} - void findCopyZip(String board, String findDir, String copyDir) { - def buildDir = findBoardBuildFiles(findDir, "hw_deployment_${board}") - sh "cp -r ${buildDir}/${board} ${copyDir}/" - dir(copyDir) { - sh "zip -r ${board}.zip ${board}/" - sh "mkdir -p ${env.ARTIFACT_DIR}/${copyDir}/" - sh "cp ${board}.zip ${env.ARTIFACT_DIR}/${copyDir}/" + sh "mkdir -p ${copyDir}" + try { + sh "cp -r ${findDir}/hw_deployment_*/${board} ${copyDir}/" + dir(copyDir) { + sh "zip -r ${board}.zip ${board}/" + sh "mkdir -p ${env.ARTIFACT_DIR}/${copyDir}/" + sh "cp ${board}.zip ${env.ARTIFACT_DIR}/${copyDir}/" + } + } catch (err) { + error "No ${board} hw_deployment_* build artifacts found in ${findDir}" } } diff --git a/notebooks/end2end_example/bnn-pynq/cnv_end2end_example.ipynb b/notebooks/end2end_example/bnn-pynq/cnv_end2end_example.ipynb index 8b8cff8ee9..507b1022e6 100644 --- a/notebooks/end2end_example/bnn-pynq/cnv_end2end_example.ipynb +++ b/notebooks/end2end_example/bnn-pynq/cnv_end2end_example.ipynb @@ -484,8 +484,7 @@ "metadata": {}, "outputs": [], "source": [ - "from shutil import copy\n", - "from distutils.dir_util import copy_tree\n", + "from shutil import copy, copytree\n", "\n", "# create directory for deployment files\n", "deployment_dir = make_build_dir(prefix=\"pynq_deployment_\")\n", @@ -503,7 +502,7 @@ "\n", "# driver.py and python libraries\n", "pynq_driver_dir = model.get_metadata_prop(\"pynq_driver_dir\")\n", - "copy_tree(pynq_driver_dir, deployment_dir)" + "copytree(pynq_driver_dir, deployment_dir, dirs_exist_ok=True)" ] }, { diff --git a/notebooks/end2end_example/bnn-pynq/tfc_end2end_example.ipynb b/notebooks/end2end_example/bnn-pynq/tfc_end2end_example.ipynb index 675ba23d2d..bb5e357b66 100644 --- a/notebooks/end2end_example/bnn-pynq/tfc_end2end_example.ipynb +++ b/notebooks/end2end_example/bnn-pynq/tfc_end2end_example.ipynb @@ -895,8 +895,7 @@ "metadata": {}, "outputs": [], "source": [ - "from shutil import copy\n", - "from distutils.dir_util import copy_tree\n", + "from shutil import copy, copytree\n", "\n", "# create directory for deployment files\n", "deployment_dir = make_build_dir(prefix=\"pynq_deployment_\")\n", @@ -914,7 +913,7 @@ "\n", "# driver.py and python libraries\n", "pynq_driver_dir = model.get_metadata_prop(\"pynq_driver_dir\")\n", - "copy_tree(pynq_driver_dir, deployment_dir)" + "copytree(pynq_driver_dir, deployment_dir, dirs_exist_ok=True)" ] }, { diff --git a/run-docker.sh b/run-docker.sh index 0a91ebc422..4047205e57 100755 --- a/run-docker.sh +++ b/run-docker.sh @@ -88,7 +88,7 @@ SCRIPTPATH=$(dirname "$SCRIPT") : ${PLATFORM_REPO_PATHS="/opt/xilinx/platforms"} : ${XRT_DEB_VERSION="xrt_202220.2.14.354_22.04-amd64-xrt"} : ${FINN_HOST_BUILD_DIR="/tmp/$DOCKER_INST_NAME"} -: ${FINN_DOCKER_TAG="xilinx/finn:$(git describe --always --tags --dirty).$XRT_DEB_VERSION"} +: ${FINN_DOCKER_TAG="xilinx/finn:$(OLD_PWD=$(pwd); cd $SCRIPTPATH; git describe --always --tags --dirty; cd $OLD_PWD).$XRT_DEB_VERSION"} : ${FINN_DOCKER_PREBUILT="0"} : ${FINN_DOCKER_RUN_AS_ROOT="0"} : ${FINN_DOCKER_GPU="$(docker info | grep nvidia | wc -m)"} diff --git a/src/finn/builder/build_dataflow_steps.py b/src/finn/builder/build_dataflow_steps.py index ab2280554c..5163b2dbdb 100644 --- a/src/finn/builder/build_dataflow_steps.py +++ b/src/finn/builder/build_dataflow_steps.py @@ -33,7 +33,6 @@ import shutil import warnings from copy import deepcopy -from distutils.dir_util import copy_tree from functools import partial from qonnx.core.modelwrapper import ModelWrapper from qonnx.custom_op.registry import getCustomOp @@ -656,7 +655,9 @@ def step_create_stitched_ip(model: ModelWrapper, cfg: DataflowBuildConfig): ) ) # TODO copy all ip sources into output dir? as zip? - copy_tree(model.get_metadata_prop("vivado_stitch_proj"), stitched_ip_dir) + shutil.copytree( + model.get_metadata_prop("vivado_stitch_proj"), stitched_ip_dir, dirs_exist_ok=True + ) print("Vivado stitched IP written into " + stitched_ip_dir) if VerificationStepType.STITCHED_IP_RTLSIM in cfg._resolve_verification_steps(): # prepare ip-stitched rtlsim @@ -761,7 +762,7 @@ def step_make_pynq_driver(model: ModelWrapper, cfg: DataflowBuildConfig): if DataflowOutputType.PYNQ_DRIVER in cfg.generate_outputs: driver_dir = cfg.output_dir + "/driver" model = model.transform(MakePYNQDriver(cfg._resolve_driver_platform())) - copy_tree(model.get_metadata_prop("pynq_driver_dir"), driver_dir) + shutil.copytree(model.get_metadata_prop("pynq_driver_dir"), driver_dir, dirs_exist_ok=True) print("PYNQ Python driver written into " + driver_dir) return model @@ -862,8 +863,8 @@ def step_deployment_package(model: ModelWrapper, cfg: DataflowBuildConfig): bitfile_dir = cfg.output_dir + "/bitfile" driver_dir = cfg.output_dir + "/driver" os.makedirs(deploy_dir, exist_ok=True) - copy_tree(bitfile_dir, deploy_dir + "/bitfile") - copy_tree(driver_dir, deploy_dir + "/driver") + shutil.copytree(bitfile_dir, deploy_dir + "/bitfile", dirs_exist_ok=True) + shutil.copytree(driver_dir, deploy_dir + "/driver", dirs_exist_ok=True) return model diff --git a/src/finn/custom_op/fpgadataflow/concat.py b/src/finn/custom_op/fpgadataflow/concat.py index 210b6b7fdd..985ac83ea6 100644 --- a/src/finn/custom_op/fpgadataflow/concat.py +++ b/src/finn/custom_op/fpgadataflow/concat.py @@ -29,7 +29,6 @@ import numpy as np from qonnx.core.datatype import DataType -from qonnx.util.basic import roundup_to_integer_multiple from finn.custom_op.fpgadataflow.hwcustomop import HWCustomOp @@ -134,10 +133,6 @@ def execute_node(self, context, graph): result = np.concatenate(inp_values, axis=-1) context[node.output[0]] = result - def get_instream_width_padded(self, ind=0): - in_width = self.get_instream_width(ind) - return roundup_to_integer_multiple(in_width, 8) - def get_verilog_top_module_intf_names(self): intf_names = super().get_verilog_top_module_intf_names() n_inputs = self.get_n_inputs() diff --git a/tests/end2end/test_end2end_bnn_pynq.py b/tests/end2end/test_end2end_bnn_pynq.py index 0d3418624a..688423e581 100644 --- a/tests/end2end/test_end2end_bnn_pynq.py +++ b/tests/end2end/test_end2end_bnn_pynq.py @@ -40,7 +40,6 @@ import warnings from brevitas.export import export_qonnx from dataset_loading import cifar, mnist -from distutils.dir_util import copy_tree from qonnx.core.datatype import DataType from qonnx.core.modelwrapper import ModelWrapper from qonnx.custom_op.registry import getCustomOp @@ -59,7 +58,7 @@ from qonnx.transformation.lower_convs_to_matmul import LowerConvsToMatMul from qonnx.transformation.merge_onnx_models import MergeONNXModels from qonnx.util.cleanup import cleanup as qonnx_cleanup -from shutil import copy +from shutil import copy, copytree import finn.transformation.fpgadataflow.convert_to_hw_layers as to_hw import finn.transformation.streamline.absorb as absorb @@ -357,7 +356,7 @@ def deploy_based_on_board(model, model_title, topology, wbits, abits, board): # driver.py and python libraries pynq_driver_dir = model.get_metadata_prop("pynq_driver_dir") - copy_tree(pynq_driver_dir, deployment_dir) + copytree(pynq_driver_dir, deployment_dir, dirs_exist_ok=True) model.set_metadata_prop("pynq_deploy_dir", deployment_dir)