diff --git a/bootstrap.py b/bootstrap.py index 04e30a2249..5141013021 100644 --- a/bootstrap.py +++ b/bootstrap.py @@ -132,7 +132,7 @@ def update(operation, verbose=None, upgrade=False, offline=False, optional_requi # We must install wheel first to eliminate a bunch of scary looking # errors at first install. # TODO Look towards fixing the packaging so that it works with 0.31 - pip('install', ['wheel==0.30'], verbose, True, offline=offline) + pip('install', ['wheel~=0.30'], verbose, True, offline=offline) # Build option_requirements separately to pass install options build_option = '--build-option' if wheeling else '--install-option' diff --git a/ci-integration/run-test-docker.sh b/ci-integration/run-test-docker.sh index 94c4114373..8c9bfc41d1 100755 --- a/ci-integration/run-test-docker.sh +++ b/ci-integration/run-test-docker.sh @@ -1,4 +1,6 @@ #!/bin/bash +# set -x # log all shell commands for debugging. +set -e # fail if any command errors without being caught with an || or an 'if'. # The following script builds a volttron test image and then # runs each of the test modules inside a docker container based @@ -17,6 +19,7 @@ fi export FAST_FAIL=${FAST_FAIL:-true} +pip list echo "RUNNING $NUM_PROCESSES PARALLEL PROCESSESS AT A TIME" echo "FAST_FAIL IS $FAST_FAIL" @@ -42,10 +45,12 @@ ignoredirs=(services/core/DNP3Agent services/core/IEEE2030_5Agent services/core/ # failed. HAS_FAILED=0 +set +e # allow this script to handle errors. + # method to push a test module into the queue push_test(){ testpath="$1" - testqueue+=($testpath) + testqueue+=("$testpath") } # method to get a test module out of the queue @@ -60,15 +65,15 @@ pop_test(){ # global state arrays. run_test(){ # bash /home/osboxes/repos/volttron-rabbitmq/ci-integration/sleep_ten.sh & - local filename=$1; + local filename="$1" echo "Running test module $filename" - base_filename=`basename $filename` + base_filename="$(basename "$filename")" # Start the docker run module. - docker run -e "IGNORE_ENV_CHECK=1" --name $base_filename \ - -t volttron_test_image pytest $filename > "$base_filename.result.txt" 2>&1 & + docker run -e "IGNORE_ENV_CHECK=1" --name "$base_filename" \ + -t volttron_test_image pytest "$filename" > "$base_filename.result.txt" 2>&1 & runningprocs+=($!) outputfiles+=("$base_filename.result.txt") - containernames+=($base_filename) + containernames+=("$base_filename") sleep 0.5 } @@ -77,9 +82,9 @@ run_test(){ # and failed tests are found. exit_cleanly(){ echo "Cleaning up test containers before exiting!" - for container in ${containernames[@]}; do - docker stop $container - docker container rm $container + for container in "${containernames[@]}"; do + docker stop "$container" + docker container rm "$container" done docker system prune --force exit 1 @@ -94,7 +99,7 @@ process_pid(){ # is still executing. If it is not then we need to see what the # exit code was of the container. if [[ ! -e "/proc/${pid}" ]]; then - exitcode=$(docker inspect ${containernames[$index]} --format='{{.State.ExitCode}}') + exitcode=$(docker inspect "${containernames[$index]}" --format='{{.State.ExitCode}}') #echo "Exit code is ${exitcode}" # Exit code 5 is if there are no tests within the file so we filter that out @@ -106,7 +111,7 @@ process_pid(){ HAS_FAILED=1 #echo "FAST_FAIL is ${FAST_FAIL} if its 0 should start clean exit procedure." if [[ ${FAST_FAIL} -eq 0 && -n ${CI} ]]; then - docker logs ${containernames[$index]} + docker logs "${containernames[$index]}" fi if [[ ${FAST_FAIL} -eq 0 ]]; then echo "Exiting cleanly now!" @@ -116,36 +121,36 @@ process_pid(){ else # process passed so cleanup the result file. echo "module ${containernames[$index]} PASSED removing: ${outputfiles[$index]}" - rm ${outputfiles[$index]} + rm "${outputfiles[$index]}" fi # Clean up the test container now that this process is done. - docker container rm ${containernames[$index]} &>/dev/null + docker container rm "${containernames[$index]}" &>/dev/null # Remove pid from the array of running procs. - runningprocs=( ${runningprocs[@]:0:$index} ${runningprocs[@]:$((index + 1))} ) - outputfiles=( ${outputfiles[@]:0:$index} ${outputfiles[@]:$((index + 1))} ) - containernames=( ${containernames[@]:0:$index} ${containernames[@]:$((index + 1))} ) + runningprocs=( "${runningprocs[@]:0:$index}" "${runningprocs[@]:$((index + 1))}" ) + outputfiles=( "${outputfiles[@]:0:$index}" "${outputfiles[@]:$((index + 1))}" ) + containernames=( "${containernames[@]:0:$index}" "${containernames[@]:$((index + 1))}" ) fi i=$(( i+1 )) } #LOOP through set of directories and run bunch of test files in parallel -for dir in ${testdirs[@]} +for dir in "${testdirs[@]}" do for file in $( find $dir -type f -name "test*.py" -o -name "*test.py" ! -name "*conftest.py" ) do - echo $file; + echo "$file"; ignore=0 - for pattern in ${ignoredirs[@]}; do + for pattern in "${ignoredirs[@]}"; do if [[ $file == *"$pattern"* ]]; then - echo $file "IGNORED" + echo "$file IGNORED" ignore=1 break fi done if [[ $ignore == 0 ]]; then - push_test $file; + push_test "$file"; fi done done @@ -159,7 +164,7 @@ while [[ ${#testqueue[@]} -gt 0 ]]; do while [[ ${#runningprocs[@]} -lt ${NUM_PROCESSES} && ${#testqueue[@]} -gt 0 ]]; do # pop the front of the queue into the next_test variable pop_test - run_test ${next_test} + run_test "${next_test}" done i=0 diff --git a/ci-integration/virtualization/Dockerfile b/ci-integration/virtualization/Dockerfile index ef64201d57..7cc7e7c594 100644 --- a/ci-integration/virtualization/Dockerfile +++ b/ci-integration/virtualization/Dockerfile @@ -1,23 +1,19 @@ ARG image_user=laroque ARG image_repo=volttron-docker -ARG image_tag=deps-master-amd64 - +ARG image_tag=deps-feature_python3-amd64 + +# Note: the image should include python3 and volttron's dependencies. FROM ${image_user}/${image_repo}:${image_tag} as volttron_deps # Note I couldn't get variable expansion on the chown argument to work here # so must hard code the user. Note this is a feature request for docker # https://github.com/moby/moby/issues/35018 +# Now resolved: https://github.com/moby/moby/issues/35018#event-2301021020 COPY --chown=volttron:volttron . ${VOLTTRON_ROOT} USER $VOLTTRON_USER -# The following lines ar no longer necesary because of the copy command above. -#WORKDIR /code -#RUN git clone https://github.com/VOLTTRON/volttron -b ${VOLTTRON_GIT_BRANCH} - WORKDIR /code/volttron -#RUN pip install --user -r requirements.txt -RUN pip install -e . --user ############################################ # RABBITMQ SPECIFIC INSTALLATION @@ -38,7 +34,7 @@ RUN set -eux \ && wget -P $VOLTTRON_USER_HOME https://github.com/rabbitmq/rabbitmq-server/releases/download/v3.7.7/rabbitmq-server-generic-unix-3.7.7.tar.xz \ && tar -xf $VOLTTRON_USER_HOME/rabbitmq-server-generic-unix-3.7.7.tar.xz --directory $RMQ_ROOT \ && $RMQ_HOME/sbin/rabbitmq-plugins enable rabbitmq_management rabbitmq_federation rabbitmq_federation_management rabbitmq_shovel rabbitmq_shovel_management rabbitmq_auth_mechanism_ssl rabbitmq_trust_store -RUN pip install gevent-pika --user +RUN python3 -m pip install gevent-pika --user ############################################ diff --git a/ci-integration/virtualization/Dockerfile.testing b/ci-integration/virtualization/Dockerfile.testing index 9b89e33067..c1e87390ef 100644 --- a/ci-integration/virtualization/Dockerfile.testing +++ b/ci-integration/virtualization/Dockerfile.testing @@ -1,11 +1,9 @@ FROM volttron_test_base as test_core WORKDIR ${VOLTTRON_ROOT} -#COPY ./ci-integration/virtualization/requirements_test.txt ./requirements_test.txt USER $VOLTTRON_USER RUN pwd -#RUN ls -a -RUN pip install --user -r ./ci-integration/virtualization/requirements_test.txt +RUN python3 -m pip install --user -r ./ci-integration/virtualization/requirements_test.txt RUN echo "export VOLTTRON_ROOT=${VOLTTRON_ROOT}" > /home/volttron/.bashrc && \ echo "export VOLTTRON_USER_HOME=${VOLTTRON_USER_HOME}" >> /home/volttron/.bashrc && \ @@ -14,4 +12,3 @@ RUN echo "export VOLTTRON_ROOT=${VOLTTRON_ROOT}" > /home/volttron/.bashrc && \ USER root WORKDIR ${VOLTTRON_ROOT} ENTRYPOINT ["/startup/entrypoint.sh"] -#CMD ["/startup/bootstart.sh"] diff --git a/ci-integration/virtualization/core/bootstart.sh b/ci-integration/virtualization/core/bootstart.sh index 9d256aadf7..7c7e975adc 100644 --- a/ci-integration/virtualization/core/bootstart.sh +++ b/ci-integration/virtualization/core/bootstart.sh @@ -9,6 +9,8 @@ echo "Right before setup-platform.py is called I am calling printenv" printenv +pip list +pip install --user wheel==0.30 #python /startup/setup-platform.py PID=$? echo "PID WAS $PID"