From ae786f599517ca35f5c346913e60db1efd73d5f0 Mon Sep 17 00:00:00 2001 From: Benjamin Hugo Date: Sat, 11 Jun 2022 10:12:07 +0200 Subject: [PATCH] Bump version (#825) * Bump version * Fix installable ranges for upstream packages to match testing environment * add Python 3.8 as default supported version (Ubuntu 20.04) --- Jenkinsfile.sh | 7 +- Dockerfile => docker.1804 | 0 docker.2004 | 262 ++++++++++++++++++++++++++++++++++++++ pyproject.toml | 2 +- setup.py | 82 +++++++----- 5 files changed, 317 insertions(+), 36 deletions(-) rename Dockerfile => docker.1804 (100%) create mode 100644 docker.2004 diff --git a/Jenkinsfile.sh b/Jenkinsfile.sh index d80d4565..fbc7f9d7 100644 --- a/Jenkinsfile.sh +++ b/Jenkinsfile.sh @@ -19,9 +19,12 @@ echo "----------------------------------------------" #build using docker file in directory: cd $PROJECTS_DIR/DDFacet IMAGENAME="ddf" -docker build -t "$IMAGENAME:$BUILD_NUMBER" --no-cache=true . +#check that it still builds on 18.04 +docker build -t "$IMAGENAME:$BUILD_NUMBER" --no-cache=true -f docker.1804 . +#test on 20.04 +docker build -t "$IMAGENAME:$BUILD_NUMBER" --no-cache=true -f docker.2004 . docker run -m 100g --cap-add sys_ptrace \ - --memory-swap=-1 \ + --memory-swap=-1 \ --shm-size=150g \ --rm=true \ --name=$IMAGENAME$BUILD_NUMBER \ diff --git a/Dockerfile b/docker.1804 similarity index 100% rename from Dockerfile rename to docker.1804 diff --git a/docker.2004 b/docker.2004 new file mode 100644 index 00000000..d7182d74 --- /dev/null +++ b/docker.2004 @@ -0,0 +1,262 @@ +FROM ubuntu:focal-20220105 + +#Package dependencies +COPY apt.sources.list /etc/apt/sources.list +RUN sed -i 's/bionic/focal/g' /etc/apt/sources.list + +#Setup environment +ENV DDFACET_TEST_DATA_DIR /test_data +ENV DDFACET_TEST_OUTPUT_DIR /test_output + +# Support large mlocks +RUN echo "* - memlock unlimited" > /etc/security/limits.conf +ENV DEBIAN_FRONTEND noninteractive +ENV DEBIAN_PRIORITY critical +ENV GNUCOMPILER 10 +ENV PYTHONVER 3.8 +ENV DEB_SETUP_DEPENDENCIES \ + dpkg-dev \ + g++-$GNUCOMPILER \ + gcc-$GNUCOMPILER \ + libc-dev \ + cmake \ + gfortran-$GNUCOMPILER \ + git \ + wget \ + subversion \ + rsync + +ENV DEB_DEPENCENDIES \ + python3-virtualenv \ + python3-pip \ + libfftw3-dev \ + python3-numpy \ + libfreetype6 \ + libfreetype6-dev \ + libpng-dev \ + pkg-config \ + python3-dev \ + libboost-all-dev \ + libcfitsio-dev \ + libhdf5-dev \ + wcslib-dev \ + libatlas-base-dev \ + liblapack-dev \ + python3-tk \ + libreadline6-dev \ + subversion \ + liblog4cplus-dev \ + libhdf5-dev \ + libncurses5-dev \ + flex \ + bison \ + libbison-dev \ + libqdbm-dev \ + # Reference image generation dependencies + make + +RUN apt-get update +RUN apt-get install -y $DEB_SETUP_DEPENDENCIES +RUN apt-get install -y $DEB_DEPENCENDIES + + +RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-$GNUCOMPILER 100 && \ + update-alternatives --install /usr/bin/cc cc /usr/bin/gcc-$GNUCOMPILER 100 && \ + update-alternatives --install /usr/bin/x86_64-linux-gnu-gcc x86_64-linux-gnu-gcc /usr/bin/gcc-$GNUCOMPILER 100 && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-$GNUCOMPILER 100 && \ + update-alternatives --install /usr/bin/x86_64-linux-gnu-g++ x86_64-linux-gnu-g++ /usr/bin/g++-$GNUCOMPILER 100 && \ + update-alternatives --install /usr/bin/cpp cpp /usr/bin/g++-$GNUCOMPILER 100 && \ + update-alternatives --install /usr/bin/x86_64-linux-gnu-cpp x86_64-linux-gnu-cpp /usr/bin/g++-$GNUCOMPILER 100 && \ + update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-$GNUCOMPILER 100 && \ + update-alternatives --install /usr/bin/x86_64-linux-gnu-gfortran x86_64-linux-gnu-gfortran /usr/bin/gfortran-$GNUCOMPILER 100 + +## All python dependencies go into venv +WORKDIR /opt +RUN virtualenv venv -p python${PYTHONVER} +RUN . /opt/venv/bin/activate && python -m pip install -U pip setuptools wheel +## Current workaround for numpy >=1.20 numerical issues. We will only support numpy < 1.20 +RUN . /opt/venv/bin/activate && python -m pip install -U numpy==1.19.5 + +######################################## +# build SOFA +######################################## +WORKDIR /opt +RUN wget http://www.iausofa.org/2021_0512_C/sofa_c-20210512.tar.gz && \ + tar xvfz sofa_c-20210512.tar.gz && \ + rm sofa_c-20210512.tar.gz && \ + cd /opt/sofa/20210512/c/src && \ + sed -i '52s/.*/INSTALL_DIR = \/usr/' makefile && \ + make -j4 && \ + make install && \ + make test && \ + rm -r /opt/sofa + +################################################################# +## build blitz from source +################################################################# +WORKDIR /opt +RUN wget https://github.com/blitzpp/blitz/archive/1.0.2.tar.gz && \ + tar -xvf 1.0.2.tar.gz && \ + rm 1.0.2.tar.gz && \ + cd /opt/blitz-1.0.2 && \ + grep -rl python . | xargs sed -i "s/python/python${PYTHONVER}/g" && \ + ./configure --enable-shared && \ + make -j16 && \ + make install && \ + ldconfig && \ + rm -r /opt/blitz-1.0.2 + +##################################################################### +## Get CASACORE ephem data +##################################################################### +RUN mkdir -p /usr/share/casacore/data/ +WORKDIR /usr/share/casacore/data/ +RUN rsync -avz rsync://casa-rsync.nrao.edu/casa-data . + +##################################################################### +## BUILD CASACORE AND CASAREST FROM SOURCE +##################################################################### +WORKDIR /opt +RUN wget https://github.com/casacore/casacore/archive/v3.4.0.tar.gz && \ + tar xvf v3.4.0.tar.gz && \ + rm v3.4.0.tar.gz && \ + mkdir casacore-3.4.0/build && \ + cd /opt/casacore-3.4.0/build && \ + cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=Release -DBUILD_DEPRECATED=ON -DBUILD_PYTHON=OFF -DBUILD_PYTHON3=ON ../ && \ + make -j 4 && \ + make install && \ + rm -r /opt/casacore-3.4.0 && \ + cd /opt && \ + wget https://github.com/casacore/casarest/archive/v1.8.0.tar.gz && \ + tar xvf v1.8.0.tar.gz && \ + rm v1.8.0.tar.gz && \ + cd /opt/casarest-1.8.0 && \ + mkdir -p build && \ + cd /opt/casarest-1.8.0/build && \ + cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=Release ../ && \ + make -j 4 && \ + make install && \ + rm -r /opt/casarest-1.8.0 && \ + ldconfig && \ + cd /opt && \ + wget https://github.com/casacore/python-casacore/archive/v3.4.0.tar.gz && \ + tar xvf v3.4.0.tar.gz && \ + rm v3.4.0.tar.gz && \ + cd /opt/python-casacore-3.4.0 && \ + . /opt/venv/bin/activate && python -m pip install . && \ + cd / && \ + python -c "from pyrap.tables import table as tbl" + +##################################################################### +## BUILD MAKEMS FROM SOURCE AND TEST +##################################################################### +WORKDIR /opt +RUN wget https://github.com/ska-sa/makems/archive/v1.5.4.tar.gz && \ + tar xvf v1.5.4.tar.gz && \ + rm v1.5.4.tar.gz && \ + mkdir -p /opt/makems-1.5.4/LOFAR/build/gnu_opt && \ + cd /opt/makems-1.5.4/LOFAR/build/gnu_opt && \ + cmake -DCMAKE_MODULE_PATH:PATH=/opt/makems-1.5.4/LOFAR/CMake \ + -DUSE_LOG4CPLUS=OFF -DBUILD_TESTING=OFF -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/usr ../.. && \ + make -j 16 && \ + make install && \ + cd /opt/makems-1.5.4/test && \ + makems WSRT_makems.cfg && \ + rm -r /opt/makems-1.5.4 + +##################################################################### +## BUILD MEQTREES FROM SOURCE AND TEST +##################################################################### +WORKDIR /opt +# Get MeqTrees universe python packages +WORKDIR /opt + +RUN . /opt/venv/bin/activate && python -m pip install purr owlcat kittens meqtrees-cattery astro-tigger-lsm && \ + wget https://github.com/ska-sa/meqtrees-timba/archive/refs/tags/v1.8.3.tar.gz && \ + tar zxvf v1.8.3.tar.gz && \ + rm v1.8.3.tar.gz && \ + cd meqtrees-timba-1.8.3 && \ + mkdir build && \ + cd build && \ + cmake -DCMAKE_INSTALL_PREFIX=/usr \ + -DCMAKE_BUILD_TYPE=Release \ + -DENABLE_PYTHON_3=ON \ + -DPYTHON_PACKAGES_DIR=/opt/venv/lib/python${PYTHONVER}/site-packages \ + .. && \ + make -j4 && \ + make install && \ + ldconfig && \ + rm -r /opt/meqtrees-timba-1.8.3 && \ + cd /opt && \ + wget https://github.com/ska-sa/pyxis/archive/v1.7.4.3.tar.gz && \ + tar zxvf v1.7.4.3.tar.gz && \ + rm v1.7.4.3.tar.gz && \ + python -m pip install /opt/pyxis-1.7.4.3 nose && \ + cd /opt/pyxis-1.7.4.3/Pyxis/recipes/meqtrees-batch-test && \ + python -m "nose" && \ + rm -r /opt/pyxis-1.7.4.3 + +##################################################################### +## BUILD LOFAR FROM SOURCE +##################################################################### +WORKDIR /opt +RUN wget https://github.com/bennahugo/LOFARBeam/archive/refs/tags/DDF_KMS_20.04.tar.gz && \ + tar zxvf DDF_KMS_20.04.tar.gz && \ + rm DDF_KMS_20.04.tar.gz && \ + cd LOFARBeam-DDF_KMS_20.04 && \ + mkdir -p build/gnucxx11_opt && \ + cd build/gnucxx11_opt && \ + cmake -DCMAKE_INSTALL_PREFIX=/usr \ + -DPYTHON_PACKAGES_DIR=/opt/venv/lib/python${PYTHONVER}/site-packages \ + ../../ && \ + make -j4 && \ + make install && \ + . /opt/venv/bin/activate && python -c "import lofar.stationresponse as lsr" && \ + rm -r /opt/LOFARBeam-DDF_KMS_20.04 + +##################################################################### +## BUILD DDF FROM SOURCE +##################################################################### +#Copy DDFacet and SkyModel into the image +ADD DDFacet /opt/DDFacet/DDFacet +ADD SkyModel /opt/DDFacet/SkyModel +ADD MANIFEST.in /opt/DDFacet/MANIFEST.in +ADD setup.py /opt/DDFacet/setup.py +ADD setup.cfg /opt/DDFacet/setup.cfg +ADD README.rst /opt/DDFacet/README.rst +ADD pyproject.toml /opt/DDFacet/pyproject.toml +ADD .git /opt/DDFacet/.git +ADD .gitignore /opt/DDFacet/.gitignore +ADD .gitmodules /opt/DDFacet/.gitmodules + +RUN cd /opt/DDFacet/ && git submodule update --init --recursive && cd / +# Finally install DDFacet +RUN rm -rf /opt/DDFacet/DDFacet/cbuild +RUN . /opt/venv/bin/activate && python -m pip install "/opt/DDFacet[dft-support,moresane-support,testing-requirements,fits-beam-support,kms-support]" + +# test montblanc install +WORKDIR /tmp +RUN . /opt/venv/bin/activate && python -c "import montblanc" +RUN . /opt/venv/bin/activate && python -m nose /opt/venv/lib/python${PYTHONVER}/site-packages/montblanc/impl/rime/tensorflow/rime_ops/test* + +# ensure bdsf backend still works +RUN . /opt/venv/bin/activate && python -c "import bdsf" + +# Finally Make VENV globally available in this container +ENV LD_LIBRARY_PATH /opt/venv/lib:$LD_LIBRARY_PATH +ENV PATH /opt/venv/bin:$PATH +ENV PYTHONPATH /opt/venv/lib/python${PYTHONVER}/site-packages + +# perform some basic tests +RUN DDF.py --help +RUN MakeMask.py --help +RUN MakeCatalog.py --help +RUN MakeModel.py --help +RUN MaskDicoModel.py --help +RUN ClusterCat.py --help + + +# set as entrypoint - user should be able to run docker run ddftag and get help printed +ENTRYPOINT ["DDF.py"] +CMD ["--help"] diff --git a/pyproject.toml b/pyproject.toml index 91aee4ff..a0ee6cf1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] requires = ["setuptools", "wheel", - "numpy>=1.15.1", # Ubuntu 18.04 + "numpy>=1.15.1,<=1.19.5", "pybind11 >= 2.2.2"] \ No newline at end of file diff --git a/setup.py b/setup.py index 8ecbdcaf..ea136125 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,10 @@ pkg='DDFacet' skymodel_pkg='SkyModel' -__version__ = "0.6.0.0" +__version__ = "0.7.0.0" +# Bump when building for a new release version of Python +MAJ_REQ = 3 +MIN_REQ = 8 build_root=os.path.dirname(__file__) try: @@ -42,6 +45,18 @@ raise ImportError("Pybind11 not installed. Please install C++ binding package pybind11 before running DDFacet install. " "You should not see this message unless you are not running pip install (19.x) -- run pip install!") +# it seems the deprecated python setup.py install does not check the +# version numbers required by the package in the descriptor. Although this is +# not an issue with pip install, this leads to issues with upstream +# packages. +if not (sys.version_info.major == MAJ_REQ and sys.version_info.minor == MIN_REQ): + print("Your Python version '{}' is not officially supported. This package " + "requires version {}".format(sys.version, + "{}.{}".format(MAJ_REQ, + MIN_REQ))) + print("You should not be seeing this message unless you are not using pip " + "install to install this software. Please use pip install") + def backend(compile_options): if compile_options is not None: print("Compiling extension libraries with user defined options: '%s'"%compile_options) @@ -117,38 +132,39 @@ def readme(): return f.read() def requirements(): + # Only upgrade these when testing with a new release version + # this should avoid upstream breakage on released versions requirements = ["nose >= 1.3.7; python_version >= '3'", - "Cython >= 0.25.2; python_version >= '3'", - "numpy >= 1.15.1; python_version >= '3'", - "sharedarray >= 3.2.0; python_version >= '3'", - "Polygon3 >= 3.0.8; python_version >= '3'", - "pyFFTW >= 0.10.4; python_version >= '3'", - "astropy >= 3.0; python_version >= '3'", - "deap >= 1.0.1; python_version >= '3'", - "ptyprocess>=0.5; python_version >= '3'", - "ipdb >= 0.10.3; python_version >= '3'", - "python-casacore >= 3.0.0; python_version >= '3'", - "pyephem >= 3.7.6.0; python_version >= '3'", - "numexpr >= 2.6.2; python_version >= '3'", - "matplotlib >= 2.0.0; python_version >= '3'", - "scipy >= 1.3.3; python_version >= '3'", - "astLib >= 0.8.0; python_version >= '3'", - "psutil >= 5.2.2; python_version >= '3'", - "py-cpuinfo >= 3.2.0; python_version >= '3'", - "tables >= 3.6.0; python_version >= '3'", - "prettytable >= 0.7.2; python_version >= '3'", - "pybind11 >= 2.2.2; python_version >= '3'", - "configparser >= 3.7.1; python_version >= '3'", - "pandas >=0.23.3; python_version >= '3'", - "ruamel.yaml >= 0.15.92; python_version >= '3'", - "pylru >= 1.1.0; python_version >= '3'", - "six >= 1.12.0; python_version >= '3'", - "pybind11 >= 2.2.2; python_version >= '3'", - "dask[array] >= 1.1.0; python_version >= '3'", - "codex-africanus[dask] >= 0.2.10; python_version >= '3'", - "regions", - "pywavelets", - "tqdm" + "Cython >= 0.25.2, <= 0.29.30; python_version >= '3'", + "numpy >= 1.15.1, <= 1.19.5; python_version >= '3'", + "sharedarray >= 3.2.0, <= 3.2.1; python_version >= '3'", + "Polygon3 >= 3.0.8, <= 3.0.9.1; python_version >= '3'", + "pyFFTW >= 0.10.4, <= 0.12.0; python_version >= '3'", + "astropy >= 3.0, <= 4.1; python_version >= '3'", + "deap >= 1.0.1, <= 1.3.1; python_version >= '3'", + "ptyprocess>=0.5, <= 0.7.0; python_version >= '3'", + "ipdb >= 0.10.3, <=0.13.9; python_version >= '3'", + "python-casacore >= 3.0.0, <=3.4.0; python_version >= '3'", + "pyephem >= 3.7.6.0, <=9.99; python_version >= '3'", + "numexpr >= 2.6.2,<=2.8.1; python_version >= '3'", + "matplotlib >= 2.0.0,<=3.3.4; python_version >= '3'", + "scipy >= 1.3.3,<=1.5.4; python_version >= '3'", + "astLib >= 0.8.0,<=0.11.7; python_version >= '3'", + "psutil >= 5.2.2,<=5.9.1; python_version >= '3'", + "py-cpuinfo >= 3.2.0,<=8.0.0; python_version >= '3'", + "tables >= 3.6.0,<=3.7.0; python_version >= '3'", + "prettytable >= 0.7.2,<=2.5.0; python_version >= '3'", + "pybind11 >= 2.2.2,<=2.9.2; python_version >= '3'", + "configparser >= 3.7.1,<=5.2.0; python_version >= '3'", + "pandas >=0.23.3,<=1.1.5; python_version >= '3'", + "ruamel.yaml >= 0.15.92,<=0.17.21; python_version >= '3'", + "pylru >= 1.1.0,<=1.2.1; python_version >= '3'", + "six >= 1.12.0,<=1.16.0; python_version >= '3'", + "dask[array] >= 1.1.0,<=2021.3.0; python_version >= '3'", + "codex-africanus[dask] <= 0.2.10; python_version >= '3'", + "regions <=0.5", + "pywavelets <=1.1.1", + "tqdm<=4.64.0" ] install_requirements = requirements @@ -175,7 +191,7 @@ def requirements(): 'build': custom_build, 'build_ext': custom_build_ext }, - python_requires='>=3.0,<3.7', + python_requires='>=3.0,<{}'.format("{}.{}".format(MAJ_REQ, MIN_REQ+1)), packages=[pkg, skymodel_pkg], install_requires=requirements(), include_package_data=True,