Bump version (#825)

* Bump version * Fix installable ranges for upstream packages to match testing environment * add Python 3.8 as default supported version (Ubuntu 20.04)
saopicc · Jun 11, 2022 · ae786f5 · ae786f5
1 parent 03d7395
commit ae786f5
Show file tree

Hide file tree

Showing 5 changed files with 317 additions and 36 deletions.
diff --git a/Jenkinsfile.sh b/Jenkinsfile.sh
@@ -19,9 +19,12 @@ echo "----------------------------------------------"
 #build using docker file in directory:
 cd $PROJECTS_DIR/DDFacet
 IMAGENAME="ddf"
-docker build -t "$IMAGENAME:$BUILD_NUMBER" --no-cache=true .
+#check that it still builds on 18.04
+docker build -t "$IMAGENAME:$BUILD_NUMBER" --no-cache=true -f docker.1804 .
+#test on 20.04
+docker build -t "$IMAGENAME:$BUILD_NUMBER" --no-cache=true -f docker.2004 .
 docker run -m 100g --cap-add sys_ptrace \
-				   --memory-swap=-1 \
+		   --memory-swap=-1 \
                    --shm-size=150g \
                    --rm=true \
                    --name=$IMAGENAME$BUILD_NUMBER \

diff --git a/Dockerfile → docker.1804 b/Dockerfile → docker.1804
diff --git a/docker.2004 b/docker.2004
@@ -0,0 +1,262 @@
+FROM ubuntu:focal-20220105
+
+#Package dependencies
+COPY apt.sources.list /etc/apt/sources.list
+RUN sed -i 's/bionic/focal/g' /etc/apt/sources.list
+
+#Setup environment
+ENV DDFACET_TEST_DATA_DIR /test_data
+ENV DDFACET_TEST_OUTPUT_DIR /test_output
+
+# Support large mlocks
+RUN echo "*        -   memlock     unlimited" > /etc/security/limits.conf
+ENV DEBIAN_FRONTEND noninteractive
+ENV DEBIAN_PRIORITY critical
+ENV GNUCOMPILER 10
+ENV PYTHONVER 3.8
+ENV DEB_SETUP_DEPENDENCIES \
+    dpkg-dev \
+    g++-$GNUCOMPILER \
+    gcc-$GNUCOMPILER \
+    libc-dev \
+    cmake \
+    gfortran-$GNUCOMPILER \
+    git \
+    wget \
+    subversion \
+    rsync
+
+ENV DEB_DEPENCENDIES \
+    python3-virtualenv \
+    python3-pip \
+    libfftw3-dev \
+    python3-numpy \
+    libfreetype6 \
+    libfreetype6-dev \
+    libpng-dev \
+    pkg-config \
+    python3-dev \
+    libboost-all-dev \
+    libcfitsio-dev \
+    libhdf5-dev \
+    wcslib-dev \
+    libatlas-base-dev \
+    liblapack-dev \
+    python3-tk \
+    libreadline6-dev \
+    subversion \
+    liblog4cplus-dev \
+    libhdf5-dev \
+    libncurses5-dev \
+    flex \
+    bison \
+    libbison-dev \
+    libqdbm-dev \
+    # Reference image generation dependencies
+    make
+
+RUN apt-get update
+RUN apt-get install -y $DEB_SETUP_DEPENDENCIES
+RUN apt-get install -y $DEB_DEPENCENDIES
+
+
+RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-$GNUCOMPILER 100 && \
+    update-alternatives --install /usr/bin/cc cc /usr/bin/gcc-$GNUCOMPILER 100 && \
+    update-alternatives --install /usr/bin/x86_64-linux-gnu-gcc x86_64-linux-gnu-gcc /usr/bin/gcc-$GNUCOMPILER 100 && \
+    update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-$GNUCOMPILER 100 && \
+    update-alternatives --install /usr/bin/x86_64-linux-gnu-g++ x86_64-linux-gnu-g++ /usr/bin/g++-$GNUCOMPILER 100 && \
+    update-alternatives --install /usr/bin/cpp cpp /usr/bin/g++-$GNUCOMPILER 100 && \
+    update-alternatives --install /usr/bin/x86_64-linux-gnu-cpp x86_64-linux-gnu-cpp /usr/bin/g++-$GNUCOMPILER 100 && \
+    update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-$GNUCOMPILER 100 && \
+    update-alternatives --install /usr/bin/x86_64-linux-gnu-gfortran x86_64-linux-gnu-gfortran /usr/bin/gfortran-$GNUCOMPILER 100
+
+## All python dependencies go into venv
+WORKDIR /opt
+RUN virtualenv venv -p python${PYTHONVER}
+RUN . /opt/venv/bin/activate && python -m pip install -U pip setuptools wheel
+## Current workaround for numpy >=1.20 numerical issues. We will only support numpy < 1.20
+RUN . /opt/venv/bin/activate && python -m pip install -U numpy==1.19.5
+
+########################################
+# build SOFA
+########################################
+WORKDIR /opt
+RUN wget http://www.iausofa.org/2021_0512_C/sofa_c-20210512.tar.gz && \
+    tar xvfz sofa_c-20210512.tar.gz && \
+    rm sofa_c-20210512.tar.gz && \
+    cd /opt/sofa/20210512/c/src && \
+    sed -i '52s/.*/INSTALL_DIR = \/usr/' makefile && \
+    make -j4 && \
+    make install && \
+    make test && \
+    rm -r /opt/sofa
+
+#################################################################
+## build blitz from source
+#################################################################
+WORKDIR /opt
+RUN wget https://github.com/blitzpp/blitz/archive/1.0.2.tar.gz && \
+    tar -xvf 1.0.2.tar.gz && \
+    rm 1.0.2.tar.gz && \
+    cd /opt/blitz-1.0.2 && \
+    grep -rl python . | xargs sed -i "s/python/python${PYTHONVER}/g" && \
+    ./configure --enable-shared && \
+    make -j16 && \
+    make install && \
+    ldconfig && \
+    rm -r /opt/blitz-1.0.2
+
+#####################################################################
+## Get CASACORE ephem data
+#####################################################################
+RUN mkdir -p /usr/share/casacore/data/
+WORKDIR /usr/share/casacore/data/
+RUN rsync -avz rsync://casa-rsync.nrao.edu/casa-data .
+
+#####################################################################
+## BUILD CASACORE AND CASAREST FROM SOURCE
+#####################################################################
+WORKDIR /opt
+RUN wget https://github.com/casacore/casacore/archive/v3.4.0.tar.gz && \
+    tar xvf v3.4.0.tar.gz && \
+    rm v3.4.0.tar.gz && \
+    mkdir casacore-3.4.0/build && \
+    cd /opt/casacore-3.4.0/build && \
+    cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=Release -DBUILD_DEPRECATED=ON -DBUILD_PYTHON=OFF -DBUILD_PYTHON3=ON ../ && \
+    make -j 4 && \
+    make install && \
+    rm -r /opt/casacore-3.4.0 && \
+    cd /opt && \
+    wget https://github.com/casacore/casarest/archive/v1.8.0.tar.gz && \
+    tar xvf v1.8.0.tar.gz && \
+    rm v1.8.0.tar.gz && \
+    cd /opt/casarest-1.8.0 && \
+    mkdir -p build && \
+    cd /opt/casarest-1.8.0/build && \
+    cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=Release ../ && \
+    make -j 4 && \
+    make install && \
+    rm -r /opt/casarest-1.8.0 && \
+    ldconfig && \
+    cd /opt && \
+    wget https://github.com/casacore/python-casacore/archive/v3.4.0.tar.gz && \
+    tar xvf v3.4.0.tar.gz && \
+    rm v3.4.0.tar.gz && \
+    cd /opt/python-casacore-3.4.0 && \
+    . /opt/venv/bin/activate && python -m pip install . && \
+    cd / && \
+    python -c "from pyrap.tables import table as tbl"
+
+#####################################################################
+## BUILD MAKEMS FROM SOURCE AND TEST
+#####################################################################
+WORKDIR /opt
+RUN wget https://github.com/ska-sa/makems/archive/v1.5.4.tar.gz && \
+    tar xvf v1.5.4.tar.gz && \
+    rm v1.5.4.tar.gz && \
+    mkdir -p /opt/makems-1.5.4/LOFAR/build/gnu_opt && \
+    cd /opt/makems-1.5.4/LOFAR/build/gnu_opt && \
+    cmake -DCMAKE_MODULE_PATH:PATH=/opt/makems-1.5.4/LOFAR/CMake \
+          -DUSE_LOG4CPLUS=OFF -DBUILD_TESTING=OFF -DCMAKE_BUILD_TYPE=Release \
+          -DCMAKE_INSTALL_PREFIX=/usr ../.. && \
+    make -j 16 && \
+    make install && \
+    cd /opt/makems-1.5.4/test && \
+    makems WSRT_makems.cfg && \
+    rm -r /opt/makems-1.5.4
+
+#####################################################################
+## BUILD MEQTREES FROM SOURCE AND TEST
+#####################################################################
+WORKDIR /opt
+# Get MeqTrees universe python packages
+WORKDIR /opt
+
+RUN . /opt/venv/bin/activate && python -m pip install purr owlcat kittens meqtrees-cattery astro-tigger-lsm && \
+    wget https://github.com/ska-sa/meqtrees-timba/archive/refs/tags/v1.8.3.tar.gz && \
+    tar zxvf v1.8.3.tar.gz && \
+    rm v1.8.3.tar.gz && \
+    cd meqtrees-timba-1.8.3 && \
+    mkdir build && \
+    cd build && \
+    cmake -DCMAKE_INSTALL_PREFIX=/usr \
+          -DCMAKE_BUILD_TYPE=Release \
+          -DENABLE_PYTHON_3=ON \
+          -DPYTHON_PACKAGES_DIR=/opt/venv/lib/python${PYTHONVER}/site-packages \
+          .. && \
+    make -j4 && \
+    make install && \
+    ldconfig && \
+    rm -r /opt/meqtrees-timba-1.8.3 && \
+    cd /opt && \
+    wget https://github.com/ska-sa/pyxis/archive/v1.7.4.3.tar.gz && \
+    tar zxvf v1.7.4.3.tar.gz && \
+    rm v1.7.4.3.tar.gz && \
+    python -m pip install /opt/pyxis-1.7.4.3 nose && \
+    cd /opt/pyxis-1.7.4.3/Pyxis/recipes/meqtrees-batch-test && \
+    python -m "nose" && \
+    rm -r /opt/pyxis-1.7.4.3
+
+#####################################################################
+## BUILD LOFAR FROM SOURCE
+#####################################################################
+WORKDIR /opt
+RUN wget https://github.com/bennahugo/LOFARBeam/archive/refs/tags/DDF_KMS_20.04.tar.gz && \
+    tar zxvf DDF_KMS_20.04.tar.gz && \
+    rm DDF_KMS_20.04.tar.gz && \
+    cd LOFARBeam-DDF_KMS_20.04 && \
+    mkdir -p build/gnucxx11_opt && \
+    cd build/gnucxx11_opt && \
+    cmake -DCMAKE_INSTALL_PREFIX=/usr \
+          -DPYTHON_PACKAGES_DIR=/opt/venv/lib/python${PYTHONVER}/site-packages \
+          ../../ && \
+    make -j4 && \
+    make install && \
+    . /opt/venv/bin/activate && python -c "import lofar.stationresponse as lsr" && \
+    rm -r /opt/LOFARBeam-DDF_KMS_20.04
+
+#####################################################################
+## BUILD DDF FROM SOURCE
+#####################################################################
+#Copy DDFacet and SkyModel into the image
+ADD DDFacet /opt/DDFacet/DDFacet
+ADD SkyModel /opt/DDFacet/SkyModel
+ADD MANIFEST.in /opt/DDFacet/MANIFEST.in
+ADD setup.py /opt/DDFacet/setup.py
+ADD setup.cfg /opt/DDFacet/setup.cfg
+ADD README.rst /opt/DDFacet/README.rst
+ADD pyproject.toml /opt/DDFacet/pyproject.toml
+ADD .git /opt/DDFacet/.git
+ADD .gitignore /opt/DDFacet/.gitignore
+ADD .gitmodules /opt/DDFacet/.gitmodules
+
+RUN cd /opt/DDFacet/ && git submodule update --init --recursive && cd /
+# Finally install DDFacet
+RUN rm -rf /opt/DDFacet/DDFacet/cbuild
+RUN . /opt/venv/bin/activate && python -m pip install "/opt/DDFacet[dft-support,moresane-support,testing-requirements,fits-beam-support,kms-support]"
+
+# test montblanc install
+WORKDIR /tmp
+RUN . /opt/venv/bin/activate && python -c "import montblanc"
+RUN . /opt/venv/bin/activate && python -m nose /opt/venv/lib/python${PYTHONVER}/site-packages/montblanc/impl/rime/tensorflow/rime_ops/test*
+
+# ensure bdsf backend still works
+RUN . /opt/venv/bin/activate && python -c "import bdsf"
+
+# Finally Make VENV globally available in this container
+ENV LD_LIBRARY_PATH /opt/venv/lib:$LD_LIBRARY_PATH
+ENV PATH /opt/venv/bin:$PATH
+ENV PYTHONPATH /opt/venv/lib/python${PYTHONVER}/site-packages
+
+# perform some basic tests
+RUN DDF.py --help
+RUN MakeMask.py --help
+RUN MakeCatalog.py --help
+RUN MakeModel.py --help
+RUN MaskDicoModel.py --help
+RUN ClusterCat.py --help
+
+
+# set as entrypoint - user should be able to run docker run ddftag and get help printed
+ENTRYPOINT ["DDF.py"]
+CMD ["--help"]
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,5 +1,5 @@
 [build-system]
 requires = ["setuptools", 
             "wheel", 
-            "numpy>=1.15.1", # Ubuntu 18.04
+            "numpy>=1.15.1,<=1.19.5",
             "pybind11 >= 2.2.2"]
diff --git a/setup.py b/setup.py
@@ -33,7 +33,10 @@
 
 pkg='DDFacet'
 skymodel_pkg='SkyModel'
-__version__ = "0.6.0.0"
+__version__ = "0.7.0.0"
+# Bump when building for a new release version of Python
+MAJ_REQ = 3
+MIN_REQ = 8
 build_root=os.path.dirname(__file__)
 
 try:
@@ -42,6 +45,18 @@
     raise ImportError("Pybind11 not installed. Please install C++ binding package pybind11 before running DDFacet install. "
                       "You should not see this message unless you are not running pip install (19.x) -- run pip install!")
 
+# it seems the deprecated python setup.py install does not check the
+# version numbers required by the package in the descriptor. Although this is
+# not an issue with pip install, this leads to issues with upstream
+# packages.
+if not (sys.version_info.major == MAJ_REQ and sys.version_info.minor == MIN_REQ):
+    print("Your Python version '{}' is not officially supported. This package "
+          "requires version {}".format(sys.version,
+                                       "{}.{}".format(MAJ_REQ,
+                                                      MIN_REQ)))
+    print("You should not be seeing this message unless you are not using pip "
+          "install to install this software. Please use pip install")
+
 def backend(compile_options):
     if compile_options is not None:
         print("Compiling extension libraries with user defined options: '%s'"%compile_options)
@@ -117,38 +132,39 @@ def readme():
         return f.read()
 
 def requirements():
+    # Only upgrade these when testing with a new release version
+    # this should avoid upstream breakage on released versions
     requirements = ["nose >= 1.3.7; python_version >= '3'", 
-                    "Cython >= 0.25.2; python_version >= '3'", 
-                    "numpy >= 1.15.1; python_version >= '3'", 
-                    "sharedarray >= 3.2.0; python_version >= '3'", 
-                    "Polygon3 >= 3.0.8; python_version >= '3'", 
-                    "pyFFTW >= 0.10.4; python_version >= '3'", 
-                    "astropy >= 3.0; python_version >= '3'", 
-                    "deap >= 1.0.1; python_version >= '3'", 
-                    "ptyprocess>=0.5; python_version >= '3'", 
-                    "ipdb >= 0.10.3; python_version >= '3'", 
-                    "python-casacore >= 3.0.0; python_version >= '3'", 
-                    "pyephem >= 3.7.6.0; python_version >= '3'", 
-                    "numexpr >= 2.6.2; python_version >= '3'", 
-                    "matplotlib >= 2.0.0; python_version >= '3'", 
-                    "scipy >= 1.3.3; python_version >= '3'", 
-                    "astLib >= 0.8.0; python_version >= '3'", 
-                    "psutil >= 5.2.2; python_version >= '3'", 
-                    "py-cpuinfo >= 3.2.0; python_version >= '3'", 
-                    "tables >= 3.6.0; python_version >= '3'", 
-                    "prettytable >= 0.7.2; python_version >= '3'", 
-                    "pybind11 >= 2.2.2; python_version >= '3'", 
-                    "configparser >= 3.7.1; python_version >= '3'", 
-                    "pandas >=0.23.3; python_version >= '3'", 
-                    "ruamel.yaml >= 0.15.92; python_version >= '3'", 
-                    "pylru >= 1.1.0; python_version >= '3'", 
-                    "six >= 1.12.0; python_version >= '3'", 
-                    "pybind11 >= 2.2.2; python_version >= '3'", 
-                    "dask[array] >= 1.1.0; python_version >= '3'", 
-                    "codex-africanus[dask] >= 0.2.10; python_version >= '3'", 
-                    "regions",
-                    "pywavelets",
-                    "tqdm"
+                    "Cython >= 0.25.2, <= 0.29.30; python_version >= '3'", 
+                    "numpy >= 1.15.1, <= 1.19.5; python_version >= '3'", 
+                    "sharedarray >= 3.2.0, <= 3.2.1; python_version >= '3'", 
+                    "Polygon3 >= 3.0.8, <= 3.0.9.1; python_version >= '3'", 
+                    "pyFFTW >= 0.10.4, <= 0.12.0; python_version >= '3'", 
+                    "astropy >= 3.0, <= 4.1; python_version >= '3'", 
+                    "deap >= 1.0.1, <= 1.3.1; python_version >= '3'", 
+                    "ptyprocess>=0.5, <= 0.7.0; python_version >= '3'", 
+                    "ipdb >= 0.10.3, <=0.13.9; python_version >= '3'", 
+                    "python-casacore >= 3.0.0, <=3.4.0; python_version >= '3'", 
+                    "pyephem >= 3.7.6.0, <=9.99; python_version >= '3'", 
+                    "numexpr >= 2.6.2,<=2.8.1; python_version >= '3'", 
+                    "matplotlib >= 2.0.0,<=3.3.4; python_version >= '3'", 
+                    "scipy >= 1.3.3,<=1.5.4; python_version >= '3'", 
+                    "astLib >= 0.8.0,<=0.11.7; python_version >= '3'", 
+                    "psutil >= 5.2.2,<=5.9.1; python_version >= '3'", 
+                    "py-cpuinfo >= 3.2.0,<=8.0.0; python_version >= '3'", 
+                    "tables >= 3.6.0,<=3.7.0; python_version >= '3'", 
+                    "prettytable >= 0.7.2,<=2.5.0; python_version >= '3'", 
+                    "pybind11 >= 2.2.2,<=2.9.2; python_version >= '3'", 
+                    "configparser >= 3.7.1,<=5.2.0; python_version >= '3'", 
+                    "pandas >=0.23.3,<=1.1.5; python_version >= '3'", 
+                    "ruamel.yaml >= 0.15.92,<=0.17.21; python_version >= '3'", 
+                    "pylru >= 1.1.0,<=1.2.1; python_version >= '3'", 
+                    "six >= 1.12.0,<=1.16.0; python_version >= '3'", 
+                    "dask[array] >= 1.1.0,<=2021.3.0; python_version >= '3'", 
+                    "codex-africanus[dask] <= 0.2.10; python_version >= '3'", 
+                    "regions <=0.5",
+                    "pywavelets <=1.1.1",
+                    "tqdm<=4.64.0"
                     ] 
     install_requirements = requirements
 
@@ -175,7 +191,7 @@ def requirements():
                 'build': custom_build,
                 'build_ext': custom_build_ext
                },
-      python_requires='>=3.0,<3.7',
+      python_requires='>=3.0,<{}'.format("{}.{}".format(MAJ_REQ, MIN_REQ+1)),
       packages=[pkg, skymodel_pkg],
       install_requires=requirements(),
       include_package_data=True,