From fb61934bb7d9c088b3cfe28df9744a4827f410b8 Mon Sep 17 00:00:00 2001 From: Juha Jeronen Date: Wed, 26 Apr 2017 13:51:19 +0300 Subject: [PATCH] initial commit --- .gitignore | 8 + CHANGELOG.md | 3 + LICENSE.md | 2 + README.md | 148 ++++++++++++ mylibrary/__init__.py | 11 + mylibrary/compute.pyx | 68 ++++++ mylibrary/dostuff.pyx | 26 +++ mylibrary/submodule/__init__.py | 0 mylibrary/submodule/helloworld.pxd | 8 + mylibrary/submodule/helloworld.pyx | 14 ++ setup.py | 362 +++++++++++++++++++++++++++++ test/cython_module.pyx | 14 ++ test/mylibrary_test.py | 56 +++++ test/setup.py | 205 ++++++++++++++++ 14 files changed, 925 insertions(+) create mode 100644 .gitignore create mode 100644 CHANGELOG.md create mode 100644 LICENSE.md create mode 100644 README.md create mode 100644 mylibrary/__init__.py create mode 100644 mylibrary/compute.pyx create mode 100644 mylibrary/dostuff.pyx create mode 100644 mylibrary/submodule/__init__.py create mode 100644 mylibrary/submodule/helloworld.pxd create mode 100644 mylibrary/submodule/helloworld.pyx create mode 100644 setup.py create mode 100644 test/cython_module.pyx create mode 100644 test/mylibrary_test.py create mode 100644 test/setup.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..cc44ade --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +*~ +*.pyc +*.c +*.so +*.egg-info +build +test/build +dist diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..683166d --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,3 @@ +## [v0.1.0] + - initial release + diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..c3a9fb1 --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,2 @@ +Public Domain + diff --git a/README.md b/README.md new file mode 100644 index 0000000..be22950 --- /dev/null +++ b/README.md @@ -0,0 +1,148 @@ +## setup-template-cython + +Setuptools-based `setup.py` template for Cython projects + +### Introduction + +Setuptools[[1]][setuptools] has become the tool of choice[[2]][packaging] for packaging Python projects, yet not much documentation is available on how to use setuptools in Cython projects. As of this writing (Cython 0.25), Cython's [official packaging instructions](http://cython.readthedocs.io/en/latest/src/reference/compilation.html#distributing-cython-modules) are based on distutils. + +For packages to be distributed (especially through [PyPI](https://pypi.python.org/pypi)), setuptools is preferable, since the documentation on distributing packages[[3]][distributing] assumes that is what the developer uses. Also, [setuptools adds dependency resolution](http://stackoverflow.com/questions/25337706/setuptools-vs-distutils-why-is-distutils-still-a-thing) (over distutils), which is an essential feature of `pip`. + +This very minimal project documents the author's best guess at current best practices for the packaging and distribution of Cython projects, by piecing together information from various sources (mainly documentation and StackOverflow). Possible corrections, if any, are welcome. + +This is intended as a template for new Cython projects. For completeness, a minimal Cython module is included. This project is placed in the public domain so as to allow its use anywhere. + +This is similar to [simple-cython-example](https://github.com/thearn/simple-cython-example), but different. Here the focus is on numerical scientific projects, where a custom Cython extension (containing all-new code) can bring a large speedup. + +The aim is to help open-sourcing such extensions in a manner that lets others effortlessly compile them (also in practice), thus advancing the openness and repeatability of science. + + +### Features + +Our [`setup.py`](setup.py) features the following: + + - The most important fields of `setup()` + - If this is all you need, [simple-cython-example](https://github.com/thearn/simple-cython-example) is much cleaner. + - If this is all you need, and you somehow ended up here even though your project is pure Python, [PyPA's sampleproject](https://github.com/pypa/sampleproject/blob/master/setup.py) (as mentioned in [[4]][setup-py]) has more detail on this. + - How to get `absolute_import` working in a Cython project + - For compatibility with both Python 3 and Python 2.7 (with `from __future__ import absolute_import`) + - For scientific developers used to Python 2.7, this is perhaps the only tricky part in getting custom Cython code to play nicely with Python 3. (As noted elsewhere[[a]](http://www.python3statement.org/)[[b]](http://python-3-for-scientists.readthedocs.io/en/latest/), it is time to move to Python 3.) + - How to automatically grab `__version__` from `mylibrary/__init__.py` (using AST; no import or regexes), so that you can declare your package version [OnceAndOnlyOnce](http://wiki.c2.com/?OnceAndOnlyOnce) (based on [[5]][getversion]) + - Hopefully appropriate compiler and linker flags for math and non-math modules on `x86_64`, in production and debug configurations. + - Also compiler and linker flags for OpenMP, to support `cython.parallel.prange`. + - How to make `setup.py` pick up non-package data files, such as your documentation and usage examples (based on [[6]][datafolder]) + - How to make `setup.py` pick up data files inside your Python packages + - How to enforce that `setup.py` is running under a given minimum Python version ([considered harmful](http://stackoverflow.com/a/1093331), but if duck-checking for individual features is not an option for a reason or another) (based on [[7]][enforcing]) + - Disabling `zip_safe`. Having `zip_safe` enabled (which will in practice happen by default) is a bad idea for Cython projects, because: + - Cython (as of this writing, version 0.25) will not see `.pxd` headers inside installed `.egg` files. Thus other libraries cannot `cimport` modules from yours if it has `zip_safe` set. + - At (Python-level) `import` time, the OS's dynamic library loader usually needs to have the `.so` unzipped (from the `.egg`) to a temporary directory anyway. + + +### Usage + +See the [setuptools manual](http://setuptools.readthedocs.io/en/latest/setuptools.html#command-reference). Perhaps the most useful commands are: + +```bash +python setup.py build_ext +python setup.py build # copies .py files in the Python packages +python setup.py install # will automatically "build" and "bdist" +python setup.py sdist +``` + +Substitute `python2` or `python3` for `python` if needed. + +For `build_ext`, the switch `--inplace` may be useful for one-file throwaway projects, but packages to be installed are generally much better off by letting setuptools create a `build/` subdirectory. + +For `install`, the switch `--user` may be useful. As can, alternatively, running the command through `sudo`, depending on your installation. + +#### Binary distributions + +As noted in the [Python packaging guide](https://packaging.python.org/distributing/#platform-wheels), PyPI accepts platform wheels (platform-specific binary distributions) for Linux only if they conform to [the `manylinux1` ABI](https://www.python.org/dev/peps/pep-0513/), so running `python setup.py bdist_egg` on an arbitrary development machine is generally not very useful for the purposes of distribution. + +For the adventurous, PyPA provides [instructions](https://github.com/pypa/manylinux) along with a Docker image. + +For the less adventurous, just make an sdist and upload that; scientific Linux users are likely not scared by an automatic compilation step. + + +### Notes + + 0. This project assumes the end user will have Cython installed, which is likely the case for people writing and interacting with numerics code in Python. Indeed, our [`setup.py`](setup.py) has Cython set as a requirement, and hence the eventual `pip install mylibrary` will pull in Cython if it is not installed. + + The Cython extensions are always compiled using Cython. Or in other words, regular-end-user-friendly logic for conditionally compiling only the Cython-generated C source, or the original Cython source, has **not** been included. If this is needed, see [this StackOverflow discussion](http://stackoverflow.com/questions/4505747/how-should-i-[5]-a-python-package-that-contains-cython-code) for hints. See also item 2 below. + + The generated C source files, however, are included in the resulting distribution (both sdist and bdist). + + 1. In Cython projects, it is preferable to always use absolute module paths when `absolute_import` is in use, even if the module to be cimported is located in the same directory (as the module that is doing the cimport). This allows using the same module paths for imports and cimports. + + The reason for this recommendation is that the relative variant (`from . cimport foo`), although in line with [PEP 328](https://www.python.org/dev/peps/pep-0328/), is difficult to get to work properly with Cython's include path. + + Our [`setup.py`](setup.py) adds `.`, the top-level directory containing `setup.py`, to Cython's include path, but does not add any of its subdirectories. This makes the cimports with absolute module paths work correctly[[8]][packagehierarchy] (also when pointing to the library being compiled), assuming `mylibrary` lives in a `mylibrary/` subdirectory of the top-level directory that contains `setup.py`. See the included example. + + 2. Historically, it was common practice in `setup.py` to import Cython's replacement for distutils' `build_ext`, in order to make `setup()` recognize `.pyx` source files. + + Instead, we let setuptools keep its `build_ext`, and call `cythonize()` explicitly in the invocation of `setup()`. As of this writing, this is the approach given in [Cython's documentation](http://cython.readthedocs.io/en/latest/src/reference/compilation.html#compiling-with-distutils), albeit it refers to distutils instead of setuptools. + + This gives us some additional bonuses: + - Cython extensions can be compiled in debug mode (for use with [cygdb](http://cython.readthedocs.io/en/latest/src/userguide/debugging.html)). + - We get `make`-like dependency resolution; a `.pyx` source file is automatically re-cythonized, if a `.pxd` file it cimports, changes. + - We get the nice `[1/4] Cythonizing mylibrary/main.pyx` progress messages when `setup.py` runs, whenever Cython detects it needs to compile `.pyx` sources to C. + - By requiring Cython, there is no need to store the generated C source files in version control; they are not meant to be directly human-editable. + + The setuptools documentation gives advice that, depending on interpretation, [may be in conflict with this](https://setuptools.readthedocs.io/en/latest/setuptools.html#distributing-extensions-compiled-with-pyrex) (considering Cython is based on Pyrex). We do not import Cython's replacement for `build_ext`, but following the Cython documentation, we do import `cythonize` and call it explicitly. + + Because we do this, setuptools sees only C sources, so we miss setuptools' [automatic switching](https://setuptools.readthedocs.io/en/latest/setuptools.html#distributing-extensions-compiled-with-pyrex) of Cython and C compilation depending on whether Cython is installed (see the source code for `setuptools.extension.Extension`). Our approach requires having Cython installed even if the generated C sources are up to date (in which case the Cython compilation step will no-op, skipping to the C compilation step). + + Note that as a side effect, `cythonize()` will run even if the command-line options given to `setup.py` are nonsense (or more commonly, contain a typo), since it runs first, before control even passes to `setup()`. (I.e., don't go grab your coffee until the build starts compiling the generated C sources.) + + For better or worse, the chosen approach favors Cython's own mechanism for handling `.pyx` sources over the one provided by setuptools. + + 3. Old versions of Cython may choke on the `cythonize()` options `include_path` and/or `gdb_debug`. If `setup.py` gives mysterious errors that can be traced back to these, try upgrading your Cython installation. + + Note that `pip install cython --upgrade` gives you the latest version. (You may need to add `--user`, or run it through `sudo`, depending on your installation.) + + 4. Using setuptools with Cython projects needs `setuptools >= 18.0`, to correctly support Cython in `requires`[[9]][setuptools18]. + + In practice this is not a limitation, as `18.0` is already a very old version (`35.0` being current at the time of this writing). In the unlikely event that it is necessary to support versions of setuptools even older than `18.0`, it is possible[[9]][setuptools18] to use [setuptools-cython](https://pypi.python.org/pypi/setuptools_cython/) from PyPI. (This package is not needed if `setuptools >= 18.0`.) + + 5. If you are familiar with distutils, but new to setuptools, see [the list of new and changed keywords](https://setuptools.readthedocs.io/en/latest/setuptools.html#new-and-changed-setup-keywords) in the setuptools documentation. + + +### Distributing your package + +If you choose to release your package for distribution: + + 0. See the [distributing](https://packaging.python.org/distributing/) section of the packaging guide, and especially the subsection on [uploading to PyPI](https://packaging.python.org/distributing/#uploading-your-project-to-pypi). + + Especially if your package has dependencies, it is important to get at least an sdist onto PyPI to make the package easy to install (via `pip install`). + + - Also, keep in mind that outside managed environments such as Anaconda, `pip` is the preferred way for installing scientific Python packages, even though having multiple package managers on the same system could be considered harmful. Scientific packages are relatively rapidly gaining new features, thus making access to the latest release crucial. + + (Debian-based Linux distributions avoid conflicts between the two sets of managed files by making `sudo pip install` install to `/usr/local`, while the system `apt-get` installs to `/usr`. This does not, however, prevent breakage caused by overrides (loading a newer version from `/usr/local`), if it happens that some Python package is not fully backward-compatible. A proper solution requires [one of the virtualenv tools](http://stackoverflow.com/questions/41573587/what-is-the-difference-between-venv-pyvenv-pyenv-virtualenv-virtualenvwrappe) at the user end.) + + 1. Be sure to use `twine upload`, **not** ~~`python -m setup upload`~~, since the latter may transmit your password in plaintext. + + Before the first upload of a new project, use `twine register`. + + [Official instructions for twine](https://pypi.python.org/pypi/twine). + + 2. Generally speaking, it is [a good idea to disregard old advice](http://stackoverflow.com/a/14753678) on Python packaging. By 2020 when Python 2.7 support ends, that probably includes this document. + + For example, keep in mind that `pip` has replaced `ez_setup`, and nowadays `pip` (in practice) comes with Python. + + Many Python distribution tools have been sidelined by history, or merged back into the supported ones (see [the StackOverflow answer already linked above](http://stackoverflow.com/a/14753678)). Distutils and setuptools remain, nowadays [fulfilling different roles](http://stackoverflow.com/a/40176290). + + +### License + +Public Domain + +[setuptools]: https://packaging.python.org/key_projects/#easy-install +[packaging]: https://packaging.python.org/current/#packaging-tool-recommendations +[distributing]: https://packaging.python.org/distributing/ +[setup-py]: https://packaging.python.org/distributing/#setup-py +[getversion]: http://stackoverflow.com/questions/2058802/how-can-i-get-the-version-defined-in-setup-py-setuptools-in-my-package +[datafolder]: http://stackoverflow.com/questions/13628979/setuptools-how-to-make-package-contain-extra-data-folder-and-all-folders-inside +[enforcing]: http://stackoverflow.com/questions/19534896/enforcing-python-version-in-setup-py +[packagehierarchy]: https://github.com/cython/cython/wiki/PackageHierarchy +[setuptools18]: http://stackoverflow.com/a/27420487 + diff --git a/mylibrary/__init__.py b/mylibrary/__init__.py new file mode 100644 index 0000000..3fd38a2 --- /dev/null +++ b/mylibrary/__init__.py @@ -0,0 +1,11 @@ +# -*- coding: utf-8 -*- +# +"""Init for mylibrary.""" + +from __future__ import absolute_import + +# This is extracted automatically by the top-level setup.py. +__version__ = '0.1.0' + +# add any imports here, if you wish to bring things into the library's top-level namespace. + diff --git a/mylibrary/compute.pyx b/mylibrary/compute.pyx new file mode 100644 index 0000000..f91c9de --- /dev/null +++ b/mylibrary/compute.pyx @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +# +# cython: wraparound = False +# cython: boundscheck = False +# cython: cdivision = True +"""Example Cython module.""" + +from __future__ import division, print_function, absolute_import + + +# import something from libm +from libc.math cimport sqrt as c_sqrt + +# we use NumPy for memory allocation +import numpy as np + + +# The docstring conforms to the NumPyDoc style: +# +# https://github.com/numpy/numpy/blob/master/doc/HOWTO_DOCUMENT.rst.txt +# +# See also +# https://pypi.python.org/pypi/numpydoc +# http://docutils.sourceforge.net/docs/user/rst/demo.txt +# +# We use the buffer protocol: +# http://cython.readthedocs.io/en/latest/src/userguide/memoryviews.html +# +def f( double[::1] x ): + """Example math function. + +Take the square root, elementwise. + +Parameters: + x : rank-1 np.array of double + Numbers to be square-rooted. + +Return value: + rank-1 np.array of double + The square roots. +""" + cdef int n = x.shape[0] + + # np.empty() is a pretty good mechanism for dynamic allocation of arrays, + # as long as memory allocation can be done on the Python side, + # + # If you absolutely need to dynamically allocate memory in nogil code, + # then "from libc.stdlib cimport malloc, free", and be ready for pain. + # + cdef double[::1] out = np.empty( (n,), dtype=np.float64, order="C" ) + + # Memoryview slices don't do out[:] = ... assignments, so we loop. + # + # Everything is typed, though, so the loop will run at C speed. + # + # As a bonus, we release the GIL, so any other Python threads + # can proceed while this one is computing. + # + # We could also "cimport cython.parallel" and "for j in cython.parallel.prange(n):" + # if we wanted to link this with OpenMP. + # + cdef int j + with nogil: + for j in range(n): + out[j] = c_sqrt(x[j]) + + return np.asanyarray(out) # return proper np.ndarray, not memoryview slice + diff --git a/mylibrary/dostuff.pyx b/mylibrary/dostuff.pyx new file mode 100644 index 0000000..47d92e1 --- /dev/null +++ b/mylibrary/dostuff.pyx @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# +# cython: wraparound = False +# cython: boundscheck = False +# cython: cdivision = True +"""Example Cython module.""" + +from __future__ import division, print_function, absolute_import + + +# Use absolute module names, even from this library itself. +# +cimport mylibrary.submodule.helloworld as helloworld + + +def hello(s): + """Python interface to mylibrary.submodule.helloworld. + +This is mainly an example of absolute imports in Cython modules. + +Parameters: + s : str + The string to echo. +""" + helloworld.hello(s) + diff --git a/mylibrary/submodule/__init__.py b/mylibrary/submodule/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mylibrary/submodule/helloworld.pxd b/mylibrary/submodule/helloworld.pxd new file mode 100644 index 0000000..bab4370 --- /dev/null +++ b/mylibrary/submodule/helloworld.pxd @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- +# +# Cython-level declarations for mylibrary.submodule.helloworld, available for cimport from other Cython modules. + +from __future__ import absolute_import + +cdef void hello(str s) + diff --git a/mylibrary/submodule/helloworld.pyx b/mylibrary/submodule/helloworld.pyx new file mode 100644 index 0000000..dde1a97 --- /dev/null +++ b/mylibrary/submodule/helloworld.pyx @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +# +# cython: wraparound = False +# cython: boundscheck = False +# cython: cdivision = True +"""Example Cython module.""" # this is the Python-level docstring + +from __future__ import division, print_function, absolute_import + +# Echo the string s. +# +cdef void hello(str s): + print(s) # this is really, really silly (Python print() in a cdef function) + diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..2953d89 --- /dev/null +++ b/setup.py @@ -0,0 +1,362 @@ +# -*- coding: utf-8 -*- +# +"""setuptools-based setup.py template for Cython projects. + +Main setup for the library. + +Supports Python 2.7 and 3.4. + +Usage as usual with setuptools: + python setup.py build_ext + python setup.py build + python setup.py install + python setup.py sdist + +For details, see + http://setuptools.readthedocs.io/en/latest/setuptools.html#command-reference +""" + +from __future__ import division, print_function, absolute_import + +try: + # Python 3 + MyFileNotFoundError = FileNotFoundError +except: # FileNotFoundError does not exist in Python 2.7 + # Python 2.7 + # - open() raises IOError + # - remove() (not currently used here) raises OSError + MyFileNotFoundError = (IOError, OSError) + +######################################################### +# General config +######################################################### + +# Name of the top-level package of your library. +# +# This is also the top level of its source tree, relative to the top-level project directory setup.py resides in. +# +libname="mylibrary" + +# Choose build type. +# +build_type="optimized" +#build_type="debug" + +# Short description for package list on PyPI +# +SHORTDESC="setuptools template for Cython projects" + +# Long description for package homepage on PyPI +# +DESC="""setuptools-based setup.py template for Cython projects. + +The focus of this template is on numerical scientific projects, +where a custom Cython extension (containing all-new code) can bring a large speedup. + +For completeness, a minimal Cython module is included. + +Supports Python 2.7 and 3.4. +""" + +# Set up data files for packaging. +# +# Directories (relative to the top-level directory where setup.py resides) in which to look for data files. +datadirs = ("test",) + +# File extensions to be considered as data files. (Literal, no wildcards.) +dataexts = (".py", ".pyx", ".pxd", ".c", ".cpp", ".h", ".sh", ".lyx", ".tex", ".txt", ".pdf") + +# Standard documentation to detect (and package if it exists). +# +standard_docs = ["README", "LICENSE", "TODO", "CHANGELOG", "AUTHORS"] # just the basename without file extension +standard_doc_exts = [".md", ".rst", ".txt", ""] # commonly .md for GitHub projects, but other projects may use .rst or .txt (or even blank). + + +######################################################### +# Init +######################################################### + +# check for Python 2.7 or later +# http://stackoverflow.com/questions/19534896/enforcing-python-version-in-setup-py +import sys +if sys.version_info < (2,7): + sys.exit('Sorry, Python < 2.7 is not supported') + +import os + +from setuptools import setup +from setuptools.extension import Extension + +try: + from Cython.Build import cythonize +except ImportError: + sys.exit("Cython not found. Cython is needed to build the extension modules.") + + +######################################################### +# Definitions +######################################################### + +# Define our base set of compiler and linker flags. +# +# This is geared toward x86_64, see +# https://gcc.gnu.org/onlinedocs/gcc-4.6.4/gcc/i386-and-x86_002d64-Options.html +# +# Customize these as needed. +# +# Note that -O3 may sometimes cause mysterious problems, so we limit ourselves to -O2. + +# Modules involving numerical computations +# +extra_compile_args_math_optimized = ['-march=native', '-O2', '-msse', '-msse2', '-mfma', '-mfpmath=sse'] +extra_compile_args_math_debug = ['-march=native', '-O0', '-g'] +extra_link_args_math_optimized = [] +extra_link_args_math_debug = [] + +# Modules that do not involve numerical computations +# +extra_compile_args_nonmath_optimized = ['-O2'] +extra_compile_args_nonmath_debug = ['-O0', '-g'] +extra_link_args_nonmath_optimized = [] +extra_link_args_nonmath_debug = [] + +# Additional flags to compile/link with OpenMP +# +openmp_compile_args = ['-fopenmp'] +openmp_link_args = ['-fopenmp'] + + +######################################################### +# Helpers +######################################################### + +# Make absolute cimports work. +# +# See +# https://github.com/cython/cython/wiki/PackageHierarchy +# +my_include_dirs = ["."] + + +# Choose the base set of compiler and linker flags. +# +if build_type == 'optimized': + my_extra_compile_args_math = extra_compile_args_math_optimized + my_extra_compile_args_nonmath = extra_compile_args_nonmath_optimized + my_extra_link_args_math = extra_link_args_math_optimized + my_extra_link_args_nonmath = extra_link_args_nonmath_optimized + my_debug = False + print( "build configuration selected: optimized" ) +elif build_type == 'debug': + my_extra_compile_args_math = extra_compile_args_math_debug + my_extra_compile_args_nonmath = extra_compile_args_nonmath_debug + my_extra_link_args_math = extra_link_args_math_debug + my_extra_link_args_nonmath = extra_link_args_nonmath_debug + my_debug = True + print( "build configuration selected: debug" ) +else: + raise ValueError("Unknown build configuration '%s'; valid: 'optimized', 'debug'" % (build_type)) + + +def declare_cython_extension(extName, use_math=False, use_openmp=False): + """Declare a Cython extension module for setuptools. + +Parameters: + extName : str + Absolute module name, e.g. use `mylibrary.mymodule.submodule` + for the Cython source file `mylibrary/mymodule/submodule.pyx`. + + use_math : bool + If True, set math flags and link with ``libm``. + + use_openmp : bool + If True, compile and link with OpenMP. + +Return value: + Extension object + that can be passed to ``setuptools.setup``. +""" + extPath = extName.replace(".", os.path.sep)+".pyx" + + if use_math: + compile_args = list(my_extra_compile_args_math) # copy + link_args = list(my_extra_link_args_math) + libraries = ["m"] # link libm; this is a list of library names without the "lib" prefix + else: + compile_args = list(my_extra_compile_args_nonmath) + link_args = list(my_extra_link_args_nonmath) + libraries = None # value if no libraries, see setuptools.extension._Extension + + # OpenMP + if use_openmp: + compile_args.insert( 0, openmp_compile_args ) + link_args.insert( 0, openmp_link_args ) + + # See + # http://docs.cython.org/src/tutorial/external.html + # + # on linking libraries to your Cython extensions. + # + return Extension( extName, + [extPath], + extra_compile_args=compile_args, + extra_link_args=link_args, + libraries=libraries + ) + + +# Gather user-defined data files +# +# http://stackoverflow.com/questions/13628979/setuptools-how-to-make-package-contain-extra-data-folder-and-all-folders-inside +# +datafiles = [] +getext = lambda filename: os.path.splitext(filename)[1] +for datadir in datadirs: + datafiles.extend( [(root, [os.path.join(root, f) for f in files if getext(f) in dataexts]) + for root, dirs, files in os.walk(datadir)] ) + + +# Add standard documentation (README et al.), if any, to data files +# +detected_docs = [] +for docname in standard_docs: + for ext in standard_doc_exts: + filename = "".join( (docname, ext) ) # relative to the directory in which setup.py resides + if os.path.isfile(filename): + detected_docs.append(filename) +datafiles.append( ('.', detected_docs) ) + + +# Extract __version__ from the package __init__.py +# (since it's not a good idea to actually run __init__.py during the build process). +# +# http://stackoverflow.com/questions/2058802/how-can-i-get-the-version-defined-in-setup-py-setuptools-in-my-package +# +import ast +init_py_path = os.path.join(libname, '__init__.py') +version = '0.0.unknown' +try: + with open(init_py_path) as f: + for line in f: + if line.startswith('__version__'): + version = ast.parse(line).body[0].value.s + break + else: + print( "WARNING: Version information not found in '%s', using placeholder '%s'" % (init_py_path, version), file=sys.stderr ) +except MyFileNotFoundError: + print( "WARNING: Could not find file '%s', using placeholder version information '%s'" % (init_py_path, version), file=sys.stderr ) + + +######################################################### +# Set up modules +######################################################### + +# declare Cython extension modules here +# +ext_module_dostuff = declare_cython_extension( "mylibrary.dostuff", use_math=False, use_openmp=False ) +ext_module_compute = declare_cython_extension( "mylibrary.compute", use_math=True, use_openmp=False ) +ext_module_helloworld = declare_cython_extension( "mylibrary.submodule.helloworld", use_math=False, use_openmp=False ) + +# this is mainly to allow a manual logical ordering of the declared modules +# +cython_ext_modules = [ext_module_dostuff, + ext_module_compute, + ext_module_helloworld] + +# Call cythonize() explicitly, as recommended in the Cython documentation. See +# http://cython.readthedocs.io/en/latest/src/reference/compilation.html#compiling-with-distutils +# +# This will favor Cython's own handling of '.pyx' sources over that provided by setuptools. +# +# Note that my_ext_modules is just a list of Extension objects. We could add any C sources (not coming from Cython modules) here if needed. +# cythonize() just performs the Cython-level processing, and returns a list of Extension objects. +# +my_ext_modules = cythonize( cython_ext_modules, include_path=my_include_dirs, gdb_debug=my_debug ) + + +######################################################### +# Call setup() +######################################################### + +setup( + name = "setup-template-cython", + version = version, + author = "Juha Jeronen", + author_email = "juha.jeronen@jyu.fi", + url = "https://github.com/Technologicat/setup-template-cython", + + description = SHORTDESC, + long_description = DESC, + + # CHANGE THIS + license = "PD", + + # free-form text field; http://stackoverflow.com/questions/34994130/what-platforms-argument-to-setup-in-setup-py-does + platforms = ["Linux"], + + # See + # https://pypi.python.org/pypi?%3Aaction=list_classifiers + # + # for the standard classifiers. + # + # Remember to configure these appropriately for your project, especially license! + # + classifiers = [ "Development Status :: 4 - Beta", + "Environment :: Console", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "License :: Public Domain", # CHANGE THIS + "Operating System :: POSIX :: Linux", + "Programming Language :: Cython", + "Programming Language :: Python", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.4", + "Topic :: Scientific/Engineering", + "Topic :: Scientific/Engineering :: Mathematics", + "Topic :: Software Development :: Libraries", + "Topic :: Software Development :: Libraries :: Python Modules" + ], + + # See + # http://setuptools.readthedocs.io/en/latest/setuptools.html + # + setup_requires = ["cython", "numpy"], + install_requires = ["numpy"], + provides = ["setup_template_cython"], + + # keywords for PyPI (in case you upload your project) + # + # e.g. the keywords your project uses as topics on GitHub, minus "python" (if there) + # + keywords = ["setuptools setup.py template example cython"], + + # All extension modules (list of Extension objects) + # + ext_modules = my_ext_modules, + + # Declare packages so that python -m setup build will copy .py files (especially __init__.py). + # + # This **does not** automatically recurse into subpackages, so they must also be declared. + # + packages = ["mylibrary", "mylibrary.submodule"], + + # Install also Cython headers so that other Cython modules can cimport ours + # + # Fileglobs relative to each package, **does not** automatically recurse into subpackages. + # + # FIXME: force sdist, but sdist only, to keep the .pyx files (this puts them also in the bdist) + package_data={'mylibrary': ['*.pxd', '*.pyx'], + 'mylibrary.submodule': ['*.pxd', '*.pyx']}, + + # Disable zip_safe, because: + # - Cython won't find .pxd files inside installed .egg, hard to compile libs depending on this one + # - dynamic loader may need to have the library unzipped to a temporary directory anyway (at import time) + # + zip_safe = False, + + # Custom data files not inside a Python package + data_files = datafiles +) + diff --git a/test/cython_module.pyx b/test/cython_module.pyx new file mode 100644 index 0000000..a76b68c --- /dev/null +++ b/test/cython_module.pyx @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +# +# cython: wraparound = False +# cython: boundscheck = False +# cython: cdivision = True +"""Example Cython module.""" # this is the Python-level docstring + +from __future__ import division, print_function, absolute_import + +# Silly example: check if input is 42, return True or False. +# +def g(int x): + return (x == 42) + diff --git a/test/mylibrary_test.py b/test/mylibrary_test.py new file mode 100644 index 0000000..aeed4c9 --- /dev/null +++ b/test/mylibrary_test.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +"""Test mylibrary.""" + +from __future__ import division, print_function, absolute_import + +import sys + +import numpy as np + +# This requires mylibrary to be compiled and installed first (using the top-level setup.py). +# +try: + import mylibrary.dostuff as dostuff + import mylibrary.compute as compute +except ImportError: + print( "ERROR: mylibrary not found; is it installed (in this Python)?", file=sys.stderr ) + raise + +# Import local Cython module. +# +# This module belongs to the tests, and is not part of mylibrary. +# +try: + import cython_module +except ImportError: + print( "ERROR: cython_module.pyx must be compiled first; run 'python -m setup build_ext --inplace' to do this", file=sys.stderr ) + raise + + +def test(): + # Test the dostuff module in the installed mylibrary + # + # This should just print "Hello world" after jumping through all the API hoops + dostuff.hello("Hello world") + + # Test the compute module in the installed mylibrary + x = np.arange(1000, dtype=np.float64) + y1 = np.sqrt(x) + y2 = compute.f(x) + if np.allclose( y1, y2 ): + print("**PASS** compute.f()") + else: + print("**FAIL** compute.f()") + + # Test the local Cython module + b1 = cython_module.g(42) + b2 = cython_module.g(23) + if b1 and not b2: + print("**PASS** cython_module.g()") + else: + print("**FAIL** cython_module.g()") + + +if __name__ == '__main__': + test() + diff --git a/test/setup.py b/test/setup.py new file mode 100644 index 0000000..a7fdb26 --- /dev/null +++ b/test/setup.py @@ -0,0 +1,205 @@ +# -*- coding: utf-8 -*- +# +"""setuptools-based setup.py template for Cython projects. + +Setup for local modules in this directory, that are not installed as part of the library. + +Supports Python 2.7 and 3.4. + +Usage: + python setup.py build_ext --inplace +""" + +from __future__ import division, print_function, absolute_import + +try: + # Python 3 + MyFileNotFoundError = FileNotFoundError +except: # FileNotFoundError does not exist in Python 2.7 + # Python 2.7 + # - open() raises IOError + # - remove() (not currently used here) raises OSError + MyFileNotFoundError = (IOError, OSError) + +######################################################### +# General config +######################################################### + +# Choose build type. +# +build_type="optimized" +#build_type="debug" + + +######################################################### +# Init +######################################################### + +# check for Python 2.7 or later +# http://stackoverflow.com/questions/19534896/enforcing-python-version-in-setup-py +import sys +if sys.version_info < (2,7): + sys.exit('Sorry, Python < 2.7 is not supported') + +import os + +from setuptools import setup +from setuptools.extension import Extension + +try: + from Cython.Build import cythonize +except ImportError: + sys.exit("Cython not found. Cython is needed to build the extension modules.") + + +######################################################### +# Definitions +######################################################### + +# Define our base set of compiler and linker flags. +# +# This is geared toward x86_64, see +# https://gcc.gnu.org/onlinedocs/gcc-4.6.4/gcc/i386-and-x86_002d64-Options.html +# +# Customize these as needed. +# +# Note that -O3 may sometimes cause mysterious problems, so we limit ourselves to -O2. + +# Modules involving numerical computations +# +extra_compile_args_math_optimized = ['-march=native', '-O2', '-msse', '-msse2', '-mfma', '-mfpmath=sse'] +extra_compile_args_math_debug = ['-march=native', '-O0', '-g'] +extra_link_args_math_optimized = [] +extra_link_args_math_debug = [] + +# Modules that do not involve numerical computations +# +extra_compile_args_nonmath_optimized = ['-O2'] +extra_compile_args_nonmath_debug = ['-O0', '-g'] +extra_link_args_nonmath_optimized = [] +extra_link_args_nonmath_debug = [] + +# Additional flags to compile/link with OpenMP +# +openmp_compile_args = ['-fopenmp'] +openmp_link_args = ['-fopenmp'] + + +######################################################### +# Helpers +######################################################### + +# Make absolute cimports work. +# +# See +# https://github.com/cython/cython/wiki/PackageHierarchy +# +my_include_dirs = ["."] + + +# Choose the base set of compiler and linker flags. +# +if build_type == 'optimized': + my_extra_compile_args_math = extra_compile_args_math_optimized + my_extra_compile_args_nonmath = extra_compile_args_nonmath_optimized + my_extra_link_args_math = extra_link_args_math_optimized + my_extra_link_args_nonmath = extra_link_args_nonmath_optimized + my_debug = False + print( "build configuration selected: optimized" ) +elif build_type == 'debug': + my_extra_compile_args_math = extra_compile_args_math_debug + my_extra_compile_args_nonmath = extra_compile_args_nonmath_debug + my_extra_link_args_math = extra_link_args_math_debug + my_extra_link_args_nonmath = extra_link_args_nonmath_debug + my_debug = True + print( "build configuration selected: debug" ) +else: + raise ValueError("Unknown build configuration '%s'; valid: 'optimized', 'debug'" % (build_type)) + + +def declare_cython_extension(extName, use_math=False, use_openmp=False): + """Declare a Cython extension module for setuptools. + +Parameters: + extName : str + Absolute module name, e.g. use `mylibrary.mymodule.submodule` + for the Cython source file `mylibrary/mymodule/submodule.pyx`. + + use_math : bool + If True, set math flags and link with ``libm``. + + use_openmp : bool + If True, compile and link with OpenMP. + +Return value: + Extension object + that can be passed to ``setuptools.setup``. +""" + extPath = extName.replace(".", os.path.sep)+".pyx" + + if use_math: + compile_args = list(my_extra_compile_args_math) # copy + link_args = list(my_extra_link_args_math) + libraries = ["m"] # link libm; this is a list of library names without the "lib" prefix + else: + compile_args = list(my_extra_compile_args_nonmath) + link_args = list(my_extra_link_args_nonmath) + libraries = None # value if no libraries, see setuptools.extension._Extension + + # OpenMP + if use_openmp: + compile_args.insert( 0, openmp_compile_args ) + link_args.insert( 0, openmp_link_args ) + + # See + # http://docs.cython.org/src/tutorial/external.html + # + # on linking libraries to your Cython extensions. + # + return Extension( extName, + [extPath], + extra_compile_args=compile_args, + extra_link_args=link_args, + libraries=libraries + ) + + +######################################################### +# Set up modules +######################################################### + +# declare Cython extension modules here +# +ext_module_cythonmodule = declare_cython_extension( "cython_module", use_math=False, use_openmp=False ) + +# this is mainly to allow a manual logical ordering of the declared modules +# +cython_ext_modules = [ext_module_cythonmodule] + +# Call cythonize() explicitly, as recommended in the Cython documentation. See +# http://cython.readthedocs.io/en/latest/src/reference/compilation.html#compiling-with-distutils +# +# This will favor Cython's own handling of '.pyx' sources over that provided by setuptools. +# +# Note that my_ext_modules is just a list of Extension objects. We could add any C sources (not coming from Cython modules) here if needed. +# cythonize() just performs the Cython-level processing, and returns a list of Extension objects. +# +my_ext_modules = cythonize( cython_ext_modules, include_path=my_include_dirs, gdb_debug=my_debug ) + + +######################################################### +# Call setup() +######################################################### + +setup( + # See + # http://setuptools.readthedocs.io/en/latest/setuptools.html + # + setup_requires = ["cython", "numpy"], + install_requires = ["numpy"], + + # All extension modules (list of Extension objects) + # + ext_modules = my_ext_modules +) +