From a20fd8c5364e4494277e1a5ce51052b7d6605e47 Mon Sep 17 00:00:00 2001 From: Josh Hadley <51727949+josh-hadley@users.noreply.github.com> Date: Sat, 13 Feb 2021 18:19:12 -0800 Subject: [PATCH] Updates for latest ots - Update setup.cfg - use latest ots version & hash - Update setup.py - tweaks for changes in recent ots build scheme - completely revamp Extension build - build woff2 as part of Extension, don't try to use meson build .a files - try to make build scheme a little less fragile (needs work) - some updates to GitHub Workflow - update tests --- .github/workflows/release.yml | 2 +- build.py | 12 ++-- requirements.txt | 2 +- setup.cfg | 4 +- setup.py | 101 +++++++++++++++++++++++++++++----- tests/test_ots_suite.py | 1 + 6 files changed, 100 insertions(+), 22 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 94e8da2..99b1d49 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -46,7 +46,7 @@ jobs: id: build_wheels uses: RalfG/python-wheels-manylinux-build@v0.2.2-manylinux2010_x86_64 with: - python-versions: 'cp36-cp36m cp37-cp37m cp38-cp38' + python-versions: 'cp36-cp36m cp37-cp37m cp38-cp38 cp39-cp39' build-requirements: 'meson ninja' system-packages: 'zlib-devel' diff --git a/build.py b/build.py index a174b47..37526db 100755 --- a/build.py +++ b/build.py @@ -10,11 +10,13 @@ import errno import argparse - ROOT = Path(__file__).parent.resolve() -BUILD_ROOT = ROOT / "build" +BUILD_ROOT = ROOT / "src" / "ots" / "build" BUILD_DIR = BUILD_ROOT / "meson" -SRC_DIR = ROOT.joinpath("src", "ots") +SRC_DIR = ROOT / "src" +OTS_SRC_DIR = SRC_DIR / "ots" +SUB_DIR = OTS_SRC_DIR / "subprojects" + if 'manylinux' in os.environ.get("AUDITWHEEL_PLAT", ''): os.environ["PATH"] += os.pathsep + os.path.dirname(sys.executable) @@ -31,7 +33,7 @@ "--strip", "-Ddebug=true", str(BUILD_DIR), - str(SRC_DIR), + str(OTS_SRC_DIR), ] NINJA_CMD = [TOOLS["ninja"], "-C", str(BUILD_DIR)] @@ -50,6 +52,7 @@ def check_tools(): def configure(reconfigure=False): + print(f"build.py: Running {' '.join(MESON_CMD)}") if not (BUILD_DIR / "build.ninja").exists(): subprocess.run(MESON_CMD, check=True, env=os.environ) elif reconfigure: @@ -59,6 +62,7 @@ def configure(reconfigure=False): def make(*targets, clean=False): + print(f"build.py: Running {' '.join(NINJA_CMD)}") targets = list(targets) if clean: subprocess.run(NINJA_CMD + ["-t", "clean"] + targets, diff --git a/requirements.txt b/requirements.txt index 4341013..b040907 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ black cpplint +cmake flake8 pytest meson ninja -Brotli diff --git a/setup.cfg b/setup.cfg index 9d38f13..309357e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [download] ; OpenType Sanitizer version that is downloaded from setup.py -version = 8.1.1 +version = 8.1.3 ; expected SHA-256 of the downloaded tarball. E.g. you can calculate it with: ; $ shasum -a 256 ots-X.X.X.tar.xz -sha256 = 8b912fae494228c5e308225ff15d1aaaf026587661c3c2d20676adb28795ff48 +sha256 = d26ba3956568163ea9c34c8c6f46ba15e4659cdd558b998b5f353070159f0a39 diff --git a/setup.py b/setup.py index c3e7a47..26dc96a 100644 --- a/setup.py +++ b/setup.py @@ -1,3 +1,4 @@ +from build import SRC_DIR, OTS_SRC_DIR from distutils.dir_util import mkpath, remove_tree from distutils import log import io @@ -11,6 +12,83 @@ PY = sys.executable +BUILD_DIR = OTS_SRC_DIR / "build" / "meson" +BUILD_SUB_DIR = BUILD_DIR / "subprojects" +SRC_SUB_DIR = OTS_SRC_DIR / "subprojects" + +# TODO: try to make this scheme less fragile/less dependent on hard-coded tags +# to avoid changing it with every new ots release (although it seems like every +# release of ots has something that causes this build to break anyway so it's +# not really that urgent. We just have to adjust every release. +BROTLI_TAG = "1.0.7" +LZ4_TAG = "1.9.3" +WOFF2_TAG = "a0d0ed7da27b708c0a4e96ad7a998bddc933c06e" + + +def _get_extra_objects(): + """ + Create the list of 'extra_ojects' for building the extension. This is done + in a way to try to be forward-compatible with changes to ots's + dependencies, but could still break if those dependencies change the names + of the static libs generated. + """ + # libots + xo = [BUILD_DIR / "libots.a"] + + # brotli + # NOTE: decoder needs to come before common! + xo.append(BUILD_SUB_DIR / f"brotli-{BROTLI_TAG}" / "libbrotli_decoder.a") + xo.append(BUILD_SUB_DIR / f"brotli-{BROTLI_TAG}" / "libbrotli_common.a") + + # lz4 + xo.append(BUILD_SUB_DIR / f"lz4-{LZ4_TAG}" / "contrib" / "meson" / "meson" / "lib" / "liblz4.a") # noqa: E501 + + # woff2 -- skipped for now, building as part of Extension + # xo.append(BUILD_SUB_DIR / f"woff2-{WOFF2_TAG}" / "libwoff2_decoder.a") + # xo.append(BUILD_SUB_DIR / f"woff2-{WOFF2_TAG}" / "libwoff2_common.a") + + return [str(p) for p in xo] + + +def _get_include_dirs(): + """ + Create the list of 'include_dirs' for building the Extension. + """ + ip = [ + BUILD_DIR, + SRC_DIR / "_pyots", + OTS_SRC_DIR / "include", + ] + + # lz4 + ip.append(SRC_SUB_DIR / f"lz4-{LZ4_TAG}" / "lib") + + # brotli + ip.append(SRC_SUB_DIR / f"brotli-{BROTLI_TAG}" / "c" / "include") + + # woff2 + ip.append(SRC_SUB_DIR / f"woff2-{WOFF2_TAG}" / "include") + + return [str(p) for p in ip] + + +def _get_sources(): + """ + Create the list of 'sources' for building the Extension. + """ + sp = [ + SRC_DIR / "_pyots" / "bindings.cpp", + ] + + # woff2 sources + sp.append(SRC_SUB_DIR / f"woff2-{WOFF2_TAG}" / "src" / "table_tags.cc") + sp.append(SRC_SUB_DIR / f"woff2-{WOFF2_TAG}" / "src" / "variable_length.cc") # noqa: E501 + sp.append(SRC_SUB_DIR / f"woff2-{WOFF2_TAG}" / "src" / "woff2_common.cc") + sp.append(SRC_SUB_DIR / f"woff2-{WOFF2_TAG}" / "src" / "woff2_dec.cc") + sp.append(SRC_SUB_DIR / f"woff2-{WOFF2_TAG}" / "src" / "woff2_out.cc") + + return [str(p) for p in sp] + class BuildStaticLibs(Command): """ @@ -34,7 +112,6 @@ class BuildPy(build_py.build_py): """ Custom python build command. Calls 'build_static' prior to Python build. """ - def run(self): self.run_command('build_static') build_py.build_py.run(self) @@ -156,6 +233,10 @@ def run(self): with open("src/ots/meson.build", 'r') as f: meson = f.read() + # back up original + with open("src/ots/meson.build.orig", 'w') as f_out: + f_out.write(meson) + # update default_options meson = re.sub( r"default_options : \[(.+)],", @@ -184,20 +265,11 @@ def run(self): pyots_mod = Extension( name='_pyots', - extra_compile_args=['-std=c++11'], - extra_objects=[ - 'build/meson/libots.a', - 'build/meson/libwoff2.a', - 'build/meson/libbrotli.a', - 'build/meson/liblz4.a'], libraries=['z'], - include_dirs=['build/meson/', - 'src/ots/include', - 'src/ots/include/src', - 'src/ots/third_party/brotli/c/include', - 'src/ots/third_party/lz4/lib', - 'src/ots/third_party/woff2/include/woff2'], - sources=['src/_pyots/bindings.cpp'], + extra_compile_args=['-fPIC', '-std=c++11'], + extra_objects=_get_extra_objects(), + include_dirs=_get_include_dirs(), + sources=_get_sources(), ) with io.open("README.md", encoding="utf-8") as readme: @@ -231,4 +303,5 @@ def run(self): url='https://github.com/adobe-type-tools/pyots', use_scm_version=True, setup_requires=['setuptools_scm'], + zip_safe=False, ) diff --git a/tests/test_ots_suite.py b/tests/test_ots_suite.py index 866b264..d338559 100644 --- a/tests/test_ots_suite.py +++ b/tests/test_ots_suite.py @@ -62,4 +62,5 @@ def test_ots_suite(): 'fuzzing/aca5f8ef7bc0754b0b6fd7a1abd4c69ca7801780.ttf', 'fuzzing/8240789f6d12d4cfc4b5e8e6f246c3701bcf861f.otf', 'fuzzing/3857535d8c0d2bfeab7ee2cd6ba5e39bcb4abd90.ttf', + 'fuzzing/adb242cbc61b3ca428903e397a2c9dcf97fe3042.ttf', }