Updates for latest ots

- Update setup.cfg
- use latest ots version & hash
- Update setup.py
- tweaks for changes in recent ots build scheme
- completely revamp Extension build
- build woff2 as part of Extension, don't try to use meson build .a files
- try to make build scheme a little less fragile (needs work)
- some updates to GitHub Workflow
- update tests

.github/workflows/release.yml

@@ -46,7 +46,7 @@ jobs:
       id: build_wheels
       uses: RalfG/python-wheels-manylinux-build@v0.2.2-manylinux2010_x86_64
       with:
-        python-versions: 'cp36-cp36m cp37-cp37m cp38-cp38'
+        python-versions: 'cp36-cp36m cp37-cp37m cp38-cp38 cp39-cp39'
         build-requirements: 'meson ninja'
         system-packages: 'zlib-devel'
 

build.py

@@ -10,11 +10,13 @@
 import errno
 import argparse
 
-
 ROOT = Path(__file__).parent.resolve()
-BUILD_ROOT = ROOT / "build"
+BUILD_ROOT = ROOT / "src" / "ots" / "build"
 BUILD_DIR = BUILD_ROOT / "meson"
-SRC_DIR = ROOT.joinpath("src", "ots")
+SRC_DIR = ROOT / "src"
+OTS_SRC_DIR = SRC_DIR / "ots"
+SUB_DIR = OTS_SRC_DIR / "subprojects"
+
 
 if 'manylinux' in os.environ.get("AUDITWHEEL_PLAT", ''):
     os.environ["PATH"] += os.pathsep + os.path.dirname(sys.executable)
@@ -31,7 +33,7 @@
     "--strip",
     "-Ddebug=true",
     str(BUILD_DIR),
-    str(SRC_DIR),
+    str(OTS_SRC_DIR),
 ]
 
 NINJA_CMD = [TOOLS["ninja"], "-C", str(BUILD_DIR)]
@@ -50,6 +52,7 @@ def check_tools():
 
 
 def configure(reconfigure=False):
+    print(f"build.py: Running {' '.join(MESON_CMD)}")
     if not (BUILD_DIR / "build.ninja").exists():
         subprocess.run(MESON_CMD, check=True, env=os.environ)
     elif reconfigure:
@@ -59,6 +62,7 @@ def configure(reconfigure=False):
 
 
 def make(*targets, clean=False):
+    print(f"build.py: Running {' '.join(NINJA_CMD)}")
     targets = list(targets)
     if clean:
         subprocess.run(NINJA_CMD + ["-t", "clean"] + targets,

requirements.txt

@@ -1,7 +1,7 @@
 black
 cpplint
+cmake
 flake8
 pytest
 meson
 ninja
-Brotli

setup.cfg

@@ -1,6 +1,6 @@
 [download]
 ; OpenType Sanitizer version that is downloaded from setup.py
-version = 8.1.1
+version = 8.1.3
 ; expected SHA-256 of the downloaded tarball. E.g. you can calculate it with:
 ; $ shasum -a 256 ots-X.X.X.tar.xz
-sha256 = 8b912fae494228c5e308225ff15d1aaaf026587661c3c2d20676adb28795ff48
+sha256 = d26ba3956568163ea9c34c8c6f46ba15e4659cdd558b998b5f353070159f0a39

setup.py

@@ -1,3 +1,4 @@
+from build import SRC_DIR, OTS_SRC_DIR
 from distutils.dir_util import mkpath, remove_tree
 from distutils import log
 import io
@@ -11,6 +12,83 @@
 
 PY = sys.executable
 
+BUILD_DIR = OTS_SRC_DIR / "build" / "meson"
+BUILD_SUB_DIR = BUILD_DIR / "subprojects"
+SRC_SUB_DIR = OTS_SRC_DIR / "subprojects"
+
+# TODO: try to make this scheme less fragile/less dependent on hard-coded tags
+# to avoid changing it with every new ots release (although it seems like every
+# release of ots has something that causes this build to break anyway so it's
+# not really that urgent. We just have to adjust every release.
+BROTLI_TAG = "1.0.7"
+LZ4_TAG = "1.9.3"
+WOFF2_TAG = "a0d0ed7da27b708c0a4e96ad7a998bddc933c06e"
+
+
+def _get_extra_objects():
+    """
+    Create the list of 'extra_ojects' for building the extension. This is done
+    in a way to try to be forward-compatible with changes to ots's
+    dependencies, but could still break if those dependencies change the names
+    of the static libs generated.
+    """
+    # libots
+    xo = [BUILD_DIR / "libots.a"]
+
+    # brotli
+    # NOTE: decoder needs to come before common!
+    xo.append(BUILD_SUB_DIR / f"brotli-{BROTLI_TAG}" / "libbrotli_decoder.a")
+    xo.append(BUILD_SUB_DIR / f"brotli-{BROTLI_TAG}" / "libbrotli_common.a")
+
+    # lz4
+    xo.append(BUILD_SUB_DIR / f"lz4-{LZ4_TAG}" / "contrib" / "meson" / "meson" / "lib" / "liblz4.a")  # noqa: E501
+
+    # woff2 -- skipped for now, building as part of Extension
+    # xo.append(BUILD_SUB_DIR / f"woff2-{WOFF2_TAG}" / "libwoff2_decoder.a")
+    # xo.append(BUILD_SUB_DIR / f"woff2-{WOFF2_TAG}" / "libwoff2_common.a")
+
+    return [str(p) for p in xo]
+
+
+def _get_include_dirs():
+    """
+    Create the list of 'include_dirs' for building the Extension.
+    """
+    ip = [
+        BUILD_DIR,
+        SRC_DIR / "_pyots",
+        OTS_SRC_DIR / "include",
+        ]
+
+    # lz4
+    ip.append(SRC_SUB_DIR / f"lz4-{LZ4_TAG}" / "lib")
+
+    # brotli
+    ip.append(SRC_SUB_DIR / f"brotli-{BROTLI_TAG}" / "c" / "include")
+
+    # woff2
+    ip.append(SRC_SUB_DIR / f"woff2-{WOFF2_TAG}" / "include")
+
+    return [str(p) for p in ip]
+
+
+def _get_sources():
+    """
+    Create the list of 'sources' for building the Extension.
+    """
+    sp = [
+        SRC_DIR / "_pyots" / "bindings.cpp",
+    ]
+
+    # woff2 sources
+    sp.append(SRC_SUB_DIR / f"woff2-{WOFF2_TAG}" / "src" / "table_tags.cc")
+    sp.append(SRC_SUB_DIR / f"woff2-{WOFF2_TAG}" / "src" / "variable_length.cc")  # noqa: E501
+    sp.append(SRC_SUB_DIR / f"woff2-{WOFF2_TAG}" / "src" / "woff2_common.cc")
+    sp.append(SRC_SUB_DIR / f"woff2-{WOFF2_TAG}" / "src" / "woff2_dec.cc")
+    sp.append(SRC_SUB_DIR / f"woff2-{WOFF2_TAG}" / "src" / "woff2_out.cc")
+
+    return [str(p) for p in sp]
+
 
 class BuildStaticLibs(Command):
     """
@@ -34,7 +112,6 @@ class BuildPy(build_py.build_py):
     """
     Custom python build command. Calls 'build_static' prior to Python build.
     """
-
     def run(self):
         self.run_command('build_static')
         build_py.build_py.run(self)
@@ -156,6 +233,10 @@ def run(self):
                 with open("src/ots/meson.build", 'r') as f:
                     meson = f.read()
 
+                    # back up original
+                    with open("src/ots/meson.build.orig", 'w') as f_out:
+                        f_out.write(meson)
+
                 # update default_options
                 meson = re.sub(
                     r"default_options : \[(.+)],",
@@ -184,20 +265,11 @@ def run(self):
 
 pyots_mod = Extension(
     name='_pyots',
-    extra_compile_args=['-std=c++11'],
-    extra_objects=[
-        'build/meson/libots.a',
-        'build/meson/libwoff2.a',
-        'build/meson/libbrotli.a',
-        'build/meson/liblz4.a'],
     libraries=['z'],
-    include_dirs=['build/meson/',
-                  'src/ots/include',
-                  'src/ots/include/src',
-                  'src/ots/third_party/brotli/c/include',
-                  'src/ots/third_party/lz4/lib',
-                  'src/ots/third_party/woff2/include/woff2'],
-    sources=['src/_pyots/bindings.cpp'],
+    extra_compile_args=['-fPIC', '-std=c++11'],
+    extra_objects=_get_extra_objects(),
+    include_dirs=_get_include_dirs(),
+    sources=_get_sources(),
 )
 
 with io.open("README.md", encoding="utf-8") as readme:
@@ -231,4 +303,5 @@ def run(self):
     url='https://github.com/adobe-type-tools/pyots',
     use_scm_version=True,
     setup_requires=['setuptools_scm'],
+    zip_safe=False,
 )

tests/test_ots_suite.py

@@ -62,4 +62,5 @@ def test_ots_suite():
     'fuzzing/aca5f8ef7bc0754b0b6fd7a1abd4c69ca7801780.ttf',
     'fuzzing/8240789f6d12d4cfc4b5e8e6f246c3701bcf861f.otf',
     'fuzzing/3857535d8c0d2bfeab7ee2cd6ba5e39bcb4abd90.ttf',
+    'fuzzing/adb242cbc61b3ca428903e397a2c9dcf97fe3042.ttf',
 }