Add mypy linter step

.github/workflows/lint.yml

@@ -4,7 +4,7 @@ on:
   pull_request:
 
 jobs:
-  lint:
+  flake8:
     runs-on: ubuntu-latest
 
     steps:
@@ -15,4 +15,22 @@ jobs:
       - uses: TrueBrain/actions-flake8@v2
         with:
           flake8_version: 6.0.0
-          plugins: flake8-isort==6.1.1 flake8-quotes==3.4.0 flake8-commas==4.0.0 
+          plugins: flake8-isort==6.1.1 flake8-quotes==3.4.0 flake8-commas==4.0.0
+
+  mypy:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.9
+
+      - run: pip install mypy==1.14.1 boto3-stubs
+
+      - run: |
+          mypy \
+            --non-interactive \
+            --install-types \
+            --check-untyped-defs \
+            mandible

mandible/metadata_mapper/directive/reformatted.py

@@ -4,8 +4,9 @@
 
 from mandible.metadata_mapper.exception import MetadataMapperError
 from mandible.metadata_mapper.format import FORMAT_REGISTRY
+from mandible.metadata_mapper.types import Key
 
-from .directive import Key, TemplateDirective, get_key
+from .directive import TemplateDirective, get_key
 
 
 @dataclass

mandible/metadata_mapper/exception.py

@@ -1,3 +1,6 @@
+from typing import Optional
+
+
 class MetadataMapperError(Exception):
     """A generic error raised by the MetadataMapper"""
 
@@ -8,7 +11,7 @@ def __init__(self, msg: str):
 class TemplateError(MetadataMapperError):
     """An error that occurred while processing the metadata template."""
 
-    def __init__(self, msg: str, debug_path: str = None):
+    def __init__(self, msg: str, debug_path: Optional[str] = None):
         super().__init__(msg)
         self.debug_path = debug_path
 
@@ -26,7 +29,7 @@ class ContextValueError(MetadataMapperError):
     def __init__(
         self,
         msg: str,
-        source_name: str = None,
+        source_name: Optional[str] = None,
     ):
         super().__init__(msg)
         self.source_name = source_name

mandible/metadata_mapper/format/__init__.py

@@ -11,12 +11,12 @@
 try:
     from .h5 import H5
 except ImportError:
-    from .format import H5
+    from .format import H5  # type: ignore
 
 try:
     from .xml import Xml
 except ImportError:
-    from .format import Xml
+    from .format import Xml  # type: ignore
 
 
 __all__ = (

mandible/metadata_mapper/format/format.py

@@ -3,9 +3,9 @@
 import re
 import zipfile
 from abc import ABC, abstractmethod
-from collections.abc import Iterable
+from collections.abc import Generator, Iterable
 from dataclasses import dataclass
-from typing import IO, Any, TypeVar
+from typing import IO, Any, Generic, TypeVar
 
 from mandible import jsonpath
 from mandible.metadata_mapper.key import RAISE_EXCEPTION, Key
@@ -50,7 +50,7 @@ def get_value(self, file: IO[bytes], key: Key) -> Any:
 
 
 @dataclass
-class FileFormat(Format, ABC, register=False):
+class FileFormat(Format, Generic[T], ABC, register=False):
     """A Format for querying files from a standard data file.
 
     Simple, single format data types such as 'json' that can be queried
@@ -76,7 +76,7 @@ def get_value(self, file: IO[bytes], key: Key) -> Any:
         with self.parse_data(file) as data:
             return self._eval_key_wrapper(data, key)
 
-    def _eval_key_wrapper(self, data, key: Key) -> Any:
+    def _eval_key_wrapper(self, data: T, key: Key) -> Any:
         try:
             return self.eval_key(data, key)
         except KeyError as e:
@@ -116,7 +116,7 @@ def eval_key(data: T, key: Key) -> Any:
 
 
 @dataclass
-class _PlaceholderBase(FileFormat, register=False):
+class _PlaceholderBase(FileFormat[None], register=False):
     """
     Base class for defining placeholder implementations for classes that
     require extra dependencies to be installed
@@ -128,12 +128,14 @@ def __init__(self, dep: str):
         )
 
     @staticmethod
-    def parse_data(file: IO[bytes]) -> contextlib.AbstractContextManager[T]:
-        pass
+    def parse_data(file: IO[bytes]) -> contextlib.AbstractContextManager[None]:
+        # __init__ always raises
+        raise RuntimeError("Unreachable!")
 
     @staticmethod
-    def eval_key(data: T, key: Key):
-        pass
+    def eval_key(data: None, key: Key):
+        # __init__ always raises
+        raise RuntimeError("Unreachable!")
 
 
 @dataclass
@@ -151,10 +153,10 @@ def __init__(self):
 # Define formats that don't require extra dependencies
 
 @dataclass
-class Json(FileFormat):
+class Json(FileFormat[dict]):
     @staticmethod
     @contextlib.contextmanager
-    def parse_data(file: IO[bytes]) -> dict:
+    def parse_data(file: IO[bytes]) -> Generator[dict]:
         yield json.load(file)
 
     @staticmethod
@@ -238,12 +240,12 @@ def _matches_filters(self, zipinfo: zipfile.ZipInfo) -> bool:
 
 
 @dataclass
-class ZipInfo(FileFormat):
+class ZipInfo(FileFormat[dict]):
     """Query Zip headers and directory information."""
 
     @staticmethod
     @contextlib.contextmanager
-    def parse_data(file: IO[bytes]) -> dict:
+    def parse_data(file: IO[bytes]) -> Generator[dict]:
         with zipfile.ZipFile(file, "r") as zf:
             yield {
                 "infolist": [

mandible/metadata_mapper/format/h5.py

@@ -11,13 +11,13 @@
 
 
 @dataclass
-class H5(FileFormat):
+class H5(FileFormat[Any]):
     @staticmethod
     def parse_data(file: IO[bytes]) -> contextlib.AbstractContextManager[Any]:
         return h5py.File(file, "r")
 
     @staticmethod
-    def eval_key(data, key: Key) -> Any:
+    def eval_key(data: Any, key: Key) -> Any:
         return normalize(data[key.key][()])
 
 

mandible/metadata_mapper/format/xml.py

@@ -1,4 +1,5 @@
 import contextlib
+from collections.abc import Generator
 from dataclasses import dataclass
 from typing import IO, Any
 
@@ -10,14 +11,14 @@
 
 
 @dataclass
-class Xml(FileFormat):
+class Xml(FileFormat[etree._ElementTree]):
     @staticmethod
     @contextlib.contextmanager
-    def parse_data(file: IO[bytes]) -> Any:
+    def parse_data(file: IO[bytes]) -> Generator[etree._ElementTree]:
         yield etree.parse(file)
 
     @staticmethod
-    def eval_key(data: etree.ElementTree, key: Key) -> Any:
+    def eval_key(data: etree._ElementTree, key: Key) -> Any:
         nsmap = data.getroot().nsmap
         elements = data.xpath(key.key, namespaces=nsmap)
         values = [element.text for element in elements]

mandible/metadata_mapper/mapper.py

@@ -16,7 +16,7 @@ class MetadataMapper:
     def __init__(
         self,
         template: Template,
-        source_provider: SourceProvider = None,
+        source_provider: Optional[SourceProvider] = None,
         *,
         directive_marker: str = "@",
     ):
@@ -91,14 +91,21 @@ def _replace_template(
         template: Template,
         sources: dict[str, Source],
         debug_path: str = "$",
-    ):
+    ) -> Template:
         if isinstance(template, dict):
             directive_name = self._get_directive_name(
                 template,
                 debug_path,
             )
             if directive_name is not None:
                 debug_path = f"{debug_path}.{directive_name}"
+                directive_config = template[directive_name]
+                if not isinstance(directive_config, dict):
+                    raise TemplateError(
+                        "directive body should be type 'dict' not "
+                        f"{repr(directive_config.__class__.__name__)}",
+                        debug_path,
+                    )
                 directive = self._get_directive(
                     directive_name,
                     context,
@@ -110,7 +117,7 @@ def _replace_template(
                             sources,
                             debug_path=f"{debug_path}.{k}",
                         )
-                        for k, v in template[directive_name].items()
+                        for k, v in directive_config.items()
                     },
                     debug_path,
                 )
@@ -146,7 +153,7 @@ def _replace_template(
 
     def _get_directive_name(
         self,
-        value: dict,
+        value: dict[str, Template],
         debug_path: str,
     ) -> Optional[str]:
         directive_names = [

mandible/metadata_mapper/source_provider.py

@@ -11,7 +11,7 @@
 
 T = TypeVar("T")
 
-REGISTRY_TYPE_MAP = {
+REGISTRY_TYPE_MAP: dict[str, dict[str, Any]] = {
     "Format": FORMAT_REGISTRY,
     "Source": SOURCE_REGISTRY,
     "Storage": STORAGE_REGISTRY,

mandible/metadata_mapper/storage/__init__.py

@@ -11,12 +11,12 @@
 try:
     from .cmr_query import CmrQuery
 except ImportError:
-    from .storage import CmrQuery
+    from .storage import CmrQuery  # type: ignore
 
 try:
     from .http_request import HttpRequest
 except ImportError:
-    from .storage import HttpRequest
+    from .storage import HttpRequest  # type: ignore
 
 
 __all__ = (

mandible/metadata_mapper/storage/storage.py

@@ -47,7 +47,8 @@ def __init__(self, dep: str):
         )
 
     def open_file(self, context: Context) -> IO[bytes]:
-        pass
+        # __init__ always raises
+        raise RuntimeError("Unreachable!")
 
 
 @dataclass

pyproject.toml

@@ -52,3 +52,14 @@ markers = [
 
 [tool.isort]
 profile = "black"
+
+[tool.mypy]
+# warn_redundant_casts = true
+# warn_unused_ignores = true
+# warn_unreachable = true
+# strict_equality = true
+# check_untyped_defs = true
+# install_types = true
+# non_interactive = true
+# pretty = true
+disable_error_code = ["import-untyped"]

tests/integration_tests/test_metadata_mapper.py

@@ -287,6 +287,26 @@ def test_invalid_directive(context):
         mapper.get_metadata(context)
 
 
+def test_invalid_directive_config_type(context):
+    mapper = MetadataMapper(
+        template={
+            "foo": {
+                "@mapped": 100,
+            },
+        },
+        source_provider=ConfigSourceProvider({}),
+    )
+
+    with pytest.raises(
+        MetadataMapperError,
+        match=(
+            r"failed to process template at \$\.foo\.@mapped: "
+            "directive body should be type 'dict' not 'int'"
+        ),
+    ):
+        mapper.get_metadata(context)
+
+
 def test_multiple_directives(context):
     mapper = MetadataMapper(
         template={