Add tests for *.data/ directory file copies (#91)

doc/guide/limitations.md

@@ -13,14 +13,6 @@ in dependencies and probably will not occur that often in practice.
 We handle this by simplying changinge `===` to `==` but
 since this will often not work we also issue a warning.
 
-## Wheel data directories not supported
-
-Wheels with `*.data` directies are not fully supported.
-Any such data directories will not be copied.
-
-This will be addressed in a future release
-(see [issue 91](https://github.com/zuzukin/whl2conda/issues/91))
-
 ## Cannot convert from sdist
 
 Conversion from python sdist distributions is not currently supported. 

test/api/test_external.py

@@ -1,4 +1,4 @@
-#  Copyright 2023 Christopher Barber
+#  Copyright 2023-2024 Christopher Barber
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
@@ -73,6 +73,30 @@ def test_pypi_colorama(test_case: ConverterTestCaseFactory):
     ).build()
 
 
+@pytest.mark.external
+def test_argcomplete(test_case: ConverterTestCaseFactory):
+    """
+    Test argcomplete package
+
+    This package uses *.data/scripts (at least as of 3.2.1)
+    """
+    test_case(
+        "pypi:argcomplete ==3.2.1",
+    ).build()
+
+
+@pytest.mark.external
+def test_linkchecker(test_case: ConverterTestCaseFactory):
+    """
+    Test linkchecker package
+
+    This package uses *.data/data/
+    """
+    test_case(
+        "pypi:linkchecker ==10.4.0",
+    ).build()
+
+
 @pytest.mark.external
 def test_pypi_orix(test_case: ConverterTestCaseFactory) -> None:
     """

test/api/validator.py

@@ -155,8 +155,11 @@ def _validate_unpacked(self) -> None:
         self._validate_index(info_dir)
         self._validate_link(info_dir)
         self._validate_paths(info_dir)
+        self._validate_file_copy()
         self._validate_hash_input(info_dir)
 
+        # TODO - validate *.data/ files
+
         self._validate_dist_info()
 
     # pylint: disable=too-many-locals,too-many-branches
@@ -304,6 +307,9 @@ def _validate_index(self, info_dir: Path) -> None:
         self._validate_dependencies(index["depends"])
 
     def _validate_dependencies(self, dependencies: Sequence[str]) -> None:
+        """
+        Validates dependencies
+        """
         output_depends = set(dependencies)
         expected_depends: set[str] = set()
 
@@ -340,6 +346,9 @@ def _validate_dependencies(self, dependencies: Sequence[str]) -> None:
             )
 
     def _validate_link(self, info_dir: Path) -> None:
+        """
+        Validates the contents of the info/link.json file
+        """
         link_file = info_dir / "link.json"
         assert link_file.is_file()
         jobj = json.loads(link_file.read_text("utf8"))
@@ -354,6 +363,17 @@ def _validate_link(self, info_dir: Path) -> None:
         assert entry_points == expected_entry_points
 
     def _validate_paths(self, info_dir: Path) -> None:
+        """
+        Validate info/files and info/paths.json files
+
+        Performs the following checks:
+        - checks correspondence betweens files and paths.json entries
+        - checks size an hash values of paths.json entries
+        - verifies that file/paths.json entries exist in the conda distribution
+
+        Args:
+            info_dir: location of info/ conda subdirectory
+        """
         rel_files = info_dir.joinpath("files").read_text().splitlines()
         pkg_dir = self._unpacked_conda
         files: set[Path] = set(
@@ -384,9 +404,70 @@ def _validate_paths(self, info_dir: Path) -> None:
         all_files = set(f for f in pkg_dir.glob("**/*") if f.is_file())
         info_files = set(info_dir.glob("**/*"))
         non_info_files = all_files - info_files
-
         assert files == non_info_files
 
+    def _validate_file_copy(self) -> None:
+        """
+        Validates that files have been copied from wheel into corresponding location in conda package
+        """
+        pkg_dir = self._unpacked_conda
+        wheel_dir = self._unpacked_wheel
+        assert wheel_dir.is_dir()
+        all_wheel_files = set(wheel_dir.glob("**/*"))
+        wheel_distinfo_files = set(wheel_dir.glob("*.dist-info/**/*"))
+        wheel_data_files = set(wheel_dir.glob("*.data/**/*"))
+        wheel_site_package_files = (
+            all_wheel_files - wheel_distinfo_files
+        ) - wheel_data_files
+
+        # Check that all package files were copied into site-packages/
+        site_packages_dir = pkg_dir / "site-packages"
+        for wheel_file in wheel_site_package_files:
+            if wheel_file.is_dir():
+                continue
+            rel_path = wheel_file.relative_to(wheel_dir)
+            conda_file = site_packages_dir / rel_path
+            assert (
+                conda_file.exists()
+            ), f"{conda_file.relative_to(pkg_dir)} does not exist"
+            assert wheel_file.read_bytes() == conda_file.read_bytes()
+
+        # Check that all *.data/data/ files get copied into top level
+        wheel_data_data_files = set(wheel_dir.glob("*.data/data/**/*"))
+        for wheel_file in wheel_data_data_files:
+            if wheel_file.is_dir():
+                continue
+            rel_path = wheel_file.relative_to(wheel_dir)
+            rel_path = Path(
+                *rel_path.parts[2:]
+            )  # strip *.data/data/ from head of rel path
+            conda_file = pkg_dir / rel_path
+            assert (
+                conda_file.exists()
+            ), f"{conda_file.relative_to(pkg_dir)} does not exist"
+            # NOTE: in theory this could fail if there is more than one *.data dir that
+            #  specify the same file path with different contents, but in practice we do not
+            #  expect that to ever happen.
+            assert wheel_file.read_bytes() == conda_file.read_bytes()
+
+        # Check that all *.data/script/ files get copied into python-scripts/
+        wheel_data_script_files = set(wheel_dir.glob("*.data/scripts/**/*"))
+        for wheel_file in wheel_data_script_files:
+            if wheel_file.is_dir():
+                continue
+            rel_path = wheel_file.relative_to(wheel_dir)
+            rel_path = Path(
+                *rel_path.parts[2:]
+            )  # strip *.data/scripts/ from head of rel path
+            conda_file = pkg_dir / "python-scripts" / rel_path
+            assert (
+                conda_file.exists()
+            ), f"{conda_file.relative_to(pkg_dir)} does not exist"
+            # NOTE: in theory this could fail if there is more than one *.data dir that
+            #  specify the same file path with different contents, but in practice we do not
+            #  expect that to ever happen.
+            assert wheel_file.read_bytes() == conda_file.read_bytes()
+
     __call__ = validate