Merge pull request #120 from zuzukin/91-data-dir

Support for wheels containing *.data directories.
zuzukin · Jan 14, 2024 · 6ed76e5 · 6ed76e5
2 parents 42ae7e2 + cf95b41
commit 6ed76e5
Show file tree

Hide file tree

Showing 14 changed files with 215 additions and 130 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,10 @@
 # whl2conda changes
 
+## [24.1.1] - *in progress*
+
+### Features
+* Support wheels containing `*.data` directories
+
 ## [24.1.0] - 2024-1-8
 
 ### Features

diff --git a/Makefile b/Makefile
@@ -150,7 +150,7 @@ site/index.html: $(CLI_DOCS) $(MKDOCS_FILE) doc/*.md src/whl2conda/api/*.py
 doc: site/index.html
 
 site/.doc-strict: site/index.html
-	$(CONDA_RUN) linkchecker -f likcheckerrc.ini site
+	$(CONDA_RUN) linkchecker -f linkcheckerrc.ini site
 	$(TOUCH) $@
 
 doc-strict: site/.doc-strict

diff --git a/doc/guide/limitations.md b/doc/guide/limitations.md
@@ -13,14 +13,6 @@ in dependencies and probably will not occur that often in practice.
 We handle this by simplying changinge `===` to `==` but
 since this will often not work we also issue a warning.
 
-## Wheel data directories not supported
-
-Wheels with `*.data` directies are not fully supported.
-Any such data directories will not be copied.
-
-This will be addressed in a future release
-(see [issue 91](https://github.com/zuzukin/whl2conda/issues/91))
-
 ## Cannot convert from sdist
 
 Conversion from python sdist distributions is not currently supported. 

diff --git a/src/whl2conda/VERSION b/src/whl2conda/VERSION
@@ -1 +1 @@
-24.1.0
+24.1.1
diff --git a/src/whl2conda/api/converter.py b/src/whl2conda/api/converter.py
@@ -343,8 +343,15 @@ def convert(self) -> Path:
             conda_info_dir = conda_dir.joinpath("info")
             conda_dir.mkdir()
 
-            # Copy files into site packages and get relative paths
-            rel_files = self._copy_site_packages(extracted_wheel_dir, conda_dir)
+            # Copy files into conda package
+            self._copy_wheel_files(extracted_wheel_dir, conda_dir)
+
+            # collect relative paths before constructing info/ directory
+            rel_files = list(
+                str(f.relative_to(conda_dir))
+                for f in conda_dir.glob("**/*")
+                if f.is_file()
+            )
 
             conda_dependencies = self._compute_conda_dependencies(wheel_md.dependencies)
 
@@ -648,24 +655,54 @@ def _compute_conda_dependencies(
             conda_dependencies.append(dep)
         return conda_dependencies
 
-    def _copy_site_packages(self, wheel_dir: Path, conda_dir: Path) -> list[str]:
+    def _copy_wheel_files(self, wheel_dir: Path, conda_dir: Path) -> None:
+        """
+        Copies files from wheels to corresponding location in conda package:
+
+        This copies files using the mapping:
+        - <wheel-dir>/*.data/data/* -> <conda-dir>/*
+        - <wheel-dir>/*.data/scripts/* -> <conda-dir>/python-scripts/*
+        - <wheel-dir>/*.data/* -> ignored
+        - <wheel-dir>/* -> <conda-dir>/site-packages
+        """
         conda_site_packages = conda_dir.joinpath("site-packages")
         conda_site_packages.mkdir()
         conda_info_dir = conda_dir.joinpath("info")
         conda_info_dir.mkdir()
-        shutil.copytree(wheel_dir, conda_site_packages, dirs_exist_ok=True)
+        for entry in wheel_dir.iterdir():
+            if not entry.is_dir():
+                shutil.copyfile(entry, conda_site_packages / entry.name)
+            elif not entry.name.endswith(".data"):
+                shutil.copytree(
+                    entry, conda_site_packages / entry.name, dirs_exist_ok=True
+                )
+            else:
+                for datapath in entry.iterdir():
+                    if not datapath.is_dir():
+                        self._warn(
+                            "Do not support top level file '%s' in '%s' directory - ignored",
+                            datapath.name,
+                            entry.relative_to(wheel_dir),
+                        )
+                    if datapath.name == "data":
+                        conda_target = conda_dir
+                    elif datapath.name == "scripts":
+                        conda_target = conda_dir / "python-scripts"
+                    else:
+                        self._warn(
+                            "Do not support '%s' path in '%s' directory - ignored",
+                            datapath.name,
+                            entry.relative_to(wheel_dir),
+                        )
+                        continue
+                    shutil.copytree(datapath, conda_target, dirs_exist_ok=True)
+
         assert self.wheel_md is not None
         dist_info_dir = conda_site_packages / self.wheel_md.wheel_info_dir.name
         installer_file = dist_info_dir / "INSTALLER"
         installer_file.write_text("whl2conda")
         requested_file = dist_info_dir / "REQUESTED"
         requested_file.write_text("")
-        rel_files = list(
-            str(f.relative_to(conda_dir))
-            for f in conda_site_packages.glob("**/*")
-            if f.is_file()
-        )
-        return rel_files
 
     def _copy_licenses(self, conda_info_dir: Path, wheel_md: MetadataFromWheel) -> None:
         to_license_dir = conda_info_dir / "licenses"

diff --git a/src/whl2conda/cli/build.py b/src/whl2conda/cli/build.py
@@ -54,8 +54,7 @@ def build_main(
 ) -> None:
     """Main procedure for `whl2conda build` command"""
     parser = argparse.ArgumentParser(
-        description=dedent(
-            """
+        description=dedent("""
             Build a conda package from a pure python wheel.
             
             This command is limited drop-in replacement for `conda build`.
@@ -64,8 +63,7 @@ def build_main(
             
             This is an experimental feature and is still under active
             change and development.
-            """
-        ),
+            """),
         formatter_class=argparse.RawTextHelpFormatter,
         prog=prog,
     )
@@ -137,13 +135,11 @@ def _render_recipe(self):
                 "base",
                 "python",
                 "-c",
-                dedent(
-                    f"""
+                dedent(f"""
                     import conda_build.api as api
                     mds = api.render("{self.args.recipe_path}", bypass_env_check=True)
                     api.output_yaml(mds[0][0], file_path="{tmp_recipe_file}")
-                    """
-                ),
+                    """),
             ]
 
             with subprocess.Popen(cmd, encoding="utf8", stdout=subprocess.PIPE) as p:

diff --git a/src/whl2conda/cli/config.py b/src/whl2conda/cli/config.py
@@ -40,11 +40,9 @@ def config_main(
     """Main routine for `whl2conda config` subcommand"""
 
     parser = argparse.ArgumentParser(
-        description=dedent(
-            """
+        description=dedent("""
             whl2conda configuration
-            """
-        ),
+            """),
         formatter_class=argparse.RawTextHelpFormatter,
         prog=prog,
     )
@@ -54,8 +52,7 @@ def config_main(
         metavar="<dir-or-toml>",
         nargs='?',
         const='out',
-        help=dedent(
-            """
+        help=dedent("""
             Add default whl2conda tool entries to a pyproject file. 
             If argument is a directory entries will be added to 
             `pyproject.toml` in that directory. If argument ends
@@ -64,8 +61,7 @@ def config_main(
             will be written to stdout. Other values will result in an error.
             This will create file if it does not already exist.
             It will not overwrite existing entires.
-            """
-        ),
+            """),
     )
 
     parser.add_argument(
@@ -74,13 +70,11 @@ def config_main(
         metavar="<file>",
         const=user_stdrenames_path(),
         type=Path,
-        help=dedent(
-            """
+        help=dedent("""
             Update list of standard pypi to conda renames from internet and exit.
             If a <file> is not named, the default copy will be updated at
             %(const)s.
-            """
-        ),
+            """),
     )
     parser.add_argument(
         "-n",

diff --git a/src/whl2conda/cli/convert.py b/src/whl2conda/cli/convert.py
@@ -76,18 +76,14 @@ def _create_argparser(prog: Optional[str] = None) -> argparse.ArgumentParser:
     Whl2CondaArgs
     """
     parser = argparse.ArgumentParser(
-        usage=dedent(
-            """
+        usage=dedent("""
             %(prog)s <wheel> [options]
                    %(prog)s [<project-root>] [options]
-            """
-        ),
+            """),
         prog=prog,
-        description=dedent(
-            """
+        description=dedent("""
             Generates a conda package from a pure python wheel
-            """
-        ),
+            """),
         formatter_class=argparse.RawTextHelpFormatter,
         add_help=False,
     )
@@ -103,12 +99,10 @@ def _create_argparser(prog: Optional[str] = None) -> argparse.ArgumentParser:
         nargs="?",
         metavar="[<wheel> | <project-root>]",
         type=existing_path,
-        help=dedent(
-            """
+        help=dedent("""
         Either path to a wheel file to convert or a project root
         directory containing a pyproject.toml or setup.py file.
-        """
-        ),
+        """),
     )
 
     input_opts.add_argument(
@@ -117,8 +111,7 @@ def _create_argparser(prog: Optional[str] = None) -> argparse.ArgumentParser:
         dest="project_root",
         metavar="<dir>",
         type=existing_dir,
-        help=dedent(
-            """
+        help=dedent("""
             Project root directory. This is a directory containing either a
             pyproject.toml or a (deprecated) setup.py file. This option may
             not be used if the project directory was given as the positional
@@ -127,21 +120,18 @@ def _create_argparser(prog: Optional[str] = None) -> argparse.ArgumentParser:
             If not specified, the project root will be located by searching
             the wheel directory and its parent directories, or if no wheel
             given, will default to the current directory.
-        """
-        ),
+        """),
     )
 
     input_opts.add_argument(
         "-w",
         "--wheel-dir",
         metavar="<dir>",
         type=maybe_existing_dir,
-        help=dedent(
-            """
+        help=dedent("""
             Location of wheel directory. Defaults to dist/ subdirectory of 
             project.
-            """
-        ),
+            """),
     )
 
     input_opts.add_argument(
@@ -156,22 +146,18 @@ def _create_argparser(prog: Optional[str] = None) -> argparse.ArgumentParser:
         dest="out_dir",
         metavar="<dir>",
         type=maybe_existing_dir,
-        help=dedent(
-            """
+        help=dedent("""
             Output directory for conda package. Defaults to wheel directory
             or else project dist directory.
-            """
-        ),
+            """),
     )
 
     output_opts.add_argument(
         "--overwrite",
         action="store_true",
-        help=dedent(
-            """
+        help=dedent("""
             Overwrite existing output files.
-            """
-        ),
+            """),
     )
 
     output_opts.add_argument(
@@ -184,11 +170,9 @@ def _create_argparser(prog: Optional[str] = None) -> argparse.ArgumentParser:
     output_opts.add_argument(
         "--build-wheel",
         action="store_true",
-        help=dedent(
-            """
+        help=dedent("""
             Build wheel
-            """
-        ),
+            """),
     )
     output_opts.add_argument(
         "--build-number",
@@ -209,11 +193,9 @@ def _create_argparser(prog: Optional[str] = None) -> argparse.ArgumentParser:
         action="append",
         default=[],
         dest="dep_renames",
-        help=dedent(
-            """
+        help=dedent("""
         Rename pip dependency for conda. May be specified muliple times.
-        """
-        ),
+        """),
     )
     override_opts.add_argument(
         "-A",
@@ -222,11 +204,9 @@ def _create_argparser(prog: Optional[str] = None) -> argparse.ArgumentParser:
         metavar="<conda-dep>",
         action="append",
         default=[],
-        help=dedent(
-            """
+        help=dedent("""
             Add an additional conda dependency. May be specified multiple times.
-            """
-        ),
+            """),
     )
     override_opts.add_argument(
         "-D",
@@ -235,12 +215,10 @@ def _create_argparser(prog: Optional[str] = None) -> argparse.ArgumentParser:
         dest="dropped_deps",
         action="append",
         default=[],
-        help=dedent(
-            """
+        help=dedent("""
             Drop dependency with given name from conda dependency list.
             May be specified multiple times.
-            """
-        ),
+            """),
     )
     override_opts.add_argument(
         "-K",

diff --git a/src/whl2conda/cli/diff.py b/src/whl2conda/cli/diff.py
@@ -53,8 +53,7 @@ def diff_main(
     """Main routine for `whl2conda diff` subcommand"""
 
     parser = argparse.ArgumentParser(
-        description=dedent(
-            """
+        description=dedent("""
             Compare the content of two conda packages
             
             This will unpack each conda packaeg into temporary
@@ -64,8 +63,7 @@ def diff_main(
             
             This can be used to compare packages generated using
             this tool against those created using conda-build.            
-            """
-        ),
+            """),
         formatter_class=argparse.RawTextHelpFormatter,
         prog=prog,
     )
@@ -89,15 +87,13 @@ def diff_main(
         "--diff-tool",
         metavar="<tool>",
         required=True,
-        help=dedent(
-            """
+        help=dedent("""
             Diff tool to use. This is currently required.
             
             The tool is expected to take positional arguments
             for each directory. Additional arguments may be
             passed after --args.
-            """
-        ),
+            """),
     )
 
     parser.add_argument(