Merge pull request #121 from zuzukin/91-data-dir

Support downloading of wheel to convert (#37)

CHANGELOG.md

@@ -3,7 +3,8 @@
 ## [24.1.1] - *in progress*
 
 ### Features
-* Support wheels containing `*.data` directories
+* Support conversion of wheels containing `*.data` directories
+* Support direct download of wheels to convert from pypi repositories
 
 ## [24.1.0] - 2024-1-8
 

doc/guide/basic-usage.md

@@ -31,11 +31,18 @@ using [pip download][pip-download]. You do not need to download dependencies and
 want a wheel, not an sdist, so use:
 
 ```bash
-$ pip download --only-binary:all: --no-deps <some-package>
+$ pip download --only-binary:all: --no-deps <some-package-spec>
 ```
 
 Then you can convert the downloaded wheel using `whl2conda convert`.
 
+Or you can use either the `--from-pypi` or `--from-index` options to `whl2conda convert`
+to do this download for you, for example:
+
+```bash
+$ whl2conda convert --from-pypi 'some-package ==1.2.3'
+```
+
 ## Building from project directories
 
 If you are creating a conda package for your own python project that uses

src/whl2conda/cli/convert.py

@@ -20,12 +20,14 @@
 import argparse
 import logging
 import subprocess
+import tempfile
 import time
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Optional, Sequence
 
 # this project
+from ..impl.download import download_wheel
 from ..impl.prompt import is_interactive, choose_wheel
 from ..api.converter import Wheel2CondaConverter, CondaPackageFormat, DependencyRename
 from ..impl.pyproject import read_pyproject, PyProjInfo
@@ -53,6 +55,8 @@ class Whl2CondaArgs:
     dropped_deps: Sequence[str]
     dry_run: bool
     extra_deps: list[str]
+    from_index: Optional[tuple[str, str]]
+    from_pypi: Optional[str]
     ignore_pyproject: bool
     interactive: bool
     keep_pip_deps: bool
@@ -123,6 +127,23 @@ def _create_argparser(prog: Optional[str] = None) -> argparse.ArgumentParser:
         """),
     )
 
+    download_opts = input_opts.add_mutually_exclusive_group()
+    download_opts.add_argument(
+        "--from-pypi",
+        metavar="<package-spec>",
+        help=dedent("""
+            Download package satisfying <package-spec> from standard pypi.org repository.
+        """),
+    )
+    download_opts.add_argument(
+        "--from-index",
+        nargs=2,
+        metavar=("<index-url>", "<package-spec>"),
+        help=dedent("""
+            Download package satisfying <package-spec> from repository at <index-url>.
+        """),
+    )
+
     input_opts.add_argument(
         "-w",
         "--wheel-dir",
@@ -313,6 +334,13 @@ def convert_main(args: Optional[Sequence[str]] = None, prog: Optional[str] = Non
     build_wheel = parsed.build_wheel
     build_no_deps = True  # pylint: disable=unused-variable
 
+    download_index = ""
+    download_spec = ""
+    if parsed.from_pypi:
+        download_spec = parsed.from_pypi
+    elif parsed.from_index:
+        download_index, download_spec = parsed.from_index
+
     wheel_or_root = parsed.wheel_or_root
     saw_positional_root = False
     if not wheel_or_root:
@@ -322,7 +350,6 @@ def convert_main(args: Optional[Sequence[str]] = None, prog: Optional[str] = Non
             project_root = wheel_or_root
             saw_positional_root = True
         else:
-            # TODO - also support "pypi:<name> <version>" ?
             wheel_file = wheel_or_root
             if wheel_file.suffix != ".whl":
                 parser.error(f"Input file '{wheel_file} does not have .whl suffix")
@@ -380,7 +407,15 @@ def convert_main(args: Optional[Sequence[str]] = None, prog: Optional[str] = Non
     except ValueError as ex:
         parser.error(f"Bad rename pattern from {source}:\n{ex}")
 
-    if not wheel_file and wheel_dir and not build_wheel:
+    out_dir: Optional[Path] = None
+    if parsed.out_dir:
+        out_dir = parsed.out_dir.expanduser().absolute()
+    elif pyproj_info.out_dir:
+        out_dir = pyproj_info.out_dir
+    else:
+        out_dir = wheel_dir
+
+    if not wheel_file and wheel_dir and not build_wheel and not download_spec:
         # find wheel in directory
         try:
             wheel_file = choose_wheel(
@@ -404,14 +439,6 @@ def convert_main(args: Optional[Sequence[str]] = None, prog: Optional[str] = Non
         except Exception as ex:  # pylint: disable=broad-except
             parser.error(str(ex))
 
-    out_dir: Optional[Path] = None
-    if parsed.out_dir:
-        out_dir = parsed.out_dir.expanduser().absolute()
-    elif pyproj_info.out_dir:
-        out_dir = pyproj_info.out_dir
-    else:
-        out_dir = wheel_dir
-
     if fmtname := parsed.out_format:
         if fmtname in ("V1", "tar.bz2"):
             out_fmt = CondaPackageFormat.V1
@@ -438,33 +465,44 @@ def convert_main(args: Optional[Sequence[str]] = None, prog: Optional[str] = Non
     logging.getLogger().setLevel(level)
     logging.basicConfig(level=level, format="%(message)s")
 
-    if not wheel_file:
-        if build_wheel:
-            assert project_root and wheel_dir
-            wheel_file = do_build_wheel(
-                project_root,
-                wheel_dir,
-                no_deps=build_no_deps,
-                dry_run=dry_run,
-                capture_output=level > logging.INFO,
-            )
-
-    assert wheel_file
-
-    converter = Wheel2CondaConverter(wheel_file, out_dir)
-    converter.dry_run = parsed.dry_run
-    converter.package_name = parsed.name or pyproj_info.conda_name or pyproj_info.name
-    converter.out_format = out_fmt
-    converter.overwrite = parsed.overwrite
-    converter.keep_pip_dependencies = parsed.keep_pip_deps
-    converter.extra_dependencies.extend(pyproj_info.extra_dependencies)
-    converter.extra_dependencies.extend(parsed.extra_deps)
-    converter.interactive = interactive
-    converter.build_number = parsed.build_number
+    with tempfile.TemporaryDirectory(
+        dir=Path.cwd(), prefix="whl2conda-convert-"
+    ) as tmpdirname:
+        if not wheel_file:
+            if download_spec:
+                wheel_file = download_wheel(
+                    download_spec,
+                    into=Path(tmpdirname),
+                    index=download_index,
+                )
+            elif build_wheel:
+                assert project_root and wheel_dir
+                wheel_file = do_build_wheel(
+                    project_root,
+                    wheel_dir,
+                    no_deps=build_no_deps,
+                    dry_run=dry_run,
+                    capture_output=level > logging.INFO,
+                )
+
+        assert wheel_file
+
+        converter = Wheel2CondaConverter(wheel_file, out_dir)
+        converter.dry_run = parsed.dry_run
+        converter.package_name = (
+            parsed.name or pyproj_info.conda_name or pyproj_info.name
+        )
+        converter.out_format = out_fmt
+        converter.overwrite = parsed.overwrite
+        converter.keep_pip_dependencies = parsed.keep_pip_deps
+        converter.extra_dependencies.extend(pyproj_info.extra_dependencies)
+        converter.extra_dependencies.extend(parsed.extra_deps)
+        converter.interactive = interactive
+        converter.build_number = parsed.build_number
 
-    converter.dependency_rename.extend(renames)
+        converter.dependency_rename.extend(renames)
 
-    _conda_package = converter.convert()
+        _conda_package = converter.convert()
 
 
 def do_build_wheel(

src/whl2conda/impl/download.py

@@ -0,0 +1,89 @@
+#  Copyright 2024 Christopher Barber
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+"""
+Support for downloading wheels
+"""
+
+from __future__ import annotations
+
+import shutil
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+from typing import Optional
+
+__all__ = [
+    "download_wheel",
+]
+
+
+def download_wheel(
+    spec: str,
+    index: str = "",
+    into: Optional[Path] = None,
+) -> Path:
+    """
+    Downloads wheel with given specification from pypi index.
+
+    Args:
+        spec: requirement specifier for wheel to download: package name and optional version
+        index: URL of index from which to download. Defaults to pypi.org
+        into: directory into which wheel will be download. Defaults to current directory.
+
+    Returns:
+        Path of downloaded file.
+    """
+
+    with tempfile.TemporaryDirectory(
+        dir=Path.cwd(), prefix="whl2conda-download-"
+    ) as tmpdirname:
+        tmpdir = Path(tmpdirname)
+        cmd = [
+            "pip",
+            "download",
+            "--only-binary",  # TODO support building from a source distribution
+            ":all:",
+            "--no-deps",
+            "--ignore-requires-python",  # TODO: support specific python version
+            "--implementation",
+            "py",
+        ]
+        if index:
+            cmd.extend(["-i", index])
+        cmd.extend(["-d", str(tmpdirname)])
+        cmd.append(spec)
+
+        p = subprocess.run(cmd, check=True, stderr=subprocess.PIPE)
+        if p.stderr:
+            print(p.stderr, file=sys.stderr)
+
+        wheels = list(tmpdir.glob("*.whl"))
+
+        # these should not happen if check_call does not throw, but check anyway
+        if not wheels:
+            raise FileNotFoundError("No wheels downloaded")
+        if len(wheels) > 1:
+            raise AssertionError(
+                f"More than one wheel downloaded: {list(w.name for w in wheels)}"
+            )
+
+        tmp_wheel = wheels[0]
+        out_dir = into or Path.cwd()
+        out_dir.mkdir(parents=True, exist_ok=True)
+        wheel = out_dir / tmp_wheel.name
+        shutil.copyfile(tmp_wheel, wheel)
+
+        return wheel