diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f5da14..7f501bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,8 @@ ## [24.1.1] - *in progress* ### Features -* Support wheels containing `*.data` directories +* Support conversion of wheels containing `*.data` directories +* Support direct download of wheels to convert from pypi repositories ## [24.1.0] - 2024-1-8 diff --git a/doc/guide/basic-usage.md b/doc/guide/basic-usage.md index 8f48b42..a058ae2 100644 --- a/doc/guide/basic-usage.md +++ b/doc/guide/basic-usage.md @@ -31,11 +31,18 @@ using [pip download][pip-download]. You do not need to download dependencies and want a wheel, not an sdist, so use: ```bash -$ pip download --only-binary:all: --no-deps +$ pip download --only-binary:all: --no-deps ``` Then you can convert the downloaded wheel using `whl2conda convert`. +Or you can use either the `--from-pypi` or `--from-index` options to `whl2conda convert` +to do this download for you, for example: + +```bash +$ whl2conda convert --from-pypi 'some-package ==1.2.3' +``` + ## Building from project directories If you are creating a conda package for your own python project that uses diff --git a/src/whl2conda/cli/convert.py b/src/whl2conda/cli/convert.py index df04fbc..b62187e 100644 --- a/src/whl2conda/cli/convert.py +++ b/src/whl2conda/cli/convert.py @@ -20,12 +20,14 @@ import argparse import logging import subprocess +import tempfile import time from dataclasses import dataclass from pathlib import Path from typing import Optional, Sequence # this project +from ..impl.download import download_wheel from ..impl.prompt import is_interactive, choose_wheel from ..api.converter import Wheel2CondaConverter, CondaPackageFormat, DependencyRename from ..impl.pyproject import read_pyproject, PyProjInfo @@ -53,6 +55,8 @@ class Whl2CondaArgs: dropped_deps: Sequence[str] dry_run: bool extra_deps: list[str] + from_index: Optional[tuple[str, str]] + from_pypi: Optional[str] ignore_pyproject: bool interactive: bool keep_pip_deps: bool @@ -123,6 +127,23 @@ def _create_argparser(prog: Optional[str] = None) -> argparse.ArgumentParser: """), ) + download_opts = input_opts.add_mutually_exclusive_group() + download_opts.add_argument( + "--from-pypi", + metavar="", + help=dedent(""" + Download package satisfying from standard pypi.org repository. + """), + ) + download_opts.add_argument( + "--from-index", + nargs=2, + metavar=("", ""), + help=dedent(""" + Download package satisfying from repository at . + """), + ) + input_opts.add_argument( "-w", "--wheel-dir", @@ -313,6 +334,13 @@ def convert_main(args: Optional[Sequence[str]] = None, prog: Optional[str] = Non build_wheel = parsed.build_wheel build_no_deps = True # pylint: disable=unused-variable + download_index = "" + download_spec = "" + if parsed.from_pypi: + download_spec = parsed.from_pypi + elif parsed.from_index: + download_index, download_spec = parsed.from_index + wheel_or_root = parsed.wheel_or_root saw_positional_root = False if not wheel_or_root: @@ -322,7 +350,6 @@ def convert_main(args: Optional[Sequence[str]] = None, prog: Optional[str] = Non project_root = wheel_or_root saw_positional_root = True else: - # TODO - also support "pypi: " ? wheel_file = wheel_or_root if wheel_file.suffix != ".whl": parser.error(f"Input file '{wheel_file} does not have .whl suffix") @@ -380,7 +407,15 @@ def convert_main(args: Optional[Sequence[str]] = None, prog: Optional[str] = Non except ValueError as ex: parser.error(f"Bad rename pattern from {source}:\n{ex}") - if not wheel_file and wheel_dir and not build_wheel: + out_dir: Optional[Path] = None + if parsed.out_dir: + out_dir = parsed.out_dir.expanduser().absolute() + elif pyproj_info.out_dir: + out_dir = pyproj_info.out_dir + else: + out_dir = wheel_dir + + if not wheel_file and wheel_dir and not build_wheel and not download_spec: # find wheel in directory try: wheel_file = choose_wheel( @@ -404,14 +439,6 @@ def convert_main(args: Optional[Sequence[str]] = None, prog: Optional[str] = Non except Exception as ex: # pylint: disable=broad-except parser.error(str(ex)) - out_dir: Optional[Path] = None - if parsed.out_dir: - out_dir = parsed.out_dir.expanduser().absolute() - elif pyproj_info.out_dir: - out_dir = pyproj_info.out_dir - else: - out_dir = wheel_dir - if fmtname := parsed.out_format: if fmtname in ("V1", "tar.bz2"): out_fmt = CondaPackageFormat.V1 @@ -438,33 +465,44 @@ def convert_main(args: Optional[Sequence[str]] = None, prog: Optional[str] = Non logging.getLogger().setLevel(level) logging.basicConfig(level=level, format="%(message)s") - if not wheel_file: - if build_wheel: - assert project_root and wheel_dir - wheel_file = do_build_wheel( - project_root, - wheel_dir, - no_deps=build_no_deps, - dry_run=dry_run, - capture_output=level > logging.INFO, - ) - - assert wheel_file - - converter = Wheel2CondaConverter(wheel_file, out_dir) - converter.dry_run = parsed.dry_run - converter.package_name = parsed.name or pyproj_info.conda_name or pyproj_info.name - converter.out_format = out_fmt - converter.overwrite = parsed.overwrite - converter.keep_pip_dependencies = parsed.keep_pip_deps - converter.extra_dependencies.extend(pyproj_info.extra_dependencies) - converter.extra_dependencies.extend(parsed.extra_deps) - converter.interactive = interactive - converter.build_number = parsed.build_number + with tempfile.TemporaryDirectory( + dir=Path.cwd(), prefix="whl2conda-convert-" + ) as tmpdirname: + if not wheel_file: + if download_spec: + wheel_file = download_wheel( + download_spec, + into=Path(tmpdirname), + index=download_index, + ) + elif build_wheel: + assert project_root and wheel_dir + wheel_file = do_build_wheel( + project_root, + wheel_dir, + no_deps=build_no_deps, + dry_run=dry_run, + capture_output=level > logging.INFO, + ) + + assert wheel_file + + converter = Wheel2CondaConverter(wheel_file, out_dir) + converter.dry_run = parsed.dry_run + converter.package_name = ( + parsed.name or pyproj_info.conda_name or pyproj_info.name + ) + converter.out_format = out_fmt + converter.overwrite = parsed.overwrite + converter.keep_pip_dependencies = parsed.keep_pip_deps + converter.extra_dependencies.extend(pyproj_info.extra_dependencies) + converter.extra_dependencies.extend(parsed.extra_deps) + converter.interactive = interactive + converter.build_number = parsed.build_number - converter.dependency_rename.extend(renames) + converter.dependency_rename.extend(renames) - _conda_package = converter.convert() + _conda_package = converter.convert() def do_build_wheel( diff --git a/src/whl2conda/impl/download.py b/src/whl2conda/impl/download.py new file mode 100644 index 0000000..5819b88 --- /dev/null +++ b/src/whl2conda/impl/download.py @@ -0,0 +1,89 @@ +# Copyright 2024 Christopher Barber +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +""" +Support for downloading wheels +""" + +from __future__ import annotations + +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path +from typing import Optional + +__all__ = [ + "download_wheel", +] + + +def download_wheel( + spec: str, + index: str = "", + into: Optional[Path] = None, +) -> Path: + """ + Downloads wheel with given specification from pypi index. + + Args: + spec: requirement specifier for wheel to download: package name and optional version + index: URL of index from which to download. Defaults to pypi.org + into: directory into which wheel will be download. Defaults to current directory. + + Returns: + Path of downloaded file. + """ + + with tempfile.TemporaryDirectory( + dir=Path.cwd(), prefix="whl2conda-download-" + ) as tmpdirname: + tmpdir = Path(tmpdirname) + cmd = [ + "pip", + "download", + "--only-binary", # TODO support building from a source distribution + ":all:", + "--no-deps", + "--ignore-requires-python", # TODO: support specific python version + "--implementation", + "py", + ] + if index: + cmd.extend(["-i", index]) + cmd.extend(["-d", str(tmpdirname)]) + cmd.append(spec) + + p = subprocess.run(cmd, check=True, stderr=subprocess.PIPE) + if p.stderr: + print(p.stderr, file=sys.stderr) + + wheels = list(tmpdir.glob("*.whl")) + + # these should not happen if check_call does not throw, but check anyway + if not wheels: + raise FileNotFoundError("No wheels downloaded") + if len(wheels) > 1: + raise AssertionError( + f"More than one wheel downloaded: {list(w.name for w in wheels)}" + ) + + tmp_wheel = wheels[0] + out_dir = into or Path.cwd() + out_dir.mkdir(parents=True, exist_ok=True) + wheel = out_dir / tmp_wheel.name + shutil.copyfile(tmp_wheel, wheel) + + return wheel diff --git a/test/impl/test_download.py b/test/impl/test_download.py new file mode 100644 index 0000000..b4d70fd --- /dev/null +++ b/test/impl/test_download.py @@ -0,0 +1,135 @@ +# Copyright 2024 Christopher Barber +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +""" +Unit tests for whl2conda.impl.download module +""" + +import argparse +import subprocess +from pathlib import Path +from typing import List + +import pytest + +from whl2conda.impl.download import download_wheel + +# pylint: disable=too-many-statements + + +def test_download_wheel_whitebox( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture +) -> None: + """ + Whitebox test of download_wheel function. This does not do any + downloading and just tests that the expected pip command is + issued. + """ + n_wheels = 1 + download_args: List[argparse.Namespace] = [] + stderr = b"" + + def call_pip_download(cmd: List[str], **_kwargs) -> subprocess.CompletedProcess: + """ + Fake implementation of check_call for pip download + """ + assert cmd[:2] == ["pip", "download"] + + parser = argparse.ArgumentParser() + parser.add_argument("--only-binary") + parser.add_argument("--no-deps", action="store_true") + parser.add_argument("--ignore-requires-python", action="store_true") + parser.add_argument("--implementation") + parser.add_argument("-i", "--index") + parser.add_argument("-d", "--dest") + parser.add_argument("spec") + + parsed = parser.parse_args(cmd[2:]) + + download_args[:] = [parsed] + assert parsed.only_binary == ":all:" + assert parsed.no_deps + assert parsed.ignore_requires_python + assert parsed.implementation == "py" + + download_tmpdir = Path(parsed.dest) + assert download_tmpdir.is_dir() + + if n_wheels > 0: + fake_wheel_file = download_tmpdir / "fake.whl" + fake_wheel_file.write_text(parsed.spec) + for n in range(1, n_wheels): + fake_wheel_file = download_tmpdir / f"fake{n}.whl" + fake_wheel_file.write_text(parsed.spec) + + return subprocess.CompletedProcess(cmd, 0, "", stderr) + + monkeypatch.setattr(subprocess, "run", call_pip_download) + + home_dir = tmp_path / "home" + home_dir.mkdir() + monkeypatch.chdir(home_dir) + + whl = download_wheel("pylint") + assert whl.parent == home_dir + assert whl.name == "fake.whl" + assert whl.is_file() + assert whl.read_text() == "pylint" + assert download_args[0].spec == "pylint" + assert download_args[0].index is None + out, err = capsys.readouterr() + assert not out and not err + + alt_dir = tmp_path / "alt" + assert not alt_dir.exists() + + stderr = b"something happened" + whl = download_wheel("foobar >=1.2.3", index="alt-index", into=alt_dir) + assert whl.parent == alt_dir + assert whl.name == "fake.whl" + assert whl.is_file() + assert whl.read_text() == "foobar >=1.2.3" + assert download_args[0].index == "alt-index" + + # Make sure stderr from subprcess gets output + out, err = capsys.readouterr() + assert not out + assert "something happened" in err + stderr = b"" + + n_wheels = 0 + with pytest.raises(FileNotFoundError, match="No wheels downloaded"): + download_wheel("bar") + + n_wheels = 2 + with pytest.raises(AssertionError, match="More than one wheel downloaded"): + download_wheel("bar") + + +def test_download(tmp_path: Path) -> None: + """ + Test actual downloads + """ + try: + whl = download_wheel("tomlkit", into=tmp_path) + assert whl.is_file() + assert whl.name.startswith("tomlkit") + assert whl.name.endswith(".whl") + assert whl.parent == tmp_path + except subprocess.CalledProcessError as ex: + if b"ConnectionError" in ex.stderr: + # Don't fail test if we are offline. + pytest.skip("Cannot connect to pypi index ") + else: + raise