diff --git a/.github/actions/build-app-runner/action.yml b/.github/actions/build-app-runner/action.yml index 48c11978..66519602 100644 --- a/.github/actions/build-app-runner/action.yml +++ b/.github/actions/build-app-runner/action.yml @@ -8,7 +8,7 @@ inputs: source-path: description: 'Path to app runner source code' required: false - default: 'app_runner' + default: 'bfabric_app_runner' outputs: artifact-name: description: 'Name of the uploaded artifact' diff --git a/.github/workflows/build_app_runner.yml b/.github/workflows/build_app_runner.yml index b843c06c..feea5ebe 100644 --- a/.github/workflows/build_app_runner.yml +++ b/.github/workflows/build_app_runner.yml @@ -41,7 +41,7 @@ jobs: id: build with: output-path: build-output - source-path: app_runner + source-path: bfabric_app_runner - name: Comment PR if: github.event_name == 'pull_request' uses: actions/github-script@v7 diff --git a/.github/workflows/complete_release.yml b/.github/workflows/complete_release.yml deleted file mode 100644 index 0a0b30e3..00000000 --- a/.github/workflows/complete_release.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: Complete Release -on: - push: - branches: [stable] - # TODO remove after testing - workflow_dispatch: -jobs: - create_tag: - name: Create Tag - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: 3.12 - - name: Git Config - run: | - git config --global user.name "bfabricPy Bot" - git config --global user.email "bfabricpy-bot-noreply@fgcz.ethz.ch" - - name: Create tag (with Python) - run: | - set -euxo pipefail - # Find the version - VERSION=$(python3 -c "import tomllib; from pathlib import Path; print(tomllib.loads(Path('pyproject.toml').read_text())['project']['version'], end='')") - # Check if tag exists - if git rev-parse $VERSION >/dev/null 2>&1; then - echo "Tag $VERSION already exists" - exit 1 - fi - # Create tag - git tag -a $VERSION -m "Release $VERSION" - # Push tag - git push origin $VERSION - publish_documentation: - name: Publish Docs - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: 3.12 - - name: Install nox - run: pip install nox uv - - name: Publish documentation - run: | - set -euxo pipefail - git fetch --unshallow origin gh-pages - git checkout gh-pages && git pull && git checkout - - nox -s docs - nox -s publish_docs diff --git a/.github/workflows/publish_release.yml b/.github/workflows/publish_release.yml index 655dc762..b6654e36 100644 --- a/.github/workflows/publish_release.yml +++ b/.github/workflows/publish_release.yml @@ -4,7 +4,7 @@ on: branches: - release_bfabric - release_bfabric_scripts - - release_app_runner + - release_bfabric_app_runner workflow_dispatch: inputs: package: @@ -13,7 +13,7 @@ on: options: - bfabric - bfabric_scripts - - app_runner + - bfabric_app_runner default: bfabric environment: description: 'Target PyPI' @@ -28,6 +28,7 @@ jobs: runs-on: ubuntu-latest permissions: id-token: write + contents: write # permission to create tags steps: - uses: actions/checkout@v4 # Step: Determine the package that is being built @@ -62,6 +63,13 @@ jobs: python-version: '3.11' - name: Install hatch run: pip install hatch + - name: Get package version + id: get-version + run: | + cd ${{ env.PACKAGE }} + VERSION=$(hatch version) + echo "VERSION=$VERSION" >> $GITHUB_ENV + echo "Package version: $VERSION" - name: Build package run: | cd ${{ env.PACKAGE }} @@ -71,11 +79,20 @@ jobs: with: repository-url: ${{ env.PYPI_REPOSITORY_URL }} packages-dir: ${{ env.PACKAGE }}/dist + - name: Create and push tag + run: | + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions[bot]" + TAG_NAME="${{ env.PACKAGE }}/${{ env.VERSION }}" + git tag -a "$TAG_NAME" -m "Release ${{ env.PACKAGE }} version ${{ env.VERSION }}" + git push origin "$TAG_NAME" - name: Debug package info run: | echo "Built and published package: ${{ env.PACKAGE }}" + echo "Created tag: ${{ env.PACKAGE }}/${{ env.VERSION }}" if [ "${{ env.PYPI_REPOSITORY_URL }}" == "https://pypi.org/legacy/" ]; then echo "Check it at: https://pypi.org/project/${{ env.PACKAGE }}/" else echo "Check it at: https://test.pypi.org/project/${{ env.PACKAGE }}/" fi + if: always() diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e58c0afb..204b3ce1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,7 @@ ci: autofix_commit_msg: "style: pre-commit fixes" repos: - repo: https://github.com/psf/black - rev: "24.8.0" + rev: "25.1.0" hooks: - id: black - repo: https://github.com/adamchainz/blacken-docs diff --git a/app_runner/docs/index.md b/app_runner/docs/index.md deleted file mode 100644 index 3b48680c..00000000 --- a/app_runner/docs/index.md +++ /dev/null @@ -1,18 +0,0 @@ -## Install App Runner - -```bash -uv tool install app_runner@git+https://github.com/fgcz/bfabricPy.git@main#egg=app_runner&subdirectory=app_runner -``` - -## Contents - -```{toctree} -:glob: -workunit_definition -architecture/overview -specs/input_specification -specs/output_specification -specs/app_specification -changelog -* -``` diff --git a/bfabric/pyproject.toml b/bfabric/pyproject.toml index 9757557f..7d346c25 100644 --- a/bfabric/pyproject.toml +++ b/bfabric/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "hatchling.build" [project] name = "bfabric" description = "Python client for the B-Fabric API" -version = "1.13.18" +version = "1.13.19" license = { text = "GPL-3.0" } authors = [ { name = "Christian Panse", email = "cp@fgcz.ethz.ch" }, @@ -53,7 +53,7 @@ Repository = "https://github.com/fgcz/bfabricPy" [tool.uv] -reinstall-package = ["bfabric", "bfabric_scripts", "app_runner"] +reinstall-package = ["bfabric", "bfabric_scripts", "bfabric_app_runner"] [tool.black] line-length = 120 diff --git a/bfabric/src/bfabric/bfabric.py b/bfabric/src/bfabric/bfabric.py index c0d84dc1..1ec4974d 100644 --- a/bfabric/src/bfabric/bfabric.py +++ b/bfabric/src/bfabric/bfabric.py @@ -95,9 +95,7 @@ def from_config( If it is set to None, no authentication will be used. :param engine: Engine to use for the API. Default is SUDS. """ - config, auth_config = get_system_auth( - config_env=config_env, config_path=config_path - ) + config, auth_config = get_system_auth(config_env=config_env, config_path=config_path) auth_used: BfabricAuth | None = auth_config if auth == "config" else auth return cls(config, auth_used, engine=engine) @@ -195,9 +193,7 @@ def read( response_items += results[page_offset:] page_offset = 0 - result = ResultContainer( - response_items, total_pages_api=n_available_pages, errors=errors - ) + result = ResultContainer(response_items, total_pages_api=n_available_pages, errors=errors) if check: result.assert_success() return result.get_first_n_results(max_results) @@ -217,16 +213,12 @@ def save( appropriate to be used instead. :return a ResultContainer describing the saved object if successful """ - results = self._engine.save( - endpoint=endpoint, obj=obj, auth=self.auth, method=method - ) + results = self._engine.save(endpoint=endpoint, obj=obj, auth=self.auth, method=method) if check: results.assert_success() return results - def delete( - self, endpoint: str, id: int | list[int], check: bool = True - ) -> ResultContainer: + def delete(self, endpoint: str, id: int | list[int], check: bool = True) -> ResultContainer: """Deletes the object with the specified ID from the specified endpoint. :param endpoint: the endpoint to delete from, e.g. "sample" :param id: the ID of the object to delete @@ -353,19 +345,11 @@ def get_system_auth( if not resolved_path.is_file(): if config_path: # NOTE: If user explicitly specifies a path to a wrong config file, this has to be an exception - raise OSError( - f"Explicitly specified config file does not exist: {resolved_path}" - ) + raise OSError(f"Explicitly specified config file does not exist: {resolved_path}") # TODO: Convert to log - print( - f"Warning: could not find the config file in the default location: {resolved_path}" - ) + print(f"Warning: could not find the config file in the default location: {resolved_path}") config = BfabricClientConfig(base_url=base_url) - auth = ( - None - if login is None or password is None - else BfabricAuth(login=login, password=password) - ) + auth = None if login is None or password is None else BfabricAuth(login=login, password=password) # Load config from file, override some of the fields with the provided ones else: diff --git a/bfabric/src/bfabric/bfabric2.py b/bfabric/src/bfabric/bfabric2.py index 82dde5e2..ac138da5 100755 --- a/bfabric/src/bfabric/bfabric2.py +++ b/bfabric/src/bfabric/bfabric2.py @@ -1,7 +1,5 @@ #!/usr/bin/env python3 import warnings -warnings.warn( - "bfabric.bfabric2 module is deprecated, use bfabric instead", DeprecationWarning -) +warnings.warn("bfabric.bfabric2 module is deprecated, use bfabric instead", DeprecationWarning) # TODO deprecated - import from bfabric instead diff --git a/bfabric/src/bfabric/bfabric_config.py b/bfabric/src/bfabric/bfabric_config.py index ee484101..cf4ca7e9 100644 --- a/bfabric/src/bfabric/bfabric_config.py +++ b/bfabric/src/bfabric/bfabric_config.py @@ -31,8 +31,6 @@ def read_config( - If not, finally, the parser will select the default_config specified in [GENERAL] of the .bfabricpy.yml file """ logger.debug(f"Reading configuration from: {config_path}") - config_file = ConfigFile.model_validate( - yaml.safe_load(Path(config_path).read_text()) - ) + config_file = ConfigFile.model_validate(yaml.safe_load(Path(config_path).read_text())) env_config = config_file.get_selected_config(explicit_config_env=config_env) return env_config.config, env_config.auth diff --git a/bfabric/src/bfabric/cli_formatting.py b/bfabric/src/bfabric/cli_formatting.py index 78c287b8..d9ef6be1 100644 --- a/bfabric/src/bfabric/cli_formatting.py +++ b/bfabric/src/bfabric/cli_formatting.py @@ -22,12 +22,10 @@ def setup_script_logging(debug: bool = False) -> None: if os.environ.get(setup_flag_key, "0") == "1": return logger.remove() - packages = ["bfabric", "bfabric_scripts", "app_runner", "__main__"] + packages = ["bfabric", "bfabric_scripts", "bfabric_app_runner", "__main__"] if not (debug or os.environ.get("BFABRICPY_DEBUG")): for package in packages: - logger.add( - sys.stderr, filter=package, level="INFO", format="{level} {message}" - ) + logger.add(sys.stderr, filter=package, level="INFO", format="{level} {message}") else: for package in packages: logger.add(sys.stderr, filter=package, level="DEBUG") diff --git a/bfabric/src/bfabric/config/bfabric_client_config.py b/bfabric/src/bfabric/config/bfabric_client_config.py index 7d0819ba..e7bfe72b 100644 --- a/bfabric/src/bfabric/config/bfabric_client_config.py +++ b/bfabric/src/bfabric/config/bfabric_client_config.py @@ -27,9 +27,7 @@ class BfabricClientConfig(BaseModel): def __init__(self, **kwargs: Any) -> None: # TODO remove this custom constructor (note that this is currently used in some places when "None" is passed) - super().__init__( - **{key: value for key, value in kwargs.items() if value is not None} - ) + super().__init__(**{key: value for key, value in kwargs.items() if value is not None}) def copy_with( self, @@ -39,9 +37,7 @@ def copy_with( """Returns a copy of the configuration with new values applied, if they are not None.""" return BfabricClientConfig( base_url=base_url if base_url is not None else self.base_url, - application_ids=( - application_ids if application_ids is not None else self.application_ids - ), + application_ids=(application_ids if application_ids is not None else self.application_ids), job_notification_emails=self.job_notification_emails, ) diff --git a/bfabric/src/bfabric/config/config_file.py b/bfabric/src/bfabric/config/config_file.py index 4cb6faa8..b7a1485c 100644 --- a/bfabric/src/bfabric/config/config_file.py +++ b/bfabric/src/bfabric/config/config_file.py @@ -25,11 +25,7 @@ def gather_config(cls, values: dict[str, Any]) -> dict[str, Any]: """Gathers all configs into the config field.""" if not isinstance(values, dict): return values - values["config"] = { - key: value - for key, value in values.items() - if key not in ["login", "password"] - } + values["config"] = {key: value for key, value in values.items() if key not in ["login", "password"]} return values @model_validator(mode="before") @@ -78,21 +74,13 @@ def get_selected_config_env(self, explicit_config_env: str | None) -> str: if explicit_config_env: return explicit_config_env elif "BFABRICPY_CONFIG_ENV" in os.environ: - logger.debug( - f"found BFABRICPY_CONFIG_ENV = {os.environ['BFABRICPY_CONFIG_ENV']}" - ) + logger.debug(f"found BFABRICPY_CONFIG_ENV = {os.environ['BFABRICPY_CONFIG_ENV']}") return os.environ["BFABRICPY_CONFIG_ENV"] else: - logger.debug( - f"BFABRICPY_CONFIG_ENV not found, using default environment {self.general.default_config}" - ) + logger.debug(f"BFABRICPY_CONFIG_ENV not found, using default environment {self.general.default_config}") return self.general.default_config - def get_selected_config( - self, explicit_config_env: str | None = None - ) -> EnvironmentConfig: + def get_selected_config(self, explicit_config_env: str | None = None) -> EnvironmentConfig: """Returns the selected configuration, by checking the hierarchy of config_env definitions. See selected_config_env for details.""" - return self.environments[ - self.get_selected_config_env(explicit_config_env=explicit_config_env) - ] + return self.environments[self.get_selected_config_env(explicit_config_env=explicit_config_env)] diff --git a/bfabric/src/bfabric/engine/engine_suds.py b/bfabric/src/bfabric/engine/engine_suds.py index 5888298c..592d7c6f 100644 --- a/bfabric/src/bfabric/engine/engine_suds.py +++ b/bfabric/src/bfabric/engine/engine_suds.py @@ -57,9 +57,7 @@ def read( response = service.read(full_query) return self._convert_results(response=response, endpoint=endpoint) - def save( - self, endpoint: str, obj: dict, auth: BfabricAuth, method: str = "save" - ) -> ResultContainer: + def save(self, endpoint: str, obj: dict, auth: BfabricAuth, method: str = "save") -> ResultContainer: """Saves the provided object to the specified endpoint. :param endpoint: the endpoint to save to, e.g. "sample" :param obj: the object to save @@ -72,14 +70,10 @@ def save( try: response = getattr(service, method)(query) except MethodNotFound as e: - raise BfabricRequestError( - f"SUDS failed to find save method for the {endpoint} endpoint." - ) from e + raise BfabricRequestError(f"SUDS failed to find save method for the {endpoint} endpoint.") from e return self._convert_results(response=response, endpoint=endpoint) - def delete( - self, endpoint: str, id: int | list[int], auth: BfabricAuth - ) -> ResultContainer: + def delete(self, endpoint: str, id: int | list[int], auth: BfabricAuth) -> ResultContainer: """Deletes the object with the specified ID from the specified endpoint. :param endpoint: the endpoint to delete from, e.g. "sample" :param id: the ID of the object to delete @@ -99,9 +93,7 @@ def _get_suds_service(self, endpoint: str) -> ServiceProxy: """Returns a SUDS service for the given endpoint. Reuses existing instances when possible.""" if endpoint not in self._cl: try: - self._cl[endpoint] = Client( - f"{self._base_url}/{endpoint}?wsdl", cache=None - ) + self._cl[endpoint] = Client(f"{self._base_url}/{endpoint}?wsdl", cache=None) except suds.transport.TransportError as error: if error.httpcode == 404: msg = f"Non-existent endpoint {repr(endpoint)} or the configured B-Fabric instance was not found." @@ -127,6 +119,4 @@ def _convert_results(self, response: Any, endpoint: str) -> ResultContainer: sort_keys=True, ) results += [result_parsed] - return ResultContainer( - results=results, total_pages_api=n_available_pages, errors=errors - ) + return ResultContainer(results=results, total_pages_api=n_available_pages, errors=errors) diff --git a/bfabric/src/bfabric/engine/engine_zeep.py b/bfabric/src/bfabric/engine/engine_zeep.py index 32aa8b60..75f883ef 100644 --- a/bfabric/src/bfabric/engine/engine_zeep.py +++ b/bfabric/src/bfabric/engine/engine_zeep.py @@ -65,15 +65,11 @@ def read( ) client = self._get_client(endpoint) - with client.settings( - strict=False, xml_huge_tree=True, xsd_ignore_sequence_order=True - ): + with client.settings(strict=False, xml_huge_tree=True, xsd_ignore_sequence_order=True): response = client.service.read(full_query) return self._convert_results(response=response, endpoint=endpoint) - def save( - self, endpoint: str, obj: dict, auth: BfabricAuth, method: str = "save" - ) -> ResultContainer: + def save(self, endpoint: str, obj: dict, auth: BfabricAuth, method: str = "save") -> ResultContainer: """Saves the provided object to the specified endpoint. :param endpoint: the endpoint to save to, e.g. "sample" :param obj: the object to save @@ -97,15 +93,11 @@ def save( response = getattr(client.service, method)(full_query) except AttributeError as e: if e.args[0] == "Service has no operation 'save'": - raise BfabricRequestError( - f"ZEEP failed to find save method for the {endpoint} endpoint." - ) from e + raise BfabricRequestError(f"ZEEP failed to find save method for the {endpoint} endpoint.") from e raise e return self._convert_results(response=response, endpoint=endpoint) - def delete( - self, endpoint: str, id: int | list[int], auth: BfabricAuth - ) -> ResultContainer: + def delete(self, endpoint: str, id: int | list[int], auth: BfabricAuth) -> ResultContainer: """Deletes the object with the specified ID from the specified endpoint. :param endpoint: the endpoint to delete from, e.g. "sample" :param id: the ID of the object to delete @@ -151,9 +143,7 @@ def _convert_results(self, response: Any, endpoint: str) -> ResultContainer: sort_keys=True, ) results += [results_parsed] - return ResultContainer( - results=results, total_pages_api=n_available_pages, errors=errors - ) + return ResultContainer(results=results, total_pages_api=n_available_pages, errors=errors) # TODO: The reason why Zeep requires to explicitly skip certain values remains unclear @@ -161,9 +151,7 @@ def _convert_results(self, response: Any, endpoint: str) -> ResultContainer: # formatting they appear to zeep as compulsory. The current solution is envisioned by developers of Zeep, but # it is a hack, and should ideally be handled internally by Zeep. # If developers of Zeep ever resume its maintenance, it would make sense to revisit -def _zeep_query_append_skipped( - query: dict, skipped_keys: list, inplace: bool = False, overwrite: bool = False -) -> dict: +def _zeep_query_append_skipped(query: dict, skipped_keys: list, inplace: bool = False, overwrite: bool = False) -> dict: """ This function is used to fix a buggy behaviour of Zeep/BFabric. Specifically, Zeep does not return correct query results if some of the optional parameters are not mentioned in the query. diff --git a/bfabric/src/bfabric/engine/response_format_suds.py b/bfabric/src/bfabric/engine/response_format_suds.py index 20a1f579..eda2988b 100644 --- a/bfabric/src/bfabric/engine/response_format_suds.py +++ b/bfabric/src/bfabric/engine/response_format_suds.py @@ -37,9 +37,7 @@ def suds_asdict_recursive(d: Any, convert_types: bool = False) -> dict[str, Valu items: list[Value] = [] for item in v: if hasattr(item, "__keylist__"): - items.append( - suds_asdict_recursive(item, convert_types=convert_types) - ) + items.append(suds_asdict_recursive(item, convert_types=convert_types)) else: items.append(convert_suds_type(item) if convert_types else item) out[k] = items diff --git a/bfabric/src/bfabric/entities/core/entity.py b/bfabric/src/bfabric/entities/core/entity.py index b5bd2242..78ac0806 100644 --- a/bfabric/src/bfabric/entities/core/entity.py +++ b/bfabric/src/bfabric/entities/core/entity.py @@ -76,9 +76,7 @@ def find_all(cls, ids: list[int], client: Bfabric) -> dict[int, Self]: return cls.__ensure_results_order(ids_requested, results_cached, results_fresh) @classmethod - def find_by( - cls, obj: dict[str, Any], client: Bfabric, max_results: int | None = 100 - ) -> dict[int, Self]: + def find_by(cls, obj: dict[str, Any], client: Bfabric, max_results: int | None = 100) -> dict[int, Self]: """Returns a dictionary of entities that match the given query.""" result = client.read(cls.ENDPOINT, obj=obj, max_results=max_results) return {x["id"]: cls(x, client=client) for x in result} @@ -113,9 +111,7 @@ def __check_ids_list(cls, ids: list[int]) -> list[int]: there are duplicates.""" ids_requested = [int(id) for id in ids] if len(ids_requested) != len(set(ids_requested)): - duplicates = [ - item for item in set(ids_requested) if ids_requested.count(item) > 1 - ] + duplicates = [item for item in set(ids_requested) if ids_requested.count(item) > 1] raise ValueError(f"Duplicate IDs are not allowed, duplicates: {duplicates}") return ids_requested @@ -143,11 +139,7 @@ def __ensure_results_order( ) -> dict[int, Self]: """Ensures the results are in the same order as requested and prints a warning if some results are missing.""" results = {**results_cached, **results_fresh} - results = { - entity_id: results[entity_id] - for entity_id in ids_requested - if entity_id in results - } + results = {entity_id: results[entity_id] for entity_id in ids_requested if entity_id in results} if len(results) != len(ids_requested): logger.warning(f"Only found {len(results)} out of {len(ids_requested)}.") return results diff --git a/bfabric/src/bfabric/entities/core/has_container_mixin.py b/bfabric/src/bfabric/entities/core/has_container_mixin.py index 108c23bc..4f54081b 100644 --- a/bfabric/src/bfabric/entities/core/has_container_mixin.py +++ b/bfabric/src/bfabric/entities/core/has_container_mixin.py @@ -29,21 +29,13 @@ def container(self: EntityProtocol) -> Project | Order: result: Project | Order | None if self.data_dict["container"]["classname"] == Project.ENDPOINT: - result = Project.find( - id=self.data_dict["container"]["id"], client=self._client - ) + result = Project.find(id=self.data_dict["container"]["id"], client=self._client) elif self.data_dict["container"]["classname"] == Order.ENDPOINT: - result = Order.find( - id=self.data_dict["container"]["id"], client=self._client - ) + result = Order.find(id=self.data_dict["container"]["id"], client=self._client) else: - raise ValueError( - f"Unknown container classname: {self.data_dict['container']['classname']}" - ) + raise ValueError(f"Unknown container classname: {self.data_dict['container']['classname']}") if result is None: - raise ValueError( - f"Could not find container with ID {self.data_dict['container']['id']}" - ) + raise ValueError(f"Could not find container with ID {self.data_dict['container']['id']}") return result diff --git a/bfabric/src/bfabric/entities/core/has_many.py b/bfabric/src/bfabric/entities/core/has_many.py index 51748dc4..04af9947 100644 --- a/bfabric/src/bfabric/entities/core/has_many.py +++ b/bfabric/src/bfabric/entities/core/has_many.py @@ -48,9 +48,7 @@ def __get__(self, obj: T | None, objtype: type[T] | None = None) -> _HasManyProx def _get_ids(self, obj: T) -> list[int]: if self._bfabric_field is not None: if self._ids_property is not None: - raise ValueError( - "Exactly one of bfabric_field and ids_property must be set, but both are set" - ) + raise ValueError("Exactly one of bfabric_field and ids_property must be set, but both are set") if self._optional and self._bfabric_field not in obj.data_dict: return [] return [x["id"] for x in obj.data_dict[self._bfabric_field]] @@ -59,9 +57,7 @@ def _get_ids(self, obj: T) -> list[int]: return [] return getattr(obj, self._ids_property) else: - raise ValueError( - "Exactly one of bfabric_field and ids_property must be set, but neither is set" - ) + raise ValueError("Exactly one of bfabric_field and ids_property must be set, but neither is set") class _HasManyProxy(Generic[E]): diff --git a/bfabric/src/bfabric/entities/core/has_one.py b/bfabric/src/bfabric/entities/core/has_one.py index dbbeb7d0..2fdc13ee 100644 --- a/bfabric/src/bfabric/entities/core/has_one.py +++ b/bfabric/src/bfabric/entities/core/has_one.py @@ -13,9 +13,7 @@ class HasOne(Relationship[E]): - def __init__( - self, entity: str, *, bfabric_field: str, optional: bool = False - ) -> None: + def __init__(self, entity: str, *, bfabric_field: str, optional: bool = False) -> None: super().__init__(entity) self._bfabric_field = bfabric_field self._optional = optional diff --git a/bfabric/src/bfabric/entities/core/relationship.py b/bfabric/src/bfabric/entities/core/relationship.py index e7662f9c..f543e63c 100644 --- a/bfabric/src/bfabric/entities/core/relationship.py +++ b/bfabric/src/bfabric/entities/core/relationship.py @@ -17,6 +17,6 @@ def __init__(self, entity: str) -> None: @cached_property def _entity_type(self) -> type[E]: - return importlib.import_module( - f"bfabric.entities.{self._entity_type_name.lower()}" - ).__dict__[self._entity_type_name] + return importlib.import_module(f"bfabric.entities.{self._entity_type_name.lower()}").__dict__[ + self._entity_type_name + ] diff --git a/bfabric/src/bfabric/entities/dataset.py b/bfabric/src/bfabric/entities/dataset.py index a563a47f..4ec1ef10 100644 --- a/bfabric/src/bfabric/entities/dataset.py +++ b/bfabric/src/bfabric/entities/dataset.py @@ -19,9 +19,7 @@ class Dataset(Entity): ENDPOINT: str = "dataset" - def __init__( - self, data_dict: dict[str, Any], client: Bfabric | None = None - ) -> None: + def __init__(self, data_dict: dict[str, Any], client: Bfabric | None = None) -> None: super().__init__(data_dict=data_dict, client=client) def to_polars(self) -> DataFrame: diff --git a/bfabric/src/bfabric/entities/executable.py b/bfabric/src/bfabric/entities/executable.py index f9e263ce..728acebb 100644 --- a/bfabric/src/bfabric/entities/executable.py +++ b/bfabric/src/bfabric/entities/executable.py @@ -16,9 +16,7 @@ class Executable(Entity): storage = HasOne(entity="Storage", bfabric_field="storage", optional=True) - def __init__( - self, data_dict: dict[str, Any], client: Bfabric | None = None - ) -> None: + def __init__(self, data_dict: dict[str, Any], client: Bfabric | None = None) -> None: super().__init__(data_dict=data_dict, client=client) @cached_property diff --git a/bfabric/src/bfabric/entities/externaljob.py b/bfabric/src/bfabric/entities/externaljob.py index 8779b93b..146d9dae 100644 --- a/bfabric/src/bfabric/entities/externaljob.py +++ b/bfabric/src/bfabric/entities/externaljob.py @@ -18,9 +18,7 @@ class ExternalJob(Entity): def __init__(self, data_dict: dict[str, Any], client: Bfabric | None) -> None: super().__init__(data_dict=data_dict, client=client) - executable: HasOne[Executable] = HasOne( - entity="Executable", bfabric_field="executable" - ) + executable: HasOne[Executable] = HasOne(entity="Executable", bfabric_field="executable") @cached_property def workunit(self) -> Workunit | None: @@ -30,8 +28,6 @@ def workunit(self) -> Workunit | None: if self._client is None: raise ValueError("Client must be set to resolve Workunit") - return Workunit.find( - id=self.data_dict["cliententityid"], client=self._client - ) + return Workunit.find(id=self.data_dict["cliententityid"], client=self._client) else: return None diff --git a/bfabric/src/bfabric/entities/multiplexkit.py b/bfabric/src/bfabric/entities/multiplexkit.py index a18847f6..6f78de8f 100644 --- a/bfabric/src/bfabric/entities/multiplexkit.py +++ b/bfabric/src/bfabric/entities/multiplexkit.py @@ -18,9 +18,7 @@ class MultiplexKit(Entity): def __init__(self, data_dict: dict[str, Any], client: Bfabric | None) -> None: super().__init__(data_dict=data_dict, client=client) - multiplex_ids: HasMany[MultiplexId] = HasMany( - "MultiplexId", bfabric_field="multiplexid" - ) + multiplex_ids: HasMany[MultiplexId] = HasMany("MultiplexId", bfabric_field="multiplexid") @cached_property def ids(self) -> pl.DataFrame: diff --git a/bfabric/src/bfabric/entities/resource.py b/bfabric/src/bfabric/entities/resource.py index 74cb6032..144356d8 100644 --- a/bfabric/src/bfabric/entities/resource.py +++ b/bfabric/src/bfabric/entities/resource.py @@ -15,9 +15,7 @@ class Resource(Entity): ENDPOINT = "resource" - def __init__( - self, data_dict: dict[str, Any], client: Bfabric | None = None - ) -> None: + def __init__(self, data_dict: dict[str, Any], client: Bfabric | None = None) -> None: super().__init__(data_dict=data_dict, client=client) storage: HasOne[Storage] = HasOne("Storage", bfabric_field="storage") diff --git a/bfabric/src/bfabric/entities/sample.py b/bfabric/src/bfabric/entities/sample.py index 3b7abdbb..40bf80aa 100644 --- a/bfabric/src/bfabric/entities/sample.py +++ b/bfabric/src/bfabric/entities/sample.py @@ -12,7 +12,5 @@ class Sample(Entity, HasContainerMixin): ENDPOINT = "sample" - def __init__( - self, data_dict: dict[str, Any], client: Bfabric | None = None - ) -> None: + def __init__(self, data_dict: dict[str, Any], client: Bfabric | None = None) -> None: super().__init__(data_dict=data_dict, client=client) diff --git a/bfabric/src/bfabric/entities/storage.py b/bfabric/src/bfabric/entities/storage.py index daf967cb..4d73c5a8 100644 --- a/bfabric/src/bfabric/entities/storage.py +++ b/bfabric/src/bfabric/entities/storage.py @@ -22,8 +22,4 @@ def __init__(self, data_dict: dict[str, Any], client: Bfabric | None) -> None: def scp_prefix(self) -> str | None: """SCP prefix with storage base path included.""" protocol = self.data_dict["protocol"] - return ( - f"{self.data_dict['host']}:{self.data_dict['basepath']}" - if protocol == "scp" - else None - ) + return f"{self.data_dict['host']}:{self.data_dict['basepath']}" if protocol == "scp" else None diff --git a/bfabric/src/bfabric/entities/workunit.py b/bfabric/src/bfabric/entities/workunit.py index 717ef1cc..5214c19b 100644 --- a/bfabric/src/bfabric/entities/workunit.py +++ b/bfabric/src/bfabric/entities/workunit.py @@ -27,29 +27,15 @@ class Workunit(Entity, HasContainerMixin): ENDPOINT = "workunit" - def __init__( - self, data_dict: dict[str, Any], client: Bfabric | None = None - ) -> None: + def __init__(self, data_dict: dict[str, Any], client: Bfabric | None = None) -> None: super().__init__(data_dict=data_dict, client=client) - application: HasOne[Application] = HasOne( - entity="Application", bfabric_field="application" - ) - parameters: HasMany[Parameter] = HasMany( - entity="Parameter", bfabric_field="parameter", optional=True - ) - resources: HasMany[Resource] = HasMany( - entity="Resource", bfabric_field="resource", optional=True - ) - input_resources: HasMany[Resource] = HasMany( - entity="Resource", bfabric_field="inputresource", optional=True - ) - input_dataset: HasOne[Dataset] = HasOne( - entity="Dataset", bfabric_field="inputdataset", optional=True - ) - external_jobs: HasMany[ExternalJob] = HasMany( - entity="ExternalJob", bfabric_field="externaljob", optional=True - ) + application: HasOne[Application] = HasOne(entity="Application", bfabric_field="application") + parameters: HasMany[Parameter] = HasMany(entity="Parameter", bfabric_field="parameter", optional=True) + resources: HasMany[Resource] = HasMany(entity="Resource", bfabric_field="resource", optional=True) + input_resources: HasMany[Resource] = HasMany(entity="Resource", bfabric_field="inputresource", optional=True) + input_dataset: HasOne[Dataset] = HasOne(entity="Dataset", bfabric_field="inputdataset", optional=True) + external_jobs: HasMany[ExternalJob] = HasMany(entity="ExternalJob", bfabric_field="externaljob", optional=True) @cached_property def parameter_values(self) -> dict[str, Any]: @@ -59,13 +45,9 @@ def parameter_values(self) -> dict[str, Any]: def store_output_folder(self) -> Path: """Relative path in the storage for the workunit output.""" if self.application is None: - raise ValueError( - "Cannot determine the storage path without an application." - ) + raise ValueError("Cannot determine the storage path without an application.") if self.application.storage is None: - raise ValueError( - "Cannot determine the storage path without an application storage configuration." - ) + raise ValueError("Cannot determine the storage path without an application storage configuration.") date = dateutil.parser.parse(self.data_dict["created"]) return Path( f"{self.application.storage['projectfolderprefix']}{self.container.id}", diff --git a/bfabric/src/bfabric/errors.py b/bfabric/src/bfabric/errors.py index 501d1c58..a8d2a634 100644 --- a/bfabric/src/bfabric/errors.py +++ b/bfabric/src/bfabric/errors.py @@ -30,10 +30,6 @@ def get_response_errors(response: Any, endpoint: str) -> list[BfabricRequestErro if getattr(response, "errorreport", None): return [BfabricRequestError(response.errorreport)] elif endpoint in response: - return [ - BfabricRequestError(r.errorreport) - for r in response[endpoint] - if getattr(r, "errorreport", None) - ] + return [BfabricRequestError(r.errorreport) for r in response[endpoint] if getattr(r, "errorreport", None)] else: return [] diff --git a/bfabric/src/bfabric/examples/compare_zeep_suds_pagination.py b/bfabric/src/bfabric/examples/compare_zeep_suds_pagination.py index cf9b7baa..04f621d8 100644 --- a/bfabric/src/bfabric/examples/compare_zeep_suds_pagination.py +++ b/bfabric/src/bfabric/examples/compare_zeep_suds_pagination.py @@ -37,9 +37,7 @@ def _calc_query(config, auth, engine, endpoint): return_id_only=False, includedeletableupdateable=True, ) - response_dict = response_class.to_list_dict( - drop_empty=True, have_sort_responses=True - ) + response_dict = response_class.to_list_dict(drop_empty=True, have_sort_responses=True) return list_dict_to_df(response_dict) @@ -58,9 +56,7 @@ def _set_partition_test(a, b) -> bool: return (len(unique1) == 0) and (len(unique2) == 0) -def dataframe_pagination_test( - config, auth, endpoint, use_cached: bool = False, store_cached: bool = True -): +def dataframe_pagination_test(config, auth, endpoint, use_cached: bool = False, store_cached: bool = True): pwd_zeep = "tmp_zeep_" + endpoint + ".csv" pwd_suds = "tmp_suds_" + endpoint + ".csv" @@ -101,7 +97,5 @@ def dataframe_pagination_test( config, auth = get_system_auth(config_env="TEST") -result = dataframe_pagination_test( - config, auth, "user", use_cached=False, store_cached=True -) +result = dataframe_pagination_test(config, auth, "user", use_cached=False, store_cached=True) report_test_result(result, "pagination") diff --git a/bfabric/src/bfabric/examples/compare_zeep_suds_query.py b/bfabric/src/bfabric/examples/compare_zeep_suds_query.py index 26b902d8..04e00f75 100644 --- a/bfabric/src/bfabric/examples/compare_zeep_suds_query.py +++ b/bfabric/src/bfabric/examples/compare_zeep_suds_query.py @@ -48,9 +48,7 @@ def read_suds(wsdl, fullQuery, raw=True): return suds_asdict_recursive(ret, convert_types=True) -def full_query( - auth: BfabricAuth, query: dict, includedeletableupdateable: bool = False -) -> dict: +def full_query(auth: BfabricAuth, query: dict, includedeletableupdateable: bool = False) -> dict: thisQuery = deepcopy(query) thisQuery["includedeletableupdateable"] = includedeletableupdateable @@ -142,15 +140,11 @@ def recursive_comparison(generic_container1, generic_container2, prefix: list) - print(prefix, "Not in 2: ", k, "=", generic_container1[k]) matched = False else: - matched_recursive = recursive_comparison( - generic_container1[k], generic_container2[k], prefix + [k] - ) + matched_recursive = recursive_comparison(generic_container1[k], generic_container2[k], prefix + [k]) matched = matched and matched_recursive elif isinstance(generic_container1, list): if len(generic_container1) != len(generic_container2): - print( - prefix, "length", len(generic_container1), "!=", len(generic_container2) - ) + print(prefix, "length", len(generic_container1), "!=", len(generic_container2)) matched = False else: for i, (el1, el2) in enumerate(zip(generic_container1, generic_container2)): diff --git a/bfabric/src/bfabric/examples/zeep_debug.py b/bfabric/src/bfabric/examples/zeep_debug.py index 56dda1b4..15c7e287 100644 --- a/bfabric/src/bfabric/examples/zeep_debug.py +++ b/bfabric/src/bfabric/examples/zeep_debug.py @@ -18,9 +18,7 @@ """ -def full_query( - auth: BfabricAuth, query: dict, includedeletableupdateable: bool = False -) -> dict: +def full_query(auth: BfabricAuth, query: dict, includedeletableupdateable: bool = False) -> dict: thisQuery = deepcopy(query) thisQuery["includedeletableupdateable"] = includedeletableupdateable diff --git a/bfabric/src/bfabric/experimental/entity_lookup_cache.py b/bfabric/src/bfabric/experimental/entity_lookup_cache.py index ffbeb41c..24c13c17 100644 --- a/bfabric/src/bfabric/experimental/entity_lookup_cache.py +++ b/bfabric/src/bfabric/experimental/entity_lookup_cache.py @@ -51,9 +51,7 @@ class EntityLookupCache: __class_instance = None def __init__(self, max_size: int = 0) -> None: - self._caches: dict[type[Entity], Cache[Entity | None]] = defaultdict( - lambda: Cache(max_size=max_size) - ) + self._caches: dict[type[Entity], Cache[Entity | None]] = defaultdict(lambda: Cache(max_size=max_size)) def contains(self, entity_type: type[Entity], entity_id: int) -> bool: """Returns whether the cache contains an entity with the given type and ID.""" @@ -68,24 +66,14 @@ def get(self, entity_type: type[E], entity_id: int) -> E | None: logger.debug(f"Cache miss for entity {entity_type} with ID {entity_id}") return None - def get_all( - self, entity_type: type[Entity], entity_ids: list[int] - ) -> dict[int, Entity]: + def get_all(self, entity_type: type[Entity], entity_ids: list[int]) -> dict[int, Entity]: """Returns a dictionary of entities with the given type and IDs, containing only the entities that exist in the cache. """ - results = { - entity_id: self.get(entity_type, entity_id) for entity_id in entity_ids - } - return { - entity_id: result - for entity_id, result in results.items() - if result is not None - } - - def put( - self, entity_type: type[Entity], entity_id: int, entity: Entity | None - ) -> None: + results = {entity_id: self.get(entity_type, entity_id) for entity_id in entity_ids} + return {entity_id: result for entity_id, result in results.items() if result is not None} + + def put(self, entity_type: type[Entity], entity_id: int, entity: Entity | None) -> None: """Puts an entity with the given type and ID into the cache.""" logger.debug(f"Caching entity {entity_type} with ID {entity_id}") self._caches[entity_type].put(entity_id, entity) diff --git a/bfabric/src/bfabric/experimental/multi_query.py b/bfabric/src/bfabric/experimental/multi_query.py index 29934017..22ad5253 100644 --- a/bfabric/src/bfabric/experimental/multi_query.py +++ b/bfabric/src/bfabric/experimental/multi_query.py @@ -42,9 +42,7 @@ def read_multi( """ # TODO add `check` parameter response_tot = ResultContainer([], total_pages_api=0) - obj_extended = deepcopy( - obj - ) # Make a copy of the query, not to make edits to the argument + obj_extended = deepcopy(obj) # Make a copy of the query, not to make edits to the argument # Iterate over request chunks that fit into a single API page for page_vals in page_iter(multi_query_vals): @@ -59,9 +57,7 @@ def read_multi( # exceptions to this? -> yes, when not reading by id but for instance matching a pattern, one might not # be aware that they might accidentally pull the whole db, so it's better to have max_results here as well # but it is not completely trivial to implement - response_this = self._client.read( - endpoint, obj_extended, max_results=None, return_id_only=return_id_only - ) + response_this = self._client.read(endpoint, obj_extended, max_results=None, return_id_only=return_id_only) response_tot.extend(response_this, reset_total_pages_api=True) return response_tot @@ -96,9 +92,7 @@ def delete_multi(self, endpoint: str, id_list: list[int]) -> ResultContainer: return response_tot - def exists_multi( - self, endpoint: str, key: str, value: list[int | str] | int | str - ) -> bool | list[bool]: + def exists_multi(self, endpoint: str, key: str, value: list[int | str] | int | str) -> bool | list[bool]: """ :param endpoint: endpoint :param key: A key for the query (e.g. id or name) @@ -108,9 +102,7 @@ def exists_multi( """ is_scalar = isinstance(value, (int, str)) if is_scalar: - return self._client.exists( - endpoint=endpoint, key=key, value=value, check=True - ) + return self._client.exists(endpoint=endpoint, key=key, value=value, check=True) elif not isinstance(value, list): raise ValueError("Unexpected data type", type(value)) diff --git a/bfabric/src/bfabric/experimental/upload_dataset.py b/bfabric/src/bfabric/experimental/upload_dataset.py index e5598c81..f10c451f 100644 --- a/bfabric/src/bfabric/experimental/upload_dataset.py +++ b/bfabric/src/bfabric/experimental/upload_dataset.py @@ -35,10 +35,7 @@ def polars_to_bfabric_dataset( ] items = [ { - "field": [ - {"attributeposition": i_field + 1, "value": value} - for i_field, value in enumerate(row) - ], + "field": [{"attributeposition": i_field + 1, "value": value} for i_field, value in enumerate(row)], "position": i_row + 1, } for i_row, row in enumerate(data.iter_rows()) @@ -57,9 +54,7 @@ def bfabric_save_csv2dataset( invalid_characters: str, ) -> None: """Creates a dataset in B-Fabric from a csv file.""" - data = pl.read_csv( - csv_file, separator=sep, has_header=has_header, infer_schema_length=None - ) + data = pl.read_csv(csv_file, separator=sep, has_header=has_header, infer_schema_length=None) check_for_invalid_characters(data=data, invalid_characters=invalid_characters) obj = polars_to_bfabric_dataset(data) obj["name"] = dataset_name @@ -75,9 +70,7 @@ def check_for_invalid_characters(data: pl.DataFrame, invalid_characters: str) -> """Raises a RuntimeError if any cell in the DataFrame contains an invalid character.""" if not invalid_characters: return - invalid_columns_df = data.select( - pl.col(pl.String).str.contains_any(list(invalid_characters)).any() - ) + invalid_columns_df = data.select(pl.col(pl.String).str.contains_any(list(invalid_characters)).any()) if invalid_columns_df.is_empty(): return invalid_columns = ( diff --git a/bfabric/src/bfabric/experimental/workunit_definition.py b/bfabric/src/bfabric/experimental/workunit_definition.py index d2a67628..369150ce 100644 --- a/bfabric/src/bfabric/experimental/workunit_definition.py +++ b/bfabric/src/bfabric/experimental/workunit_definition.py @@ -78,9 +78,7 @@ class WorkunitDefinition(BaseModel): registration: WorkunitRegistrationDefinition | None @classmethod - def from_ref( - cls, workunit: Path | int, client: Bfabric, cache_file: Path | None = None - ) -> WorkunitDefinition: + def from_ref(cls, workunit: Path | int, client: Bfabric, cache_file: Path | None = None) -> WorkunitDefinition: """Loads the workunit definition from the provided reference, which can be a path to a YAML file, or a workunit ID. diff --git a/bfabric/src/bfabric/results/response_format_dict.py b/bfabric/src/bfabric/results/response_format_dict.py index b4587c80..dc5daeab 100644 --- a/bfabric/src/bfabric/results/response_format_dict.py +++ b/bfabric/src/bfabric/results/response_format_dict.py @@ -24,9 +24,7 @@ def _recursive_drop_empty(response_elem: list | dict) -> None: for el in response_elem: _recursive_drop_empty(el) elif isinstance(response_elem, dict): - keys_to_delete = ( - [] - ) # NOTE: Avoid deleting keys inside iterator, may break iterator + keys_to_delete = [] # NOTE: Avoid deleting keys inside iterator, may break iterator for k, v in response_elem.items(): if (v is None) or (isinstance(v, list) and len(v) == 0): keys_to_delete += [k] @@ -37,9 +35,7 @@ def _recursive_drop_empty(response_elem: list | dict) -> None: @overload -def drop_empty_elements( - response: list[dict[str, Any]], inplace: bool -) -> list[dict[str, Any]]: ... +def drop_empty_elements(response: list[dict[str, Any]], inplace: bool) -> list[dict[str, Any]]: ... @overload @@ -72,9 +68,7 @@ def _recursive_map_keys(response_elem: list | dict, keymap: dict) -> None: for el in response_elem: _recursive_map_keys(el, keymap) elif isinstance(response_elem, dict): - keys_to_delete = ( - [] - ) # NOTE: Avoid deleting keys inside iterator, may break iterator + keys_to_delete = [] # NOTE: Avoid deleting keys inside iterator, may break iterator for k, v in response_elem.items(): _recursive_map_keys(v, keymap) if k in keymap: @@ -85,9 +79,7 @@ def _recursive_map_keys(response_elem: list | dict, keymap: dict) -> None: del response_elem[k] # Delete old key -def map_element_keys( - response: list | dict, keymap: dict, inplace: bool = True -) -> list | dict: +def map_element_keys(response: list | dict, keymap: dict, inplace: bool = True) -> list | dict: """ Iterates over all nested lists, dictionaries and basic values. Whenever a dictionary key is found for which the mapping is requested, that the key is renamed to the corresponding mapped one @@ -121,9 +113,7 @@ def _recursive_sort_dicts_by_key(response_elem: list | dict) -> None: _recursive_sort_dicts_by_key(v) -def sort_dicts_by_key( - response: list | dict, inplace: bool = True -) -> list | dict | None: +def sort_dicts_by_key(response: list | dict, inplace: bool = True) -> list | dict | None: """ Iterates over all nested lists, dictionaries and basic values. Whenever a nested dictionary is found, it is sorted by key by converting into OrderedDict and back @@ -137,9 +127,7 @@ def sort_dicts_by_key( return response_filtered -def clean_result( - result: dict, drop_underscores_suds: bool = True, sort_keys: bool = False -) -> dict: +def clean_result(result: dict, drop_underscores_suds: bool = True, sort_keys: bool = False) -> dict: """ :param result: the response dictionary to clean :param drop_underscores_suds: if True, the keys of the dictionaries in the response will have leading diff --git a/bfabric/src/bfabric/results/result_container.py b/bfabric/src/bfabric/results/result_container.py index 11696202..e7edf66e 100644 --- a/bfabric/src/bfabric/results/result_container.py +++ b/bfabric/src/bfabric/results/result_container.py @@ -79,9 +79,7 @@ def errors(self) -> list[BfabricRequestError]: """List of errors that occurred during the query. An empty list indicates success.""" return self._errors - def extend( - self, other: ResultContainer, reset_total_pages_api: bool = False - ) -> None: + def extend(self, other: ResultContainer, reset_total_pages_api: bool = False) -> None: """Merges the results of `other` into this container. :param other: The container whose elements to append to the end of this container :param reset_total_pages_api: If True, the total_pages_api attribute will be reset to None diff --git a/bfabric/src/bfabric/utils/paginator.py b/bfabric/src/bfabric/utils/paginator.py index 9d575e63..d4004982 100644 --- a/bfabric/src/bfabric/utils/paginator.py +++ b/bfabric/src/bfabric/utils/paginator.py @@ -10,9 +10,7 @@ BFABRIC_QUERY_LIMIT = 100 -def page_iter( - objs: list, page_size: int = BFABRIC_QUERY_LIMIT -) -> Generator[list, None, None]: +def page_iter(objs: list, page_size: int = BFABRIC_QUERY_LIMIT) -> Generator[list, None, None]: """ :param objs: A list of objects to provide to bfabric as part of a query :param page_size: Number of objects per page @@ -48,12 +46,7 @@ def compute_requested_pages( # Determine the page indices to request idx_max_return = math.ceil((n_item_return_max + n_item_offset) / n_item_per_page) - idx_arr = [ - idx + index_start - for idx in range( - n_item_offset // n_item_per_page, min(n_page_total, idx_max_return) - ) - ] + idx_arr = [idx + index_start for idx in range(n_item_offset // n_item_per_page, min(n_page_total, idx_max_return))] # Determine the initial offset on the first page initial_offset = min(n_item_offset, n_item_return_max) % n_item_per_page diff --git a/bfabric/src/bfabric/utils/polars_utils.py b/bfabric/src/bfabric/utils/polars_utils.py index 8bbe2e6c..ac384242 100644 --- a/bfabric/src/bfabric/utils/polars_utils.py +++ b/bfabric/src/bfabric/utils/polars_utils.py @@ -9,11 +9,6 @@ def flatten_relations(df: pl.DataFrame) -> pl.DataFrame: All non-struct fields will be kept as is. If there are conflicts this raises an error. """ - struct_cols = [ - col for col, dtype in zip(df.columns, df.dtypes) if isinstance(dtype, pl.Struct) - ] + struct_cols = [col for col, dtype in zip(df.columns, df.dtypes) if isinstance(dtype, pl.Struct)] flat_cols = [col for col in df.columns if col not in struct_cols] - return df.select( - flat_cols - + [pl.col(col).struct.unnest().name.prefix(f"{col}_") for col in struct_cols] - ) + return df.select(flat_cols + [pl.col(col).struct.unnest().name.prefix(f"{col}_") for col in struct_cols]) diff --git a/bfabric/src/bfabric/wrapper_creator/bfabric_external_job.py b/bfabric/src/bfabric/wrapper_creator/bfabric_external_job.py index da25abab..424dd6f0 100644 --- a/bfabric/src/bfabric/wrapper_creator/bfabric_external_job.py +++ b/bfabric/src/bfabric/wrapper_creator/bfabric_external_job.py @@ -27,9 +27,7 @@ def __init__(self, login=None, password=None, externaljobid=None): def logger(self, msg): if self.externaljobid: - super().save_object( - "externaljob", {"id": self.externaljobid, "logthis": str(msg)} - ) + super().save_object("externaljob", {"id": self.externaljobid, "logthis": str(msg)}) else: print(str(msg)) @@ -40,12 +38,8 @@ def save_object(self, endpoint, obj, debug=None): return res def get_workunitid_of_externaljob(self): - print( - f"DEBUG get_workunitid_of_externaljob self.externaljobid={self.externaljobid}" - ) - res = self.read_object(endpoint="externaljob", obj={"id": self.externaljobid})[ - 0 - ] + print(f"DEBUG get_workunitid_of_externaljob self.externaljobid={self.externaljobid}") + res = self.read_object(endpoint="externaljob", obj={"id": self.externaljobid})[0] print(res) print("DEBUG END") workunit_id = None @@ -62,13 +56,9 @@ def get_application_name(self): raise ValueError("no workunit available for the given externaljobid.") workunit = self.read_object(endpoint="workunit", obj={"id": workunitid})[0] if workunit is None: - raise ValueError( - "ERROR: no workunit available for the given externaljobid." - ) + raise ValueError("ERROR: no workunit available for the given externaljobid.") assert isinstance(workunit._id, int) - application = self.read_object( - "application", obj={"id": workunit.application._id} - )[0] + application = self.read_object("application", obj={"id": workunit.application._id})[0] return application.name.replace(" ", "_") def get_executable_of_externaljobid(self): @@ -85,9 +75,7 @@ def get_executable_of_externaljobid(self): return None executables = list() - for executable in self.read_object( - endpoint="executable", obj={"workunitid": workunitid} - ): + for executable in self.read_object(endpoint="executable", obj={"workunitid": workunitid}): if hasattr(executable, "base64"): executables.append(executable) diff --git a/bfabric/src/bfabric/wrapper_creator/bfabric_submitter.py b/bfabric/src/bfabric/wrapper_creator/bfabric_submitter.py index d6b1079f..b510c178 100644 --- a/bfabric/src/bfabric/wrapper_creator/bfabric_submitter.py +++ b/bfabric/src/bfabric/wrapper_creator/bfabric_submitter.py @@ -75,9 +75,7 @@ def __init__( self.application = self.workunit.application default_config = self.slurm_dict.get(self.application["name"], {}) - self.partition = self.parameters.get( - "partition", default_config.get("partition") - ) + self.partition = self.parameters.get("partition", default_config.get("partition")) self.nodelist = self.parameters.get("nodelist", default_config.get("nodelist")) self.memory = self.parameters.get("memory", default_config.get("memory")) @@ -93,9 +91,7 @@ def submit_slurm(self, script: str = "/tmp/runme.bash") -> None: res_slurm_batch = slurm.sbatch(script=script) logger.debug(f"{res_slurm_batch}") - def compose_bash_script( - self, configuration=None, configuration_parser=lambda x: yaml.safe_load(x) - ) -> str: + def compose_bash_script(self, configuration=None, configuration_parser=lambda x: yaml.safe_load(x)) -> str: """ composes the bash script which is executed by the submitter (sun grid engine). as an argument it takes a configuration file, e.g., yaml, xml, json, or whatsoever, and a parser function. @@ -242,9 +238,7 @@ def submitter_yaml(self) -> None: return None """ - executables = Executable.find_by( - {"workunitid": self.workunit.id}, client=self._client - ).values() + executables = Executable.find_by({"workunitid": self.workunit.id}, client=self._client).values() for executable in executables: if not executable["base64"]: continue diff --git a/bfabric/src/bfabric/wrapper_creator/bfabric_wrapper_creator.py b/bfabric/src/bfabric/wrapper_creator/bfabric_wrapper_creator.py index 78d0b17e..8b7753a5 100644 --- a/bfabric/src/bfabric/wrapper_creator/bfabric_wrapper_creator.py +++ b/bfabric/src/bfabric/wrapper_creator/bfabric_wrapper_creator.py @@ -64,21 +64,15 @@ def create_output_resource(self) -> Resource: # Determine the correct path output_folder = self._workunit.store_output_folder - output_filename = ( - f"{resource_id}.{self._application.data_dict['outputfileformat']}" - ) + output_filename = f"{resource_id}.{self._application.data_dict['outputfileformat']}" relative_path = str(output_folder / output_filename) # Save the path logger.info("Saving correct path") - result = self._client.save( - "resource", {"id": resource_id, "relativepath": relative_path} - ) + result = self._client.save("resource", {"id": resource_id, "relativepath": relative_path}) return Resource(result[0]) - def create_log_resource( - self, variant: Literal["out", "err"], output_resource: Resource - ) -> Resource: + def create_log_resource(self, variant: Literal["out", "err"], output_resource: Resource) -> Resource: logger.info("Creating log resource") result = self._client.save( "resource", @@ -95,9 +89,7 @@ def get_application_section(self, output_resource: Resource) -> dict[str, Any]: logger.info("Creating application section") output_url = f"bfabric@{self._application.storage.data_dict['host']}:{self._application.storage.data_dict['basepath']}{output_resource.data_dict['relativepath']}" inputs = defaultdict(list) - for resource in Resource.find_all( - self.workunit_definition.execution.resources, client=self._client - ).values(): + for resource in Resource.find_all(self.workunit_definition.execution.resources, client=self._client).values(): inputs[resource.workunit.application["name"]].append( f"bfabric@{resource.storage.scp_prefix}{resource.data_dict['relativepath']}" ) @@ -133,13 +125,9 @@ def get_job_configuration_section( } inputs = defaultdict(list) - for resource in Resource.find_all( - self.workunit_definition.execution.resources, client=self._client - ).values(): + for resource in Resource.find_all(self.workunit_definition.execution.resources, client=self._client).values(): web_url = Resource({"id": resource.id}, client=self._client).web_url - inputs[resource.workunit.application["name"]].append( - {"resource_id": resource.id, "resource_url": web_url} - ) + inputs[resource.workunit.application["name"]].append({"resource_id": resource.id, "resource_url": web_url}) return { "executable": str(self._workunit.application.executable["program"]), @@ -163,35 +151,20 @@ def get_job_configuration_section( @cached_property def _order(self) -> Order | None: - return ( - self._workunit.container - if isinstance(self._workunit.container, Order) - else None - ) + return self._workunit.container if isinstance(self._workunit.container, Order) else None @cached_property def _project(self) -> Project | None: - return ( - self._workunit.container - if isinstance(self._workunit.container, Project) - else self._order.project - ) + return self._workunit.container if isinstance(self._workunit.container, Project) else self._order.project @cached_property def _fasta_sequence(self) -> str: if self._order is not None and "fastasequence" in self._order.data_dict: - return "\n".join( - [ - x.strip() - for x in str(self._order.data_dict["fastasequence"]).split("\r") - ] - ) + return "\n".join([x.strip() for x in str(self._order.data_dict["fastasequence"]).split("\r")]) else: return "" - def write_results( - self, config_serialized: str - ) -> tuple[dict[str, Any], dict[str, Any]]: + def write_results(self, config_serialized: str) -> tuple[dict[str, Any], dict[str, Any]]: logger.info("Saving executable") yaml_workunit_executable = self._client.save( "executable", @@ -221,26 +194,18 @@ def write_results( logger.info(yaml_workunit_externaljob) logger.info("Setting external job status to 'done'") - self._client.save( - "externaljob", {"id": self._external_job_id, "status": "done"} - ) + self._client.save("externaljob", {"id": self._external_job_id, "status": "done"}) return yaml_workunit_executable, yaml_workunit_externaljob def create(self) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]]: """Creates the YAML file external job and resources, and registers everything in B-Fabric.""" output_resource = self.create_output_resource() - stdout_resource = self.create_log_resource( - variant="out", output_resource=output_resource - ) - stderr_resource = self.create_log_resource( - variant="err", output_resource=output_resource - ) + stdout_resource = self.create_log_resource(variant="out", output_resource=output_resource) + stderr_resource = self.create_log_resource(variant="err", output_resource=output_resource) config_dict = { - "application": self.get_application_section( - output_resource=output_resource - ), + "application": self.get_application_section(output_resource=output_resource), "job_configuration": self.get_job_configuration_section( output_resource=output_resource, stdout_resource=stdout_resource, @@ -248,7 +213,5 @@ def create(self) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]]: ), } config_serialized = yaml.safe_dump(config_dict) - yaml_workunit_executable, yaml_workunit_externaljob = self.write_results( - config_serialized=config_serialized - ) + yaml_workunit_executable, yaml_workunit_externaljob = self.write_results(config_serialized=config_serialized) return config_dict, yaml_workunit_executable, yaml_workunit_externaljob diff --git a/bfabric/src/bfabric/wrapper_creator/gridengine.py b/bfabric/src/bfabric/wrapper_creator/gridengine.py index bc10ddf9..22dd6e92 100755 --- a/bfabric/src/bfabric/wrapper_creator/gridengine.py +++ b/bfabric/src/bfabric/wrapper_creator/gridengine.py @@ -85,9 +85,7 @@ def qsub(self, script, arguments=""): return try: - qsub_process = subprocess.Popen( - qsub_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False - ) + qsub_process = subprocess.Popen(qsub_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False) stdout, stderr = qsub_process.communicate() return stdout diff --git a/app_runner/.gitignore b/bfabric_app_runner/.gitignore similarity index 100% rename from app_runner/.gitignore rename to bfabric_app_runner/.gitignore diff --git a/app_runner/README.md b/bfabric_app_runner/README.md similarity index 100% rename from app_runner/README.md rename to bfabric_app_runner/README.md diff --git a/app_runner/deploy/build.sh b/bfabric_app_runner/deploy/build.sh similarity index 100% rename from app_runner/deploy/build.sh rename to bfabric_app_runner/deploy/build.sh diff --git a/app_runner/deploy/build_steps.sh b/bfabric_app_runner/deploy/build_steps.sh similarity index 84% rename from app_runner/deploy/build_steps.sh rename to bfabric_app_runner/deploy/build_steps.sh index 56b7dd04..150fa39d 100644 --- a/app_runner/deploy/build_steps.sh +++ b/bfabric_app_runner/deploy/build_steps.sh @@ -7,5 +7,5 @@ python -m venv /work/venv source /work/venv/bin/activate uv pip install . uv pip install pyinstaller -pyinstaller -y --onedir --name "${TARGET_NAME}" --distpath "${TARGET_DIR}" src/app_runner/cli/__main__.py +pyinstaller -y --onedir --name "${TARGET_NAME}" --distpath "${TARGET_DIR}" src/bfabric_app_runner/cli/__main__.py deactivate diff --git a/app_runner/deploy/builder/Dockerfile b/bfabric_app_runner/deploy/builder/Dockerfile similarity index 100% rename from app_runner/deploy/builder/Dockerfile rename to bfabric_app_runner/deploy/builder/Dockerfile diff --git a/app_runner/docs/Makefile b/bfabric_app_runner/docs/Makefile similarity index 100% rename from app_runner/docs/Makefile rename to bfabric_app_runner/docs/Makefile diff --git a/app_runner/docs/_design_notes/_app_definition.md b/bfabric_app_runner/docs/_design_notes/_app_definition.md similarity index 100% rename from app_runner/docs/_design_notes/_app_definition.md rename to bfabric_app_runner/docs/_design_notes/_app_definition.md diff --git a/app_runner/docs/_design_notes/_scheduling.md b/bfabric_app_runner/docs/_design_notes/_scheduling.md similarity index 100% rename from app_runner/docs/_design_notes/_scheduling.md rename to bfabric_app_runner/docs/_design_notes/_scheduling.md diff --git a/app_runner/docs/architecture/overview.md b/bfabric_app_runner/docs/architecture/overview.md similarity index 100% rename from app_runner/docs/architecture/overview.md rename to bfabric_app_runner/docs/architecture/overview.md diff --git a/app_runner/docs/architecture/uml/app_model.plantuml b/bfabric_app_runner/docs/architecture/uml/app_model.plantuml similarity index 100% rename from app_runner/docs/architecture/uml/app_model.plantuml rename to bfabric_app_runner/docs/architecture/uml/app_model.plantuml diff --git a/app_runner/docs/architecture/uml/app_runner_activity.plantuml b/bfabric_app_runner/docs/architecture/uml/app_runner_activity.plantuml similarity index 100% rename from app_runner/docs/architecture/uml/app_runner_activity.plantuml rename to bfabric_app_runner/docs/architecture/uml/app_runner_activity.plantuml diff --git a/app_runner/docs/changelog.md b/bfabric_app_runner/docs/changelog.md similarity index 87% rename from app_runner/docs/changelog.md rename to bfabric_app_runner/docs/changelog.md index 2be94f9c..34e265c0 100644 --- a/app_runner/docs/changelog.md +++ b/bfabric_app_runner/docs/changelog.md @@ -4,9 +4,31 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ## \[Unreleased\] +## \[0.0.15\] - 2025-02-06 + +### Added + +- New input type `file` which replaces `file_scp` and preserves timestamps whenever possible and allows to create + symlinks instead of copying the file, as needed. +- `BfabricOrderFastaSpec.required` which allows specifying whether the order fasta is required or not + +### Changed + +- Better error when app version is not found. + +### Fixed + +- Config: Log messages are shown by default again. + +## \[0.0.14\] - 2025-01-30 + +### Fixed + +- Correctly consume bfabricPy from PyPI. + ## \[0.0.13\] - 2025-01-28 -## Added +### Added - `WorkunitDefinition.registration.workunit_name` field. diff --git a/app_runner/docs/conf.py b/bfabric_app_runner/docs/conf.py similarity index 100% rename from app_runner/docs/conf.py rename to bfabric_app_runner/docs/conf.py diff --git a/bfabric_app_runner/docs/index.md b/bfabric_app_runner/docs/index.md new file mode 100644 index 00000000..6820f22f --- /dev/null +++ b/bfabric_app_runner/docs/index.md @@ -0,0 +1,26 @@ +## Install App Runner + +To install the most recent released version: + +```bash +uv tool install bfabric_app_runner +``` + +To install a development version: + +```bash +uv tool install bfabric_app_runner@git+https://github.com/fgcz/bfabricPy.git@main#egg=bfabric_app_runner&subdirectory=bfabric_app_runner +``` + +## Contents + +```{toctree} +:glob: +workunit_definition +architecture/overview +specs/input_specification +specs/output_specification +specs/app_specification +changelog +* +``` diff --git a/app_runner/docs/make.bat b/bfabric_app_runner/docs/make.bat similarity index 100% rename from app_runner/docs/make.bat rename to bfabric_app_runner/docs/make.bat diff --git a/app_runner/docs/plantuml_wrapper.sh b/bfabric_app_runner/docs/plantuml_wrapper.sh similarity index 100% rename from app_runner/docs/plantuml_wrapper.sh rename to bfabric_app_runner/docs/plantuml_wrapper.sh diff --git a/app_runner/docs/specs/app_specification.md b/bfabric_app_runner/docs/specs/app_specification.md similarity index 100% rename from app_runner/docs/specs/app_specification.md rename to bfabric_app_runner/docs/specs/app_specification.md diff --git a/app_runner/docs/specs/input_specification.md b/bfabric_app_runner/docs/specs/input_specification.md similarity index 100% rename from app_runner/docs/specs/input_specification.md rename to bfabric_app_runner/docs/specs/input_specification.md diff --git a/app_runner/docs/specs/output_specification.md b/bfabric_app_runner/docs/specs/output_specification.md similarity index 100% rename from app_runner/docs/specs/output_specification.md rename to bfabric_app_runner/docs/specs/output_specification.md diff --git a/app_runner/docs/workunit_definition.md b/bfabric_app_runner/docs/workunit_definition.md similarity index 100% rename from app_runner/docs/workunit_definition.md rename to bfabric_app_runner/docs/workunit_definition.md diff --git a/app_runner/pyproject.toml b/bfabric_app_runner/pyproject.toml similarity index 77% rename from app_runner/pyproject.toml rename to bfabric_app_runner/pyproject.toml index 85ee4fbe..a084ce5d 100644 --- a/app_runner/pyproject.toml +++ b/bfabric_app_runner/pyproject.toml @@ -3,23 +3,23 @@ requires = ["hatchling"] build-backend = "hatchling.build" [project] -name = "app_runner" +name = "bfabric_app_runner" description = "Application runner for B-Fabric apps" -version = "0.0.13" +version = "0.0.15" license = { text = "GPL-3.0" } authors = [ {name = "Leonardo Schwarz", email = "leonardo.schwarz@fgcz.ethz.ch"}, ] requires-python = ">=3.12" dependencies = [ - "bfabric", + "bfabric>=1.13.19", "pydantic", "glom", "mako", ] [project.scripts] -"bfabric-app-runner"="app_runner.cli.__main__:app" +"bfabric-app-runner"="bfabric_app_runner.cli.__main__:app" [project.optional-dependencies] doc = [ @@ -39,14 +39,16 @@ dev = [ test = ["pytest", "pytest-mock", "logot"] [tool.uv] -reinstall-package = ["bfabric", "bfabric_scripts", "app_runner"] +reinstall-package = ["bfabric", "bfabric_scripts", "bfabric_app_runner"] [tool.black] line-length = 120 +target-version = ["py311"] [tool.ruff] line-length = 120 indent-width = 4 +target-version = "py311" [tool.ruff.lint] select = ["ANN", "BLE", "D103", "E", "EXE", "F", "N", "PLW", "PTH", "SIM", "TCH", "UP", "W191"] @@ -60,4 +62,4 @@ classmethod-decorators = [ [tool.ruff.lint.per-file-ignores] # This is needed because of false positives in cyclopts code -"**/app_runner/cli/**" = ["TCH001", "TCH002", "TCH003"] +"**/bfabric_app_runner/cli/**" = ["TCH001", "TCH002", "TCH003"] diff --git a/app_runner/src/app_runner/__init__.py b/bfabric_app_runner/src/bfabric_app_runner/__init__.py similarity index 100% rename from app_runner/src/app_runner/__init__.py rename to bfabric_app_runner/src/bfabric_app_runner/__init__.py diff --git a/app_runner/src/app_runner/app_runner/__init__.py b/bfabric_app_runner/src/bfabric_app_runner/app_runner/__init__.py similarity index 100% rename from app_runner/src/app_runner/app_runner/__init__.py rename to bfabric_app_runner/src/bfabric_app_runner/app_runner/__init__.py diff --git a/app_runner/src/app_runner/app_runner/resolve_app.py b/bfabric_app_runner/src/bfabric_app_runner/app_runner/resolve_app.py similarity index 86% rename from app_runner/src/app_runner/app_runner/resolve_app.py rename to bfabric_app_runner/src/bfabric_app_runner/app_runner/resolve_app.py index 1e389a7e..41ba3ae9 100644 --- a/app_runner/src/app_runner/app_runner/resolve_app.py +++ b/bfabric_app_runner/src/bfabric_app_runner/app_runner/resolve_app.py @@ -5,8 +5,8 @@ import yaml from pydantic import ValidationError -from app_runner.specs.app.app_spec import AppSpec -from app_runner.specs.app.app_version import AppVersion +from bfabric_app_runner.specs.app.app_spec import AppSpec +from bfabric_app_runner.specs.app.app_version import AppVersion from bfabric.experimental.workunit_definition import WorkunitDefinition if TYPE_CHECKING: @@ -22,6 +22,14 @@ def resolve_app(versions: AppSpec, workunit_definition: WorkunitDefinition) -> A raise ValueError("The workunit definition does not contain an application version.") app_version = workunit_definition.execution.raw_parameters["application_version"] # TODO graceful handling of invalid versions + if app_version in versions and versions[app_version] is not None: + return versions[app_version] + else: + msg = ( + f"application_version '{app_version}' is not defined in the app spec,\n" + f" available versions: {sorted(versions.available_versions)}" + ) + raise ValueError(msg) return versions[app_version] diff --git a/app_runner/src/app_runner/app_runner/runner.py b/bfabric_app_runner/src/bfabric_app_runner/app_runner/runner.py similarity index 94% rename from app_runner/src/app_runner/app_runner/runner.py rename to bfabric_app_runner/src/bfabric_app_runner/app_runner/runner.py index f97554d6..e6c43cda 100644 --- a/app_runner/src/app_runner/app_runner/runner.py +++ b/bfabric_app_runner/src/bfabric_app_runner/app_runner/runner.py @@ -9,12 +9,12 @@ from loguru import logger from pydantic import BaseModel -from app_runner.input_preparation import prepare_folder -from app_runner.output_registration import register_outputs +from bfabric_app_runner.input_preparation import prepare_folder +from bfabric_app_runner.output_registration import register_outputs from bfabric.experimental.workunit_definition import WorkunitDefinition if TYPE_CHECKING: - from app_runner.specs.app.app_version import AppVersion + from bfabric_app_runner.specs.app.app_version import AppVersion from bfabric import Bfabric diff --git a/app_runner/src/app_runner/cli/__init__.py b/bfabric_app_runner/src/bfabric_app_runner/cli/__init__.py similarity index 100% rename from app_runner/src/app_runner/cli/__init__.py rename to bfabric_app_runner/src/bfabric_app_runner/cli/__init__.py diff --git a/app_runner/src/app_runner/cli/__main__.py b/bfabric_app_runner/src/bfabric_app_runner/cli/__main__.py similarity index 54% rename from app_runner/src/app_runner/cli/__main__.py rename to bfabric_app_runner/src/bfabric_app_runner/cli/__main__.py index b5fc0756..0545aa83 100644 --- a/app_runner/src/app_runner/cli/__main__.py +++ b/bfabric_app_runner/src/bfabric_app_runner/cli/__main__.py @@ -4,13 +4,13 @@ import cyclopts -from app_runner.cli.app import app_app -from app_runner.cli.chunk import app_chunk -from app_runner.cli.inputs import app_inputs -from app_runner.cli.outputs import app_outputs -from app_runner.cli.validate import app_validate +from bfabric_app_runner.cli.app import app_app +from bfabric_app_runner.cli.chunk import app_chunk +from bfabric_app_runner.cli.inputs import app_inputs +from bfabric_app_runner.cli.outputs import app_outputs +from bfabric_app_runner.cli.validate import app_validate -package_version = importlib.metadata.version("app_runner") +package_version = importlib.metadata.version("bfabric_app_runner") app = cyclopts.App( help="Provides an entrypoint to app execution.\n\nFunctionality/API under active development!", diff --git a/app_runner/src/app_runner/cli/app.py b/bfabric_app_runner/src/bfabric_app_runner/cli/app.py similarity index 92% rename from app_runner/src/app_runner/cli/app.py rename to bfabric_app_runner/src/bfabric_app_runner/cli/app.py index b131a97e..399fce09 100644 --- a/app_runner/src/app_runner/cli/app.py +++ b/bfabric_app_runner/src/bfabric_app_runner/cli/app.py @@ -4,8 +4,8 @@ import cyclopts -from app_runner.app_runner.resolve_app import load_workunit_information -from app_runner.app_runner.runner import run_app, Runner +from bfabric_app_runner.app_runner.resolve_app import load_workunit_information +from bfabric_app_runner.app_runner.runner import run_app, Runner from bfabric import Bfabric from bfabric.cli_formatting import setup_script_logging from bfabric.experimental.entity_lookup_cache import EntityLookupCache diff --git a/app_runner/src/app_runner/cli/chunk.py b/bfabric_app_runner/src/bfabric_app_runner/cli/chunk.py similarity index 94% rename from app_runner/src/app_runner/cli/chunk.py rename to bfabric_app_runner/src/bfabric_app_runner/cli/chunk.py index 49bd4c9f..26516f3d 100644 --- a/app_runner/src/app_runner/cli/chunk.py +++ b/bfabric_app_runner/src/bfabric_app_runner/cli/chunk.py @@ -4,9 +4,9 @@ import cyclopts -from app_runner.app_runner.resolve_app import load_workunit_information -from app_runner.app_runner.runner import run_app, Runner -from app_runner.output_registration import register_outputs +from bfabric_app_runner.app_runner.resolve_app import load_workunit_information +from bfabric_app_runner.app_runner.runner import run_app, Runner +from bfabric_app_runner.output_registration import register_outputs from bfabric import Bfabric from bfabric.cli_formatting import setup_script_logging from bfabric.experimental.entity_lookup_cache import EntityLookupCache diff --git a/app_runner/src/app_runner/cli/inputs.py b/bfabric_app_runner/src/bfabric_app_runner/cli/inputs.py similarity index 93% rename from app_runner/src/app_runner/cli/inputs.py rename to bfabric_app_runner/src/bfabric_app_runner/cli/inputs.py index 1dd79901..0253b036 100644 --- a/app_runner/src/app_runner/cli/inputs.py +++ b/bfabric_app_runner/src/bfabric_app_runner/cli/inputs.py @@ -4,14 +4,14 @@ import cyclopts -from app_runner.input_preparation import prepare_folder -from app_runner.input_preparation.integrity import IntegrityState -from app_runner.input_preparation.list_inputs import ( +from bfabric_app_runner.input_preparation import prepare_folder +from bfabric_app_runner.input_preparation.integrity import IntegrityState +from bfabric_app_runner.input_preparation.list_inputs import ( list_input_states, print_input_states, FileState, ) -from app_runner.specs.inputs_spec import InputsSpec +from bfabric_app_runner.specs.inputs_spec import InputsSpec from bfabric import Bfabric from bfabric.cli_formatting import setup_script_logging diff --git a/app_runner/src/app_runner/cli/outputs.py b/bfabric_app_runner/src/bfabric_app_runner/cli/outputs.py similarity index 93% rename from app_runner/src/app_runner/cli/outputs.py rename to bfabric_app_runner/src/bfabric_app_runner/cli/outputs.py index d81f13b6..d12bc411 100644 --- a/app_runner/src/app_runner/cli/outputs.py +++ b/bfabric_app_runner/src/bfabric_app_runner/cli/outputs.py @@ -5,8 +5,8 @@ import cyclopts from rich.pretty import pprint -from app_runner.output_registration.register import register_all -from app_runner.specs.outputs_spec import OutputsSpec, CopyResourceSpec, UpdateExisting +from bfabric_app_runner.output_registration.register import register_all +from bfabric_app_runner.specs.outputs_spec import OutputsSpec, CopyResourceSpec, UpdateExisting from bfabric import Bfabric from bfabric.cli_formatting import setup_script_logging from bfabric.experimental.workunit_definition import WorkunitDefinition diff --git a/app_runner/src/app_runner/cli/validate.py b/bfabric_app_runner/src/bfabric_app_runner/cli/validate.py similarity index 85% rename from app_runner/src/app_runner/cli/validate.py rename to bfabric_app_runner/src/bfabric_app_runner/cli/validate.py index 3f855fda..76406a5d 100644 --- a/app_runner/src/app_runner/cli/validate.py +++ b/bfabric_app_runner/src/bfabric_app_runner/cli/validate.py @@ -6,9 +6,9 @@ import yaml from rich.pretty import pprint -from app_runner.specs.app.app_spec import AppSpecTemplate, AppSpec -from app_runner.specs.inputs_spec import InputsSpec -from app_runner.specs.outputs_spec import OutputsSpec +from bfabric_app_runner.specs.app.app_spec import AppSpecTemplate, AppSpec +from bfabric_app_runner.specs.inputs_spec import InputsSpec +from bfabric_app_runner.specs.outputs_spec import OutputsSpec app_validate = cyclopts.App("validate", help="Validate yaml files.") diff --git a/app_runner/src/app_runner/dispatch/__init__.py b/bfabric_app_runner/src/bfabric_app_runner/dispatch/__init__.py similarity index 100% rename from app_runner/src/app_runner/dispatch/__init__.py rename to bfabric_app_runner/src/bfabric_app_runner/dispatch/__init__.py diff --git a/app_runner/src/app_runner/dispatch/dispatch_individual_resources.py b/bfabric_app_runner/src/bfabric_app_runner/dispatch/dispatch_individual_resources.py similarity index 96% rename from app_runner/src/app_runner/dispatch/dispatch_individual_resources.py rename to bfabric_app_runner/src/bfabric_app_runner/dispatch/dispatch_individual_resources.py index 4e29f50c..ba219817 100644 --- a/app_runner/src/app_runner/dispatch/dispatch_individual_resources.py +++ b/bfabric_app_runner/src/bfabric_app_runner/dispatch/dispatch_individual_resources.py @@ -4,8 +4,8 @@ from pydantic import BaseModel, ConfigDict, model_validator -from app_runner.dispatch.generic import write_workunit_definition_file, write_chunks_file -from app_runner.dispatch.resource_flow import get_resource_flow_input_resources +from bfabric_app_runner.dispatch.generic import write_workunit_definition_file, write_chunks_file +from bfabric_app_runner.dispatch.resource_flow import get_resource_flow_input_resources from bfabric.entities import Resource, Dataset if TYPE_CHECKING: diff --git a/app_runner/src/app_runner/dispatch/dispatch_single_dataset_flow.py b/bfabric_app_runner/src/bfabric_app_runner/dispatch/dispatch_single_dataset_flow.py similarity index 90% rename from app_runner/src/app_runner/dispatch/dispatch_single_dataset_flow.py rename to bfabric_app_runner/src/bfabric_app_runner/dispatch/dispatch_single_dataset_flow.py index f1bd481a..016eb3ea 100644 --- a/app_runner/src/app_runner/dispatch/dispatch_single_dataset_flow.py +++ b/bfabric_app_runner/src/bfabric_app_runner/dispatch/dispatch_single_dataset_flow.py @@ -2,7 +2,7 @@ from loguru import logger -from app_runner.dispatch.generic import write_workunit_definition_file, write_chunks_file +from bfabric_app_runner.dispatch.generic import write_workunit_definition_file, write_chunks_file from bfabric import Bfabric from bfabric.entities import Dataset from bfabric.experimental.workunit_definition import WorkunitDefinition diff --git a/app_runner/src/app_runner/dispatch/dispatch_single_resource_flow.py b/bfabric_app_runner/src/bfabric_app_runner/dispatch/dispatch_single_resource_flow.py similarity index 91% rename from app_runner/src/app_runner/dispatch/dispatch_single_resource_flow.py rename to bfabric_app_runner/src/bfabric_app_runner/dispatch/dispatch_single_resource_flow.py index 308a2ff6..408856c8 100644 --- a/app_runner/src/app_runner/dispatch/dispatch_single_resource_flow.py +++ b/bfabric_app_runner/src/bfabric_app_runner/dispatch/dispatch_single_resource_flow.py @@ -5,11 +5,11 @@ from pathlib import Path from pydantic import BaseModel, ConfigDict -from app_runner.dispatch.generic import ( +from bfabric_app_runner.dispatch.generic import ( write_chunks_file, write_workunit_definition_file, ) -from app_runner.dispatch.resource_flow import get_resource_flow_input_resources +from bfabric_app_runner.dispatch.resource_flow import get_resource_flow_input_resources class ConfigDispatchSingleResourceFlow(BaseModel): diff --git a/app_runner/src/app_runner/dispatch/generic.py b/bfabric_app_runner/src/bfabric_app_runner/dispatch/generic.py similarity index 100% rename from app_runner/src/app_runner/dispatch/generic.py rename to bfabric_app_runner/src/bfabric_app_runner/dispatch/generic.py diff --git a/app_runner/src/app_runner/dispatch/resource_flow.py b/bfabric_app_runner/src/bfabric_app_runner/dispatch/resource_flow.py similarity index 100% rename from app_runner/src/app_runner/dispatch/resource_flow.py rename to bfabric_app_runner/src/bfabric_app_runner/dispatch/resource_flow.py diff --git a/app_runner/src/app_runner/input_preparation/__init__.py b/bfabric_app_runner/src/bfabric_app_runner/input_preparation/__init__.py similarity index 100% rename from app_runner/src/app_runner/input_preparation/__init__.py rename to bfabric_app_runner/src/bfabric_app_runner/input_preparation/__init__.py diff --git a/app_runner/src/app_runner/input_preparation/collect_annotation.py b/bfabric_app_runner/src/bfabric_app_runner/input_preparation/collect_annotation.py similarity index 91% rename from app_runner/src/app_runner/input_preparation/collect_annotation.py rename to bfabric_app_runner/src/bfabric_app_runner/input_preparation/collect_annotation.py index d4df3b28..1d19d1df 100644 --- a/app_runner/src/app_runner/input_preparation/collect_annotation.py +++ b/bfabric_app_runner/src/bfabric_app_runner/input_preparation/collect_annotation.py @@ -5,7 +5,10 @@ from bfabric.entities import Resource from bfabric.utils.polars_utils import flatten_relations -from app_runner.specs.inputs.bfabric_annotation_spec import BfabricAnnotationResourceSampleSpec, BfabricAnnotationSpec +from bfabric_app_runner.specs.inputs.bfabric_annotation_spec import ( + BfabricAnnotationResourceSampleSpec, + BfabricAnnotationSpec, +) def collect_resource_sample_annotation(spec: BfabricAnnotationResourceSampleSpec, client: Bfabric, path: Path) -> None: diff --git a/app_runner/src/app_runner/input_preparation/integrity.py b/bfabric_app_runner/src/bfabric_app_runner/input_preparation/integrity.py similarity index 71% rename from app_runner/src/app_runner/input_preparation/integrity.py rename to bfabric_app_runner/src/bfabric_app_runner/input_preparation/integrity.py index 533de22a..c9883569 100644 --- a/app_runner/src/app_runner/input_preparation/integrity.py +++ b/bfabric_app_runner/src/bfabric_app_runner/input_preparation/integrity.py @@ -2,18 +2,19 @@ from enum import Enum -from app_runner.specs.inputs.bfabric_order_fasta_spec import BfabricOrderFastaSpec -from app_runner.specs.inputs.file_scp_spec import FileScpSpec +from bfabric_app_runner.specs.inputs.bfabric_order_fasta_spec import BfabricOrderFastaSpec +from bfabric_app_runner.specs.inputs.file_copy_spec import FileSpec +from bfabric_app_runner.specs.inputs.file_scp_spec import FileScpSpec from bfabric.entities import Resource, Dataset -from app_runner.specs.inputs.bfabric_dataset_spec import BfabricDatasetSpec # noqa: TC001 -from app_runner.specs.inputs.bfabric_resource_spec import BfabricResourceSpec # noqa: TC001 -from app_runner.util.checksums import md5sum +from bfabric_app_runner.specs.inputs.bfabric_dataset_spec import BfabricDatasetSpec # noqa: TC001 +from bfabric_app_runner.specs.inputs.bfabric_resource_spec import BfabricResourceSpec # noqa: TC001 +from bfabric_app_runner.util.checksums import md5sum from typing import TYPE_CHECKING if TYPE_CHECKING: from pathlib import Path from bfabric.bfabric import Bfabric - from app_runner.specs.inputs_spec import InputSpecType + from bfabric_app_runner.specs.inputs_spec import InputSpecType class IntegrityState(Enum): @@ -39,7 +40,11 @@ def check_integrity(spec: InputSpecType, local_path: Path, client: Bfabric) -> I return _check_resource_spec(spec, local_path, client) elif isinstance(spec, BfabricDatasetSpec): return _check_dataset_spec(spec, local_path, client) - elif isinstance(spec, FileScpSpec) or spec.type == "bfabric_annotation" or isinstance(spec, BfabricOrderFastaSpec): + elif ( + isinstance(spec, FileSpec | FileScpSpec) + or spec.type == "bfabric_annotation" + or isinstance(spec, BfabricOrderFastaSpec) + ): return IntegrityState.NotChecked else: raise ValueError(f"Unsupported spec type: {type(spec)}") diff --git a/app_runner/src/app_runner/input_preparation/list_inputs.py b/bfabric_app_runner/src/bfabric_app_runner/input_preparation/list_inputs.py similarity index 91% rename from app_runner/src/app_runner/input_preparation/list_inputs.py rename to bfabric_app_runner/src/bfabric_app_runner/input_preparation/list_inputs.py index 68cb81d1..f0193bf7 100644 --- a/app_runner/src/app_runner/input_preparation/list_inputs.py +++ b/bfabric_app_runner/src/bfabric_app_runner/input_preparation/list_inputs.py @@ -5,11 +5,11 @@ from rich.console import Console from rich.table import Table, Column -from app_runner.input_preparation.integrity import check_integrity, IntegrityState +from bfabric_app_runner.input_preparation.integrity import check_integrity, IntegrityState from typing import TYPE_CHECKING if TYPE_CHECKING: - from app_runner.specs.inputs_spec import InputSpecType + from bfabric_app_runner.specs.inputs_spec import InputSpecType from pathlib import Path from bfabric.bfabric import Bfabric diff --git a/app_runner/src/app_runner/input_preparation/prepare.py b/bfabric_app_runner/src/bfabric_app_runner/input_preparation/prepare.py similarity index 80% rename from app_runner/src/app_runner/input_preparation/prepare.py rename to bfabric_app_runner/src/bfabric_app_runner/input_preparation/prepare.py index 4e55caad..98c3a07b 100644 --- a/app_runner/src/app_runner/input_preparation/prepare.py +++ b/bfabric_app_runner/src/bfabric_app_runner/input_preparation/prepare.py @@ -4,19 +4,21 @@ from loguru import logger -from app_runner.input_preparation.collect_annotation import prepare_annotation -from app_runner.input_preparation.integrity import IntegrityState -from app_runner.input_preparation.list_inputs import list_input_states -from app_runner.specs.inputs.bfabric_dataset_spec import BfabricDatasetSpec -from app_runner.specs.inputs.bfabric_order_fasta_spec import BfabricOrderFastaSpec -from app_runner.specs.inputs.bfabric_resource_spec import BfabricResourceSpec -from app_runner.specs.inputs.file_scp_spec import FileScpSpec -from app_runner.specs.inputs_spec import ( +from bfabric_app_runner.input_preparation.collect_annotation import prepare_annotation +from bfabric_app_runner.input_preparation.integrity import IntegrityState +from bfabric_app_runner.input_preparation.list_inputs import list_input_states +from bfabric_app_runner.input_preparation.prepare_file_spec import prepare_file_spec +from bfabric_app_runner.specs.inputs.bfabric_dataset_spec import BfabricDatasetSpec +from bfabric_app_runner.specs.inputs.bfabric_order_fasta_spec import BfabricOrderFastaSpec +from bfabric_app_runner.specs.inputs.bfabric_resource_spec import BfabricResourceSpec +from bfabric_app_runner.specs.inputs.file_copy_spec import FileSpec +from bfabric_app_runner.specs.inputs.file_scp_spec import FileScpSpec +from bfabric_app_runner.specs.inputs_spec import ( InputSpecType, InputsSpec, ) -from app_runner.util.checksums import md5sum -from app_runner.util.scp import scp +from bfabric_app_runner.util.checksums import md5sum +from bfabric_app_runner.util.scp import scp from bfabric.entities import Resource, Dataset, Workunit, Order if TYPE_CHECKING: @@ -40,6 +42,8 @@ def prepare_all(self, specs: list[InputSpecType]) -> None: logger.debug(f"Skipping {spec} as it already exists and passed integrity check") elif isinstance(spec, BfabricResourceSpec): self.prepare_resource(spec) + elif isinstance(spec, FileSpec): + self.prepare_file_spec(spec) elif isinstance(spec, FileScpSpec): self.prepare_file_scp(spec) elif isinstance(spec, BfabricDatasetSpec): @@ -87,6 +91,9 @@ def prepare_resource(self, spec: BfabricResourceSpec) -> None: if actual_checksum != resource["filechecksum"]: raise ValueError(f"Checksum mismatch: expected {resource['filechecksum']}, got {actual_checksum}") + def prepare_file_spec(self, spec: FileSpec) -> None: + return prepare_file_spec(spec=spec, client=self._client, working_dir=self._working_dir, ssh_user=self._ssh_user) + def prepare_file_scp(self, spec: FileScpSpec) -> None: scp_uri = f"{spec.host}:{spec.absolute_path}" result_name = spec.resolve_filename(client=self._client) @@ -102,12 +109,22 @@ def prepare_dataset(self, spec: BfabricDatasetSpec) -> None: dataset.write_csv(path=target_path, separator=spec.separator) def prepare_order_fasta(self, spec: BfabricOrderFastaSpec) -> None: + # Determine the result file. + result_name = self._working_dir / spec.filename + result_name.parent.mkdir(exist_ok=True, parents=True) + # Find the order. match spec.entity: case "workunit": workunit = Workunit.find(id=spec.id, client=self._client) if not isinstance(workunit.container, Order): - raise ValueError(f"Workunit {workunit.id} is not associated with an order") + msg = f"Workunit {workunit.id} is not associated with an order" + if spec.required: + raise ValueError(msg) + else: + logger.warning(msg) + result_name.write_text("") + return order = workunit.container case "order": order = Order.find(id=spec.id, client=self._client) @@ -115,8 +132,6 @@ def prepare_order_fasta(self, spec: BfabricOrderFastaSpec) -> None: assert_never(spec.entity) # Write the result into the file - result_name = self._working_dir / spec.filename - result_name.parent.mkdir(exist_ok=True, parents=True) fasta_content = order.data_dict.get("fastasequence", "") if fasta_content and fasta_content[-1] != "\n": fasta_content += "\n" diff --git a/bfabric_app_runner/src/bfabric_app_runner/input_preparation/prepare_file_spec.py b/bfabric_app_runner/src/bfabric_app_runner/input_preparation/prepare_file_spec.py new file mode 100644 index 00000000..27dd7584 --- /dev/null +++ b/bfabric_app_runner/src/bfabric_app_runner/input_preparation/prepare_file_spec.py @@ -0,0 +1,97 @@ +import shlex +import shutil +import subprocess +from pathlib import Path +from shutil import SameFileError +from subprocess import CalledProcessError +from typing import assert_never + +from loguru import logger + +from bfabric import Bfabric +from bfabric_app_runner.specs.inputs.file_copy_spec import ( + FileSpec, + FileSourceSsh, + FileSourceLocal, + FileSourceSshValue, +) +from bfabric_app_runner.util.scp import scp + + +def prepare_file_spec(spec: FileSpec, client: Bfabric, working_dir: Path, ssh_user: str | None) -> None: + """Prepares the file specified by the spec.""" + output_path = working_dir / spec.resolve_filename(client=client) + output_path.parent.mkdir(exist_ok=True, parents=True) + + if not spec.link: + success = _operation_copy_rsync(spec, output_path, ssh_user) + if not success: + success = _operation_copy(spec, output_path, ssh_user) + else: + success = _operation_link_symbolic(spec, output_path) + if not success: + raise RuntimeError(f"Failed to copy file: {spec}") + + +def _operation_copy_rsync(spec: FileSpec, output_path: Path, ssh_user: str | None) -> bool: + match spec.source: + case FileSourceLocal(local=local): + source_str = str(Path(local).resolve()) + case FileSourceSsh(ssh=FileSourceSshValue(host=host, path=path)): + source_str = f"{ssh_user}@{host}:{path}" if ssh_user else f"{host}:{path}" + case _: + assert_never(spec.source) + cmd = ["rsync", "-Pav", source_str, str(output_path)] + logger.info(shlex.join(cmd)) + result = subprocess.run(cmd, check=False) + return result.returncode == 0 + + +def _operation_copy(spec: FileSpec, output_path: Path, ssh_user: str | None) -> bool: + match spec.source: + case FileSourceLocal(): + return _operation_copy_cp(spec, output_path) + case FileSourceSsh(): + return _operation_copy_scp(spec, output_path, ssh_user) + case _: + assert_never(spec.source) + + +def _operation_copy_scp(spec: FileSpec, output_path: Path, ssh_user: str | None) -> bool: + try: + source_str = f"{spec.source.ssh.host}:{spec.source.ssh.path}" + scp(source=source_str, target=output_path, user=ssh_user) + except CalledProcessError: + return False + return True + + +def _operation_copy_cp(spec: FileSpec, output_path: Path) -> bool: + cmd = [str(Path(spec.source.local).resolve()), str(output_path)] + logger.info(shlex.join(["cp", *cmd])) + try: + shutil.copyfile(*cmd) + except (OSError, SameFileError): + return False + return True + + +def _operation_link_symbolic(spec: FileSpec, output_path: Path) -> bool: + # the link is created relative to the output file, so it should be more portable across apptainer images etc + source_path = Path(spec.source.local).resolve().relative_to(output_path.resolve().parent, walk_up=True) + + # if the file exists, and only if it is a link as well + if output_path.is_symlink(): + # check if it points to the same file, in which case we don't need to do anything + if output_path.resolve() == source_path.resolve(): + logger.info("Link already exists and points to the correct file") + return True + else: + logger.info(f"rm {output_path}") + output_path.unlink() + elif output_path.exists(): + raise RuntimeError(f"Output path already exists and is not a symlink: {output_path}") + cmd = ["ln", "-s", str(source_path), str(output_path)] + logger.info(shlex.join(cmd)) + result = subprocess.run(cmd, check=False) + return result.returncode == 0 diff --git a/app_runner/src/app_runner/output_registration/__init__.py b/bfabric_app_runner/src/bfabric_app_runner/output_registration/__init__.py similarity index 100% rename from app_runner/src/app_runner/output_registration/__init__.py rename to bfabric_app_runner/src/bfabric_app_runner/output_registration/__init__.py diff --git a/app_runner/src/app_runner/output_registration/register.py b/bfabric_app_runner/src/bfabric_app_runner/output_registration/register.py similarity index 80% rename from app_runner/src/app_runner/output_registration/register.py rename to bfabric_app_runner/src/bfabric_app_runner/output_registration/register.py index 978b7e08..8e78c390 100644 --- a/app_runner/src/app_runner/output_registration/register.py +++ b/bfabric_app_runner/src/bfabric_app_runner/output_registration/register.py @@ -4,15 +4,15 @@ from loguru import logger -from app_runner.specs.outputs_spec import ( +from bfabric_app_runner.specs.outputs_spec import ( CopyResourceSpec, UpdateExisting, OutputsSpec, SpecType, SaveDatasetSpec, ) -from app_runner.util.checksums import md5sum -from app_runner.util.scp import scp +from bfabric_app_runner.util.checksums import md5sum +from bfabric_app_runner.util.scp import scp from bfabric.entities import Resource from bfabric.entities import Storage, Workunit from bfabric.experimental.upload_dataset import bfabric_save_csv2dataset @@ -23,9 +23,7 @@ from bfabric.experimental.workunit_definition import WorkunitDefinition -def _get_output_folder( - spec: CopyResourceSpec, workunit_definition: WorkunitDefinition -) -> Path: +def _get_output_folder(spec: CopyResourceSpec, workunit_definition: WorkunitDefinition) -> Path: if not spec.store_folder_path: return workunit_definition.registration.storage_output_folder else: @@ -40,14 +38,8 @@ def register_file_in_workunit( ) -> None: """Registers a file in the workunit.""" existing_id = _identify_existing_resource_id(client, spec, workunit_definition) - if ( - resource_id is not None - and existing_id is not None - and resource_id != existing_id - ): - raise ValueError( - f"Resource id {resource_id} does not match existing resource id {existing_id}" - ) + if resource_id is not None and existing_id is not None and resource_id != existing_id: + raise ValueError(f"Resource id {resource_id} does not match existing resource id {existing_id}") checksum = md5sum(spec.local_path) output_folder = _get_output_folder(spec, workunit_definition=workunit_definition) @@ -85,9 +77,7 @@ def _identify_existing_resource_id( if resources: return list(resources)[0].id elif spec.update_existing == UpdateExisting.REQUIRED: - raise ValueError( - f"Resource {spec.store_entry_path.name} not found in workunit {workunit_definition.id}" - ) + raise ValueError(f"Resource {spec.store_entry_path.name} not found in workunit {workunit_definition.id}") return None @@ -104,9 +94,7 @@ def copy_file_to_storage( scp(spec.local_path, output_uri, user=ssh_user) -def _save_dataset( - spec: SaveDatasetSpec, client: Bfabric, workunit_definition: WorkunitDefinition -) -> None: +def _save_dataset(spec: SaveDatasetSpec, client: Bfabric, workunit_definition: WorkunitDefinition) -> None: """Saves a dataset to the bfabric.""" # TODO should not print to stdout in the future # TODO also it should not be imported from bfabric_scripts, but rather the generic functionality should be available @@ -123,17 +111,11 @@ def _save_dataset( ) -def find_default_resource_id( - workunit_definition: WorkunitDefinition, client: Bfabric -) -> int | None: +def find_default_resource_id(workunit_definition: WorkunitDefinition, client: Bfabric) -> int | None: """Finds the default resource's id for the workunit. Maybe in the future, this will be always `None`.""" - workunit = Workunit.find( - id=workunit_definition.registration.workunit_id, client=client - ) + workunit = Workunit.find(id=workunit_definition.registration.workunit_id, client=client) candidate_resources = [ - resource - for resource in workunit.resources - if resource["name"] not in ["slurm_stdout", "slurm_stderr"] + resource for resource in workunit.resources if resource["name"] not in ["slurm_stdout", "slurm_stderr"] ] # We also check that the resource is pending, as else we might re-use a resource that was created by the app... if len(candidate_resources) == 1 and candidate_resources[0]["status"] == "pending": @@ -153,9 +135,7 @@ def register_all( for spec in specs_list: logger.debug(f"Registering {spec}") if isinstance(spec, CopyResourceSpec): - storage = Storage.find( - workunit_definition.registration.storage_id, client=client - ) + storage = Storage.find(workunit_definition.registration.storage_id, client=client) copy_file_to_storage( spec, workunit_definition=workunit_definition, @@ -163,9 +143,7 @@ def register_all( ssh_user=ssh_user, ) if not default_resource_was_reused: - resource_id = find_default_resource_id( - workunit_definition=workunit_definition, client=client - ) + resource_id = find_default_resource_id(workunit_definition=workunit_definition, client=client) default_resource_was_reused = True else: resource_id = None diff --git a/app_runner/src/app_runner/py.typed b/bfabric_app_runner/src/bfabric_app_runner/py.typed similarity index 100% rename from app_runner/src/app_runner/py.typed rename to bfabric_app_runner/src/bfabric_app_runner/py.typed diff --git a/app_runner/src/app_runner/specs/__init__.py b/bfabric_app_runner/src/bfabric_app_runner/specs/__init__.py similarity index 100% rename from app_runner/src/app_runner/specs/__init__.py rename to bfabric_app_runner/src/bfabric_app_runner/specs/__init__.py diff --git a/app_runner/src/app_runner/specs/app/__init__.py b/bfabric_app_runner/src/bfabric_app_runner/specs/app/__init__.py similarity index 100% rename from app_runner/src/app_runner/specs/app/__init__.py rename to bfabric_app_runner/src/bfabric_app_runner/specs/app/__init__.py diff --git a/app_runner/src/app_runner/specs/app/app_spec.py b/bfabric_app_runner/src/bfabric_app_runner/specs/app/app_spec.py similarity index 90% rename from app_runner/src/app_runner/specs/app/app_spec.py rename to bfabric_app_runner/src/bfabric_app_runner/specs/app/app_spec.py index b7573db6..0644bb6b 100644 --- a/app_runner/src/app_runner/specs/app/app_spec.py +++ b/bfabric_app_runner/src/bfabric_app_runner/specs/app/app_spec.py @@ -5,7 +5,7 @@ import yaml from pydantic import BaseModel -from app_runner.specs.app.app_version import AppVersion, AppVersionMultiTemplate # noqa: TCH001 +from bfabric_app_runner.specs.app.app_version import AppVersion, AppVersionMultiTemplate # noqa: TCH001 if TYPE_CHECKING: from pathlib import Path @@ -46,6 +46,9 @@ def available_versions(self) -> set[str]: """The available versions of the app.""" return {version.version for version in self.versions} + def __contains__(self, version: str) -> bool: + return version in self.available_versions + def __getitem__(self, version: str) -> AppVersion | None: """Returns the app version with the provided version number or None if it does not exist.""" for app_version in self.versions: diff --git a/app_runner/src/app_runner/specs/app/app_version.py b/bfabric_app_runner/src/bfabric_app_runner/specs/app/app_version.py similarity index 87% rename from app_runner/src/app_runner/specs/app/app_version.py rename to bfabric_app_runner/src/bfabric_app_runner/specs/app/app_version.py index 2535faf8..373185f9 100644 --- a/app_runner/src/app_runner/specs/app/app_version.py +++ b/bfabric_app_runner/src/bfabric_app_runner/specs/app/app_version.py @@ -4,10 +4,10 @@ from pydantic import BaseModel, field_validator -from app_runner.specs import config_interpolation -from app_runner.specs.app.commands_spec import CommandsSpec # noqa: TCH001 -from app_runner.specs.config_interpolation import interpolate_config_strings -from app_runner.specs.submitter_ref import SubmitterRef # noqa: TCH001 +from bfabric_app_runner.specs import config_interpolation +from bfabric_app_runner.specs.app.commands_spec import CommandsSpec # noqa: TCH001 +from bfabric_app_runner.specs.config_interpolation import interpolate_config_strings +from bfabric_app_runner.specs.submitter_ref import SubmitterRef # noqa: TCH001 class AppVersion(BaseModel): diff --git a/app_runner/src/app_runner/specs/app/commands_spec.py b/bfabric_app_runner/src/bfabric_app_runner/specs/app/commands_spec.py similarity index 100% rename from app_runner/src/app_runner/specs/app/commands_spec.py rename to bfabric_app_runner/src/bfabric_app_runner/specs/app/commands_spec.py diff --git a/app_runner/src/app_runner/specs/common_types.py b/bfabric_app_runner/src/bfabric_app_runner/specs/common_types.py similarity index 66% rename from app_runner/src/app_runner/specs/common_types.py rename to bfabric_app_runner/src/bfabric_app_runner/specs/common_types.py index cb9d7a82..59e7c7e3 100644 --- a/app_runner/src/app_runner/specs/common_types.py +++ b/bfabric_app_runner/src/bfabric_app_runner/specs/common_types.py @@ -4,5 +4,8 @@ from pydantic import Field +AbsoluteFilePath = Annotated[str, Field(pattern=r"^/[^:]*$")] +"""Absolute file path, excluding ":" characters.""" + RelativeFilePath = Annotated[str, Field(pattern=r"^[^/][^:]*$")] """Relative file path, excluding absolute paths and ":" characters.""" diff --git a/app_runner/src/app_runner/specs/config_interpolation.py b/bfabric_app_runner/src/bfabric_app_runner/specs/config_interpolation.py similarity index 100% rename from app_runner/src/app_runner/specs/config_interpolation.py rename to bfabric_app_runner/src/bfabric_app_runner/specs/config_interpolation.py diff --git a/app_runner/src/app_runner/specs/inputs/__init__.py b/bfabric_app_runner/src/bfabric_app_runner/specs/inputs/__init__.py similarity index 100% rename from app_runner/src/app_runner/specs/inputs/__init__.py rename to bfabric_app_runner/src/bfabric_app_runner/specs/inputs/__init__.py diff --git a/app_runner/src/app_runner/specs/inputs/bfabric_annotation_spec.py b/bfabric_app_runner/src/bfabric_app_runner/specs/inputs/bfabric_annotation_spec.py similarity index 89% rename from app_runner/src/app_runner/specs/inputs/bfabric_annotation_spec.py rename to bfabric_app_runner/src/bfabric_app_runner/specs/inputs/bfabric_annotation_spec.py index 4e36a975..bba9ae08 100644 --- a/app_runner/src/app_runner/specs/inputs/bfabric_annotation_spec.py +++ b/bfabric_app_runner/src/bfabric_app_runner/specs/inputs/bfabric_annotation_spec.py @@ -4,7 +4,7 @@ from pydantic import BaseModel, Field -from app_runner.specs.common_types import RelativeFilePath # noqa: TC001 +from bfabric_app_runner.specs.common_types import RelativeFilePath # noqa: TC001 if TYPE_CHECKING: from bfabric import Bfabric diff --git a/app_runner/src/app_runner/specs/inputs/bfabric_dataset_spec.py b/bfabric_app_runner/src/bfabric_app_runner/specs/inputs/bfabric_dataset_spec.py similarity index 88% rename from app_runner/src/app_runner/specs/inputs/bfabric_dataset_spec.py rename to bfabric_app_runner/src/bfabric_app_runner/specs/inputs/bfabric_dataset_spec.py index 48b91440..4d0dbeb3 100644 --- a/app_runner/src/app_runner/specs/inputs/bfabric_dataset_spec.py +++ b/bfabric_app_runner/src/bfabric_app_runner/specs/inputs/bfabric_dataset_spec.py @@ -4,7 +4,7 @@ from pydantic import BaseModel, ConfigDict -from app_runner.specs.common_types import RelativeFilePath # noqa: TC001 +from bfabric_app_runner.specs.common_types import RelativeFilePath # noqa: TC001 if TYPE_CHECKING: from bfabric import Bfabric diff --git a/app_runner/src/app_runner/specs/inputs/bfabric_order_fasta_spec.py b/bfabric_app_runner/src/bfabric_app_runner/specs/inputs/bfabric_order_fasta_spec.py similarity index 82% rename from app_runner/src/app_runner/specs/inputs/bfabric_order_fasta_spec.py rename to bfabric_app_runner/src/bfabric_app_runner/specs/inputs/bfabric_order_fasta_spec.py index 4e741cc4..82ea5762 100644 --- a/app_runner/src/app_runner/specs/inputs/bfabric_order_fasta_spec.py +++ b/bfabric_app_runner/src/bfabric_app_runner/specs/inputs/bfabric_order_fasta_spec.py @@ -3,7 +3,7 @@ from pydantic import BaseModel, ConfigDict -from app_runner.specs.common_types import RelativeFilePath # noqa: TC001 +from bfabric_app_runner.specs.common_types import RelativeFilePath # noqa: TC001 if TYPE_CHECKING: from bfabric import Bfabric @@ -16,6 +16,7 @@ class BfabricOrderFastaSpec(BaseModel): id: int entity: Literal["workunit", "order"] filename: RelativeFilePath + required: bool = False def resolve_filename(self, client: Bfabric) -> str: return self.filename diff --git a/app_runner/src/app_runner/specs/inputs/bfabric_resource_spec.py b/bfabric_app_runner/src/bfabric_app_runner/specs/inputs/bfabric_resource_spec.py similarity index 90% rename from app_runner/src/app_runner/specs/inputs/bfabric_resource_spec.py rename to bfabric_app_runner/src/bfabric_app_runner/specs/inputs/bfabric_resource_spec.py index 05257121..5910326c 100644 --- a/app_runner/src/app_runner/specs/inputs/bfabric_resource_spec.py +++ b/bfabric_app_runner/src/bfabric_app_runner/specs/inputs/bfabric_resource_spec.py @@ -4,7 +4,7 @@ from pydantic import BaseModel, ConfigDict -from app_runner.specs.common_types import RelativeFilePath # noqa: TC001 +from bfabric_app_runner.specs.common_types import RelativeFilePath # noqa: TC001 from bfabric.entities import Resource if TYPE_CHECKING: diff --git a/bfabric_app_runner/src/bfabric_app_runner/specs/inputs/file_copy_spec.py b/bfabric_app_runner/src/bfabric_app_runner/specs/inputs/file_copy_spec.py new file mode 100644 index 00000000..50da2caa --- /dev/null +++ b/bfabric_app_runner/src/bfabric_app_runner/specs/inputs/file_copy_spec.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +from typing import Literal, TYPE_CHECKING, Self + +from pydantic import BaseModel, model_validator + +from bfabric_app_runner.specs.common_types import RelativeFilePath, AbsoluteFilePath # noqa: TC001 + +if TYPE_CHECKING: + from bfabric import Bfabric + + +class FileSourceLocal(BaseModel): + local: AbsoluteFilePath + + def get_filename(self) -> str: + return self.local.split("/")[-1] + + +class FileSourceSshValue(BaseModel): + host: str + path: AbsoluteFilePath + + +class FileSourceSsh(BaseModel): + ssh: FileSourceSshValue + + def get_filename(self) -> str: + return self.ssh.path.split("/")[-1] + + +class FileSpec(BaseModel): + type: Literal["file"] = "file" + source: FileSourceSsh | FileSourceLocal + filename: RelativeFilePath | None = None + link: bool = False + + @model_validator(mode="after") + def validate_no_link_ssh(self) -> Self: + if isinstance(self.source, FileSourceSsh) and self.link: + raise ValueError("Cannot link to a remote file.") + return self + + def resolve_filename(self, client: Bfabric) -> str: + return self.filename if self.filename else self.source.get_filename() diff --git a/app_runner/src/app_runner/specs/inputs/file_scp_spec.py b/bfabric_app_runner/src/bfabric_app_runner/specs/inputs/file_scp_spec.py similarity index 82% rename from app_runner/src/app_runner/specs/inputs/file_scp_spec.py rename to bfabric_app_runner/src/bfabric_app_runner/specs/inputs/file_scp_spec.py index 212fa927..c77f7ed6 100644 --- a/app_runner/src/app_runner/specs/inputs/file_scp_spec.py +++ b/bfabric_app_runner/src/bfabric_app_runner/specs/inputs/file_scp_spec.py @@ -4,12 +4,13 @@ from pydantic import BaseModel, ConfigDict -from app_runner.specs.common_types import RelativeFilePath # noqa: TC001 +from bfabric_app_runner.specs.common_types import RelativeFilePath # noqa: TC001 if TYPE_CHECKING: from bfabric import Bfabric +# TODO(leo): deprecate later class FileScpSpec(BaseModel): model_config = ConfigDict(extra="forbid") type: Literal["file_scp"] = "file_scp" diff --git a/app_runner/src/app_runner/specs/inputs_spec.py b/bfabric_app_runner/src/bfabric_app_runner/specs/inputs_spec.py similarity index 66% rename from app_runner/src/app_runner/specs/inputs_spec.py rename to bfabric_app_runner/src/bfabric_app_runner/specs/inputs_spec.py index eca38e24..9b23fe4b 100644 --- a/app_runner/src/app_runner/specs/inputs_spec.py +++ b/bfabric_app_runner/src/bfabric_app_runner/specs/inputs_spec.py @@ -5,18 +5,19 @@ import yaml from pydantic import BaseModel, ConfigDict, Field -from app_runner.specs.inputs.bfabric_annotation_spec import BfabricAnnotationSpec -from app_runner.specs.inputs.bfabric_dataset_spec import BfabricDatasetSpec -from app_runner.specs.inputs.bfabric_order_fasta_spec import BfabricOrderFastaSpec -from app_runner.specs.inputs.bfabric_resource_spec import BfabricResourceSpec -from app_runner.specs.inputs.file_scp_spec import FileScpSpec +from bfabric_app_runner.specs.inputs.bfabric_annotation_spec import BfabricAnnotationSpec +from bfabric_app_runner.specs.inputs.bfabric_dataset_spec import BfabricDatasetSpec +from bfabric_app_runner.specs.inputs.bfabric_order_fasta_spec import BfabricOrderFastaSpec +from bfabric_app_runner.specs.inputs.bfabric_resource_spec import BfabricResourceSpec +from bfabric_app_runner.specs.inputs.file_copy_spec import FileSpec +from bfabric_app_runner.specs.inputs.file_scp_spec import FileScpSpec if TYPE_CHECKING: from pathlib import Path from bfabric import Bfabric InputSpecType = Annotated[ - BfabricResourceSpec | FileScpSpec | BfabricDatasetSpec | BfabricOrderFastaSpec | BfabricAnnotationSpec, + BfabricResourceSpec | FileSpec | FileScpSpec | BfabricDatasetSpec | BfabricOrderFastaSpec | BfabricAnnotationSpec, Field(discriminator="type"), ] diff --git a/app_runner/src/app_runner/specs/outputs_spec.py b/bfabric_app_runner/src/bfabric_app_runner/specs/outputs_spec.py similarity index 100% rename from app_runner/src/app_runner/specs/outputs_spec.py rename to bfabric_app_runner/src/bfabric_app_runner/specs/outputs_spec.py diff --git a/app_runner/src/app_runner/specs/submitter_ref.py b/bfabric_app_runner/src/bfabric_app_runner/specs/submitter_ref.py similarity index 100% rename from app_runner/src/app_runner/specs/submitter_ref.py rename to bfabric_app_runner/src/bfabric_app_runner/specs/submitter_ref.py diff --git a/app_runner/src/app_runner/util/__init__.py b/bfabric_app_runner/src/bfabric_app_runner/util/__init__.py similarity index 100% rename from app_runner/src/app_runner/util/__init__.py rename to bfabric_app_runner/src/bfabric_app_runner/util/__init__.py diff --git a/app_runner/src/app_runner/util/checksums.py b/bfabric_app_runner/src/bfabric_app_runner/util/checksums.py similarity index 100% rename from app_runner/src/app_runner/util/checksums.py rename to bfabric_app_runner/src/bfabric_app_runner/util/checksums.py diff --git a/app_runner/src/app_runner/util/scp.py b/bfabric_app_runner/src/bfabric_app_runner/util/scp.py similarity index 93% rename from app_runner/src/app_runner/util/scp.py rename to bfabric_app_runner/src/bfabric_app_runner/util/scp.py index 117d8a5d..547bba54 100644 --- a/app_runner/src/app_runner/util/scp.py +++ b/bfabric_app_runner/src/bfabric_app_runner/util/scp.py @@ -1,5 +1,6 @@ from __future__ import annotations +import shlex import subprocess from pathlib import Path @@ -38,8 +39,9 @@ def scp(source: str | Path, target: str | Path, *, user: str | None = None, mkdi parent_path = Path(target).parent parent_path.mkdir(parents=True, exist_ok=True) - logger.info(f"scp {source} {target}") - subprocess.run(["scp", source, target], check=True) + cmd = ["scp", source, target] + logger.info(shlex.join(cmd)) + subprocess.run(cmd, check=True) def _is_remote(path: str | Path) -> bool: diff --git a/bfabric_scripts/doc/changelog.md b/bfabric_scripts/doc/changelog.md new file mode 100644 index 00000000..495358b8 --- /dev/null +++ b/bfabric_scripts/doc/changelog.md @@ -0,0 +1,21 @@ +# Changelog + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). + +Versioning currently follows `X.Y.Z` where + +- `X` is used for major changes, that contain breaking changes +- `Y` should be the current bfabric release +- `Z` is increased for feature releases, that should not break the API + +## \[Unreleased\] + +### Added + +- `bfabric-cli workunit not-available`: + - allows sorting by arbitrary fields, e.g. application id + - allows filtering inclusive or exclusive by user + +## \[1.13.19\] - 2025-01-29 + +Initial release of standalone bfabric_scripts package. diff --git a/bfabric_scripts/pyproject.toml b/bfabric_scripts/pyproject.toml index e3ce2da9..d509c541 100644 --- a/bfabric_scripts/pyproject.toml +++ b/bfabric_scripts/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "hatchling.build" [project] name = "bfabric_scripts" description = "Python command line scripts for the B-Fabric API" -version = "1.13.18" +version = "1.13.19" dependencies = [ "bfabric" diff --git a/bfabric_scripts/src/bfabric_scripts/bfabric_feeder_mascot.py b/bfabric_scripts/src/bfabric_scripts/bfabric_feeder_mascot.py index bc00d664..7ef5ba7a 100755 --- a/bfabric_scripts/src/bfabric_scripts/bfabric_feeder_mascot.py +++ b/bfabric_scripts/src/bfabric_scripts/bfabric_feeder_mascot.py @@ -50,8 +50,7 @@ "Read {len} data items from {name} using {size:.1f} GBytes.".format( len=len(DB), name=DBfilename, - size=sum(map(lambda x: int(x["resource"]["size"]), DB.values())) - / (1024 * 1024 * 1024), + size=sum(map(lambda x: int(x["resource"]["size"]), DB.values())) / (1024 * 1024 * 1024), ) ) except OSError: @@ -67,9 +66,7 @@ def query_mascot_result(file_path: str) -> bool: print("\thit") wu = DB[file_path] if "workunitid" in wu: - print( - f"\tdat file {file_path} already registered as workunit id {wu['workunitid']}. continue ..." - ) + print(f"\tdat file {file_path} already registered as workunit id {wu['workunitid']}. continue ...") return else: print("\tno workunitid found") @@ -82,9 +79,7 @@ def query_mascot_result(file_path: str) -> bool: if len(wu["inputresource"]) > 0: if re.search("autoQC4L", wu["name"]) or re.search("autoQC01", wu["name"]): - print( - f"WARNING This script ignores autoQC based mascot dat file {file_path}." - ) + print(f"WARNING This script ignores autoQC based mascot dat file {file_path}.") return print("\tquerying bfabric ...") @@ -186,9 +181,7 @@ def parse_mascot_result_file(file_path: str) -> dict[str, Any]: "^(FILE|COM|release|USERNAME|USERID|TOL|TOLU|ITOL|ITOLU|MODS|IT_MODS|CHARGE|INSTRUMENT|QUANTITATION|DECOY)=(.+)$" ) - control_chars = "".join( - map(chr, itertools.chain(range(0x00, 0x20), range(0x7F, 0xA0))) - ) + control_chars = "".join(map(chr, itertools.chain(range(0x00, 0x20), range(0x7F, 0xA0)))) control_char_re = re.compile(f"[{re.escape(control_chars)}]") line_count = 0 @@ -209,16 +202,10 @@ def parse_mascot_result_file(file_path: str) -> dict[str, Any]: md5.update(line.encode()) # check if the first character of the line is a 't' for title to save regex time if line[0] == "t": - result = regex0.match( - urllib.parse.unquote(line.strip()) - .replace("\\", "/") - .replace("//", "/") - ) + result = regex0.match(urllib.parse.unquote(line.strip()).replace("\\", "/").replace("//", "/")) if result and result.group(1) not in inputresourceHitHash: inputresourceHitHash[result.group(1)] = result.group(2) - inputresourceList.append( - dict(storageid=2, relativepath=result.group(1)) - ) + inputresourceList.append(dict(storageid=2, relativepath=result.group(1))) project = result.group(2) else: # nothing as do be done since the input_resource is already recorded @@ -232,11 +219,7 @@ def parse_mascot_result_file(file_path: str) -> dict[str, Any]: meta_data_dict[result.group(1)] = result.group(2) desc = desc.encode("ascii", errors="ignore") - name = ( - f"{meta_data_dict['COM']}; {os.path.basename(meta_data_dict['relativepath'])}"[ - :255 - ] - ) + name = f"{meta_data_dict['COM']}; {os.path.basename(meta_data_dict['relativepath'])}"[:255] rv = dict( applicationid=19, containerid=project, @@ -276,8 +259,7 @@ def print_statistics() -> None: print_project_frequency(map(lambda x: x["containerid"], DB.values())) print( "file size\t=\t{} GBytes".format( - sum(map(lambda x: int(x["resource"]["size"]), DB.values())) - / (1024 * 1024 * 1024) + sum(map(lambda x: int(x["resource"]["size"]), DB.values())) / (1024 * 1024 * 1024) ) ) @@ -286,9 +268,7 @@ def main() -> None: """Parses the CLI arguments and calls the appropriate functions.""" parser = argparse.ArgumentParser() group = parser.add_mutually_exclusive_group(required=True) - group.add_argument( - "--stdin", action="store_true", help="read file names from stdin" - ) + group.add_argument("--stdin", action="store_true", help="read file names from stdin") group.add_argument("--file", type=str, help="processes the provided file") parser.add_argument("--statistics", action="store_true", help="print statistics") diff --git a/bfabric_scripts/src/bfabric_scripts/bfabric_feeder_resource_autoQC.py b/bfabric_scripts/src/bfabric_scripts/bfabric_feeder_resource_autoQC.py index 77522a3b..3945bc39 100755 --- a/bfabric_scripts/src/bfabric_scripts/bfabric_feeder_resource_autoQC.py +++ b/bfabric_scripts/src/bfabric_scripts/bfabric_feeder_resource_autoQC.py @@ -46,9 +46,7 @@ def sample_check(self, projectid: int, name: str): :return: SID """ try: - res = self.client.read( - endpoint="sample", obj={"containerid": projectid, "name": name} - ).to_list_dict() + res = self.client.read(endpoint="sample", obj={"containerid": projectid, "name": name}).to_list_dict() except Exception: print(res) raise @@ -88,17 +86,11 @@ def sample_check(self, projectid: int, name: str): if not res: if name == "autoQC4L": - res = self.client.save( - endpoint="sample", obj=query_autoQC4L - ).to_list_dict() + res = self.client.save(endpoint="sample", obj=query_autoQC4L).to_list_dict() elif name == "autoQC01": - res = self.client.save( - endpoint="sample", obj=query_autoQC01 - ).to_list_dict() + res = self.client.save(endpoint="sample", obj=query_autoQC01).to_list_dict() elif name == "lipidQC01": - res = self.client.save( - endpoint="sample", obj=query_lipidQC01 - ).to_list_dict() + res = self.client.save(endpoint="sample", obj=query_lipidQC01).to_list_dict() print(res) print(res[0]) @@ -241,9 +233,7 @@ def feed(self, line) -> None: sampleid = self.sample_check(projectid, name=autoQCType) sys.exit(0) # print sampleid - workunitid = self.workunit_check( - projectid, name=autoQCType, applicationid=applicationid - ) + workunitid = self.workunit_check(projectid, name=autoQCType, applicationid=applicationid) # print "WUID={}".format(workunitid) resourceid = self.resource_check( @@ -258,9 +248,7 @@ def feed(self, line) -> None: ) # sampleid=0 - print( - f"p{projectid}\tA{applicationid}\t{filename}\tS{sampleid}\tWU{workunitid}\tR{resourceid}" - ) + print(f"p{projectid}\tA{applicationid}\t{filename}\tS{sampleid}\tWU{workunitid}\tR{resourceid}") except Exception as err: print(f"# Failed to register to bfabric: {err}") diff --git a/bfabric_scripts/src/bfabric_scripts/bfabric_flask.py b/bfabric_scripts/src/bfabric_scripts/bfabric_flask.py index 5608ec38..ea1c988b 100755 --- a/bfabric_scripts/src/bfabric_scripts/bfabric_flask.py +++ b/bfabric_scripts/src/bfabric_scripts/bfabric_flask.py @@ -85,9 +85,7 @@ def handle_invalid_request_content(e: InvalidRequestContent) -> Response: return jsonify({"error": f"invalid request content: {e}"}) -def get_fields( - required_fields: list[str], optional_fields: dict[str, Any] -) -> dict[str, Any]: +def get_fields(required_fields: list[str], optional_fields: dict[str, Any]) -> dict[str, Any]: """Extracts fields from a JSON request body. All `required_fields` must be present, or an error will be raised indicating the missing fields. The optional fields are filled with the default values if not present. :param required_fields: list of required fields @@ -100,10 +98,7 @@ def get_fields( raise InvalidRequestContent(sorted(missing_fields)) else: required_values = {field: request.json[field] for field in required_fields} - optional_values = { - field: request.json.get(field, default) - for field, default in optional_fields.items() - } + optional_values = {field: request.json.get(field, default) for field, default in optional_fields.items()} return {**required_values, **optional_values} diff --git a/bfabric_scripts/src/bfabric_scripts/bfabric_list_not_available_proteomics_workunits.py b/bfabric_scripts/src/bfabric_scripts/bfabric_list_not_available_proteomics_workunits.py index 1e45bd9d..48d4a15d 100755 --- a/bfabric_scripts/src/bfabric_scripts/bfabric_list_not_available_proteomics_workunits.py +++ b/bfabric_scripts/src/bfabric_scripts/bfabric_list_not_available_proteomics_workunits.py @@ -23,12 +23,8 @@ def main() -> None: """Parses the command line arguments and calls `list_not_available_proteomics_workunits`.""" setup_script_logging() - parser = ArgumentParser( - description="Lists proteomics work units that are not available on bfabric." - ) - parser.add_argument( - "--max-age", type=int, help="Max age of work units in days", default=14 - ) + parser = ArgumentParser(description="Lists proteomics work units that are not available on bfabric.") + parser.add_argument("--max-age", type=int, help="Max age of work units in days", default=14) args = parser.parse_args() list_not_available_proteomics_workunits(max_age=args.max_age) diff --git a/bfabric_scripts/src/bfabric_scripts/bfabric_list_not_existing_storage_directories.py b/bfabric_scripts/src/bfabric_scripts/bfabric_list_not_existing_storage_directories.py index b4a6f00d..0dcf6a11 100755 --- a/bfabric_scripts/src/bfabric_scripts/bfabric_list_not_existing_storage_directories.py +++ b/bfabric_scripts/src/bfabric_scripts/bfabric_list_not_existing_storage_directories.py @@ -68,9 +68,7 @@ def update(self, client: Bfabric) -> None: # add some buffer to deal e.g. with miss-configured clocks timestamp = (self._data.checked_at - datetime.timedelta(days=1)).isoformat() max_results = 300 if "BFABRICPY_DEBUG" in os.environ else None - logger.debug( - f"Checking for new container ids since {timestamp} with limit {max_results}" - ) + logger.debug(f"Checking for new container ids since {timestamp} with limit {max_results}") result = client.read( endpoint="container", obj={**self._data.query, "createdafter": timestamp}, @@ -83,9 +81,7 @@ def update(self, client: Bfabric) -> None: self._data.checked_at = now -def list_not_existing_storage_dirs( - client: Bfabric, root_dir: Path, cache_path: Path -) -> None: +def list_not_existing_storage_dirs(client: Bfabric, root_dir: Path, cache_path: Path) -> None: """Lists not existing storage directories for a given technology id.""" cache = Cache.load(path=cache_path) cache.update(client=client) @@ -101,9 +97,7 @@ def main() -> None: client = Bfabric.from_config() root_dir = Path("/srv/www/htdocs/") cache_path = Path("cache.json") - list_not_existing_storage_dirs( - client=client, root_dir=root_dir, cache_path=cache_path - ) + list_not_existing_storage_dirs(client=client, root_dir=root_dir, cache_path=cache_path) if __name__ == "__main__": diff --git a/bfabric_scripts/src/bfabric_scripts/bfabric_list_workunit_parameters.py b/bfabric_scripts/src/bfabric_scripts/bfabric_list_workunit_parameters.py index 84fe8b96..3ccf8354 100644 --- a/bfabric_scripts/src/bfabric_scripts/bfabric_list_workunit_parameters.py +++ b/bfabric_scripts/src/bfabric_scripts/bfabric_list_workunit_parameters.py @@ -9,18 +9,14 @@ from bfabric.experimental import MultiQuery -def bfabric_list_workunit_parameters( - client: Bfabric, application_id: int, max_workunits: int, format: str -) -> None: +def bfabric_list_workunit_parameters(client: Bfabric, application_id: int, max_workunits: int, format: str) -> None: """Lists the workunit parameters of the provided application. :param client: The Bfabric client to use. :param application_id: The application ID to list the workunit parameters for. :param max_workunits: The maximum number of workunits to fetch. :param format: The output format to use. """ - workunits_table_full = get_workunits_table_full( - application_id, client, max_workunits - ) + workunits_table_full = get_workunits_table_full(application_id, client, max_workunits) workunits_table_explode = workunits_table_full.explode("parameter").with_columns( parameter_id=pl.col("parameter").struct[1] ) @@ -41,15 +37,11 @@ def bfabric_list_workunit_parameters( print_results(format, merged_result) -def get_workunits_table_full( - application_id: int, client: Bfabric, max_workunits: int -) -> pl.DataFrame: +def get_workunits_table_full(application_id: int, client: Bfabric, max_workunits: int) -> pl.DataFrame: """Returns a table with the workunits for the specified application.""" # read the workunit data workunits_table_full = ( - client.read( - "workunit", {"applicationid": application_id}, max_results=max_workunits - ) + client.read("workunit", {"applicationid": application_id}, max_results=max_workunits) .to_polars() .rename({"id": "workunit_id"}) ) @@ -65,9 +57,7 @@ def get_workunits_table_full( inputdataset_id=pl.col("inputdataset").struct[1], ) else: - workunits_table_full = workunits_table_full.with_columns( - inputdataset_id=pl.lit(None) - ) + workunits_table_full = workunits_table_full.with_columns(inputdataset_id=pl.lit(None)) return workunits_table_full @@ -90,9 +80,7 @@ def print_results(format: str, merged_result: pl.DataFrame) -> None: raise ValueError("Unsupported format") -def get_parameter_table( - client: Bfabric, workunits_table_explode: pl.DataFrame -) -> pl.DataFrame: +def get_parameter_table(client: Bfabric, workunits_table_explode: pl.DataFrame) -> pl.DataFrame: """Returns a wide format table for the specified parameters, with the key `workunit_id` indicating the source.""" # load the parameters table collect = MultiQuery(client=client).read_multi( @@ -101,9 +89,7 @@ def get_parameter_table( multi_query_key="id", multi_query_vals=workunits_table_explode["parameter_id"].to_list(), ) - parameter_table_full = collect.to_polars().rename({"id": "parameter_id"})[ - ["parameter_id", "key", "value"] - ] + parameter_table_full = collect.to_polars().rename({"id": "parameter_id"})[["parameter_id", "key", "value"]] # add workunit id to parameter table parameter_table_full = parameter_table_full.join( workunits_table_explode[["workunit_id", "parameter_id"]], @@ -111,9 +97,7 @@ def get_parameter_table( how="left", ) # convert to wide format - return parameter_table_full.pivot( - values="value", index="workunit_id", columns="key" - ) + return parameter_table_full.pivot(values="value", index="workunit_id", columns="key") def main() -> None: diff --git a/bfabric_scripts/src/bfabric_scripts/bfabric_read.py b/bfabric_scripts/src/bfabric_scripts/bfabric_read.py index 986d3f4c..4299332e 100755 --- a/bfabric_scripts/src/bfabric_scripts/bfabric_read.py +++ b/bfabric_scripts/src/bfabric_scripts/bfabric_read.py @@ -55,9 +55,7 @@ def bfabric_read( possible_attributes = sorted(set(res[0].keys())) logger.info(f"possible attributes = {possible_attributes}") - output_format = _determine_output_format( - console_out=console_out, output_format=output_format, n_results=len(res) - ) + output_format = _determine_output_format(console_out=console_out, output_format=output_format, n_results=len(res)) logger.info(f"output format = {output_format}") if output_format == "json": @@ -111,9 +109,7 @@ def _print_table_tsv(res: list[dict[str, Any]]) -> None: ) -def _determine_output_format( - console_out: Console, output_format: str, n_results: int -) -> str: +def _determine_output_format(console_out: Console, output_format: str, n_results: int) -> str: """Returns the format to use, based on the number of results, and whether the output is an interactive console. If the format is already set to a concrete value instead of "auto", it will be returned unchanged. """ diff --git a/bfabric_scripts/src/bfabric_scripts/bfabric_read_samples_from_dataset.py b/bfabric_scripts/src/bfabric_scripts/bfabric_read_samples_from_dataset.py index 87d3e5a7..246649fe 100755 --- a/bfabric_scripts/src/bfabric_scripts/bfabric_read_samples_from_dataset.py +++ b/bfabric_scripts/src/bfabric_scripts/bfabric_read_samples_from_dataset.py @@ -19,16 +19,10 @@ from bfabric import Bfabric -def get_table_row( - client: Bfabric, relative_path: str -) -> tuple[str, int, str, str, str]: +def get_table_row(client: Bfabric, relative_path: str) -> tuple[str, int, str, str, str]: """Returns the row of the table with the information of the resource with the given relative path.""" - resource = client.read( - endpoint="resource", obj={"relativepath": relative_path} - ).to_list_dict()[0] - sample = client.read( - endpoint="sample", obj={"id": resource["sample"]["id"]} - ).to_list_dict()[0] + resource = client.read(endpoint="resource", obj={"relativepath": relative_path}).to_list_dict()[0] + sample = client.read(endpoint="sample", obj={"id": resource["sample"]["id"]}).to_list_dict()[0] groupingvar = (sample.get("groupingvar") or {}).get("name") or "" return ( resource["workunit"]["id"], @@ -45,13 +39,9 @@ def bfabric_read_samples_from_dataset(dataset_id: int) -> None: client = Bfabric.from_config() dataset = client.read(endpoint="dataset", obj={"id": dataset_id}).to_list_dict()[0] - positions = [ - a["position"] for a in dataset["attribute"] if a["name"] == "Relative Path" - ] + positions = [a["position"] for a in dataset["attribute"] if a["name"] == "Relative Path"] if not positions: - raise ValueError( - f"No 'Relative Path' attribute found in the dataset {dataset_id}" - ) + raise ValueError(f"No 'Relative Path' attribute found in the dataset {dataset_id}") relative_path_position = positions[0] print( @@ -67,16 +57,10 @@ def bfabric_read_samples_from_dataset(dataset_id: int) -> None: ) for item in dataset["item"]: relative_path = [ - field["value"] - for field in item["field"] - if field["attributeposition"] == relative_path_position + field["value"] for field in item["field"] if field["attributeposition"] == relative_path_position ][0] - workunitid, resourceid, resourcename, samplename, groupingvar = get_table_row( - client, relative_path - ) - print( - f"{workunitid}\t{resourceid}\t{resourcename}\t{samplename}\t{groupingvar}" - ) + workunitid, resourceid, resourcename, samplename, groupingvar = get_table_row(client, relative_path) + print(f"{workunitid}\t{resourceid}\t{resourcename}\t{samplename}\t{groupingvar}") def main() -> None: diff --git a/bfabric_scripts/src/bfabric_scripts/bfabric_read_samples_of_workunit.py b/bfabric_scripts/src/bfabric_scripts/bfabric_read_samples_of_workunit.py index ffb82e64..2893b521 100755 --- a/bfabric_scripts/src/bfabric_scripts/bfabric_read_samples_of_workunit.py +++ b/bfabric_scripts/src/bfabric_scripts/bfabric_read_samples_of_workunit.py @@ -27,22 +27,13 @@ def bfabric_read_samples_of_workunit(workunit_id: int) -> None: client = Bfabric.from_config() start_time = time.time() - res_workunit = client.read( - endpoint="workunit", obj={"id": workunit_id} - ).to_list_dict()[0] + res_workunit = client.read(endpoint="workunit", obj={"id": workunit_id}).to_list_dict()[0] input_resource_ids = [x["id"] for x in res_workunit.get("inputresource", [])] - input_resources = client.read( - endpoint="resource", obj={"id": input_resource_ids} - ).to_list_dict() + input_resources = client.read(endpoint="resource", obj={"id": input_resource_ids}).to_list_dict() input_resources_name = [(r["id"], r["name"]) for r in input_resources] - samples = client.read( - endpoint="sample", obj={"id": [x["sample"]["id"] for x in input_resources]} - ).to_list_dict() - groupingvars = [ - (s["id"], s["name"], (s.get("groupingvar") or {}).get("name", "NA")) - for s in samples - ] + samples = client.read(endpoint="sample", obj={"id": [x["sample"]["id"] for x in input_resources]}).to_list_dict() + groupingvars = [(s["id"], s["name"], (s.get("groupingvar") or {}).get("name", "NA")) for s in samples] print( "\t".join( diff --git a/bfabric_scripts/src/bfabric_scripts/bfabric_save_dataset2csv.py b/bfabric_scripts/src/bfabric_scripts/bfabric_save_dataset2csv.py index 226d7495..0d329c3f 100755 --- a/bfabric_scripts/src/bfabric_scripts/bfabric_save_dataset2csv.py +++ b/bfabric_scripts/src/bfabric_scripts/bfabric_save_dataset2csv.py @@ -24,9 +24,7 @@ from bfabric_scripts.cli.base import use_client -def bfabric_save_dataset2csv( - client: Bfabric, dataset_id: int, out_dir: Path, out_filename: Path, sep: str -) -> None: +def bfabric_save_dataset2csv(client: Bfabric, dataset_id: int, out_dir: Path, out_filename: Path, sep: str) -> None: """Saves the dataset with id `dataset_id` to a csv file at `out_dir/out_filename` or `out_filename` if it's an absolute path. """ @@ -44,12 +42,8 @@ def bfabric_save_dataset2csv( @use_client def main(*, client: Bfabric) -> None: """Parses arguments and calls `bfabric_save_dataset2csv`.""" - parser = argparse.ArgumentParser( - description="Save a B-Fabric dataset to a csv file" - ) - parser.add_argument( - "--id", metavar="int", required=True, help="dataset id", type=int - ) + parser = argparse.ArgumentParser(description="Save a B-Fabric dataset to a csv file") + parser.add_argument("--id", metavar="int", required=True, help="dataset id", type=int) parser.add_argument( "--dir", type=Path, diff --git a/bfabric_scripts/src/bfabric_scripts/bfabric_save_fasta.py b/bfabric_scripts/src/bfabric_scripts/bfabric_save_fasta.py index 20d963a6..76a614dd 100755 --- a/bfabric_scripts/src/bfabric_scripts/bfabric_save_fasta.py +++ b/bfabric_scripts/src/bfabric_scripts/bfabric_save_fasta.py @@ -25,9 +25,7 @@ def save_fasta(container_id: int, fasta_file: Path) -> None: with fasta_file.open("rb") as f: md5 = hashlib.md5(f.read()).hexdigest() - resources = client.read( - endpoint="resource", obj={"filechecksum": md5} - ).to_list_dict() + resources = client.read(endpoint="resource", obj={"filechecksum": md5}).to_list_dict() if resources: print("resource(s) already exist.") # TODO this logic was mostly carried over from before, does it still make sense? @@ -66,9 +64,7 @@ def save_fasta(container_id: int, fasta_file: Path) -> None: resource = client.save(endpoint="resource", obj=obj).to_list_dict() print(json.dumps(resource, indent=2)) - workunit = client.save( - endpoint="workunit", obj={"id": workunit[0]["id"], "status": "available"} - ).to_list_dict() + workunit = client.save(endpoint="workunit", obj={"id": workunit[0]["id"], "status": "available"}).to_list_dict() print(json.dumps(workunit, indent=2)) diff --git a/bfabric_scripts/src/bfabric_scripts/bfabric_save_importresource_sample.py b/bfabric_scripts/src/bfabric_scripts/bfabric_save_importresource_sample.py index 57ee3608..773dbd27 100755 --- a/bfabric_scripts/src/bfabric_scripts/bfabric_save_importresource_sample.py +++ b/bfabric_scripts/src/bfabric_scripts/bfabric_save_importresource_sample.py @@ -66,9 +66,7 @@ def create_importresource_dict( file_date = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(file_unix_timestamp)) bfabric_application_ids = config.application_ids if not bfabric_application_ids: - raise RuntimeError( - "No bfabric_application_ids configured. check '~/.bfabricpy.yml' file!" - ) + raise RuntimeError("No bfabric_application_ids configured. check '~/.bfabricpy.yml' file!") bfabric_application_id, bfabric_projectid = get_bfabric_application_and_project_id( bfabric_application_ids, file_path ) @@ -99,9 +97,7 @@ def get_sample_id_from_path(file_path: str) -> int | None: return int(match.group(3)) -def get_bfabric_application_and_project_id( - bfabric_application_ids: dict[str, int], file_path: str -) -> tuple[int, int]: +def get_bfabric_application_and_project_id(bfabric_application_ids: dict[str, int], file_path: str) -> tuple[int, int]: """Returns the bfabric application id and project id for a given file path.""" # linear search through dictionary. first hit counts! bfabric_applicationid = -1 diff --git a/bfabric_scripts/src/bfabric_scripts/bfabric_save_workflowstep.py b/bfabric_scripts/src/bfabric_scripts/bfabric_save_workflowstep.py index f91ca833..910df0d9 100755 --- a/bfabric_scripts/src/bfabric_scripts/bfabric_save_workflowstep.py +++ b/bfabric_scripts/src/bfabric_scripts/bfabric_save_workflowstep.py @@ -45,22 +45,14 @@ def save_workflowstep(workunit_id: int | None = None) -> None: application_id = workunit["application"]["id"] container_id = workunit["container"]["id"] - if ( - application_id in workflowtemplatestep_ids - and application_id in workflowtemplate_ids - ): - workflows = client.read( - "workflow", obj={"containerid": container_id} - ).to_list_dict() + if application_id in workflowtemplatestep_ids and application_id in workflowtemplate_ids: + workflows = client.read("workflow", obj={"containerid": container_id}).to_list_dict() # if workflows is None, no workflow is available - > create a new one daw_id = -1 if workflows: # check if the corresponding workflow exists (template id 59) for item in workflows: - if ( - item["workflowtemplate"]["id"] - == workflowtemplate_ids[application_id] - ): + if item["workflowtemplate"]["id"] == workflowtemplate_ids[application_id]: daw_id = item["id"] break # case when no workflows are available (workflows == None) @@ -88,9 +80,7 @@ def save_workflowstep(workunit_id: int | None = None) -> None: def main() -> None: """Parses command line args and calls `save_workflowstep`.""" parser = argparse.ArgumentParser(description="Create an analysis workflow step") - parser.add_argument( - "workunitid", metavar="workunitid", type=int, help="workunit id" - ) + parser.add_argument("workunitid", metavar="workunitid", type=int, help="workunit id") args = parser.parse_args() save_workflowstep(workunit_id=args.workunitid) diff --git a/bfabric_scripts/src/bfabric_scripts/bfabric_save_workunit_attribute.py b/bfabric_scripts/src/bfabric_scripts/bfabric_save_workunit_attribute.py index 21806372..1de1462a 100755 --- a/bfabric_scripts/src/bfabric_scripts/bfabric_save_workunit_attribute.py +++ b/bfabric_scripts/src/bfabric_scripts/bfabric_save_workunit_attribute.py @@ -17,14 +17,10 @@ from bfabric import Bfabric -def bfabric_save_workunit_attribute( - workunit_id: int, attribute: str, value: str -) -> None: +def bfabric_save_workunit_attribute(workunit_id: int, attribute: str, value: str) -> None: """Sets the specified attribute to the specified value for the specified workunit.""" client = Bfabric.from_config() - result = client.save( - endpoint="workunit", obj={"id": workunit_id, attribute: value} - ).to_list_dict() + result = client.save(endpoint="workunit", obj={"id": workunit_id, attribute: value}).to_list_dict() print(json.dumps(result[0], indent=2)) diff --git a/bfabric_scripts/src/bfabric_scripts/bfabric_setExternalJobStatus_done.py b/bfabric_scripts/src/bfabric_scripts/bfabric_setExternalJobStatus_done.py index 29f10207..5eb80b1e 100755 --- a/bfabric_scripts/src/bfabric_scripts/bfabric_setExternalJobStatus_done.py +++ b/bfabric_scripts/src/bfabric_scripts/bfabric_setExternalJobStatus_done.py @@ -24,9 +24,7 @@ def set_external_job_status_done(client: Bfabric, external_job_id: list[int]) -> """Sets the status of the specified external jobs to 'done'.""" for job_id in external_job_id: try: - res = client.save( - "externaljob", {"id": job_id, "status": "done"} - ).to_list_dict() + res = client.save("externaljob", {"id": job_id, "status": "done"}).to_list_dict() print(res) except Exception: print(f"failed to set externaljob with id={job_id} 'available'.") diff --git a/bfabric_scripts/src/bfabric_scripts/bfabric_setWorkunitStatus.py b/bfabric_scripts/src/bfabric_scripts/bfabric_setWorkunitStatus.py index 2cd8bb65..e120a3ec 100755 --- a/bfabric_scripts/src/bfabric_scripts/bfabric_setWorkunitStatus.py +++ b/bfabric_scripts/src/bfabric_scripts/bfabric_setWorkunitStatus.py @@ -17,9 +17,7 @@ def main_generic(result_status: str) -> None: """Main function for setting workunit status to `result_status`.""" - parser = argparse.ArgumentParser( - description=f"Sets workunit status to '{result_status}'" - ) + parser = argparse.ArgumentParser(description=f"Sets workunit status to '{result_status}'") parser.add_argument("workunit_id", type=int, help="workunit id") args = parser.parse_args() client = Bfabric.from_config() diff --git a/bfabric_scripts/src/bfabric_scripts/bfabric_slurm_queue_status.py b/bfabric_scripts/src/bfabric_scripts/bfabric_slurm_queue_status.py index ad27e769..21d4fa3e 100644 --- a/bfabric_scripts/src/bfabric_scripts/bfabric_slurm_queue_status.py +++ b/bfabric_scripts/src/bfabric_scripts/bfabric_slurm_queue_status.py @@ -12,6 +12,7 @@ from bfabric import Bfabric from bfabric.entities import Workunit, Application +from bfabric_scripts.cli.base import use_client def get_slurm_jobs(partition: str, ssh_host: str | None) -> pl.DataFrame: @@ -25,24 +26,19 @@ def get_slurm_jobs(partition: str, ssh_host: str | None) -> pl.DataFrame: command = [ "ssh", ssh_host, - "bash -l -c " - + shlex.quote(" ".join(shlex.quote(arg) for arg in target_command)), + "bash -l -c " + shlex.quote(" ".join(shlex.quote(arg) for arg in target_command)), ] - logger.info(f"Running command: {' '.join(command)}") + logger.info(f"Running command: {shlex.join(command)}") output = subprocess.run(command, stdout=subprocess.PIPE, text=True, check=True) stringio = io.StringIO(output.stdout) df = pl.read_csv(stringio, separator="\t") df = df.rename({"JOBID": "job_id", "NAME": "name", "NODELIST": "node_list"}) string_id_expr = pl.col("name").str.extract(r"WU(\d+)") - return df.with_columns( - workunit_id=pl.when(string_id_expr.is_not_null()).then(string_id_expr.cast(int)) - ) + return df.with_columns(workunit_id=pl.when(string_id_expr.is_not_null()).then(string_id_expr.cast(int))) -def get_workunit_infos( - client: Bfabric, workunit_ids: list[int] -) -> list[dict[str, str]]: +def get_workunit_infos(client: Bfabric, workunit_ids: list[int]) -> list[dict[str, str]]: """Retrieves information about the workunits with the specified ids. If a workunit was deleted, but it is in the slurm queue, it will be considered a zombie. """ @@ -57,22 +53,14 @@ def get_workunit_infos( return [ { "workunit_id": id, - "status": ( - workunits[id].data_dict["status"] if id in workunits else "ZOMBIE" - ), - "application_name": ( - app_names[workunits[id]["application"]["id"]] - if id in workunits - else "N/A" - ), + "status": (workunits[id].data_dict["status"] if id in workunits else "ZOMBIE"), + "application_name": (app_names[workunits[id]["application"]["id"]] if id in workunits else "N/A"), } for id in workunit_ids ] -def find_zombie_jobs( - client: Bfabric, partition: str, ssh_host: str | None -) -> pl.DataFrame: +def find_zombie_jobs(client: Bfabric, partition: str, ssh_host: str | None) -> pl.DataFrame: """Checks the status of the slurm jobs in the specified partition, and returns the ones that are zombies.""" slurm_jobs = get_slurm_jobs(partition=partition, ssh_host=ssh_host) if slurm_jobs.is_empty(): @@ -84,27 +72,19 @@ def find_zombie_jobs( ) ) pl.Config.set_tbl_rows(100) - logger.info( - slurm_jobs.join(workunit_info_table, on="workunit_id", how="left").sort( - "workunit_id" - ) - ) + logger.info(slurm_jobs.join(workunit_info_table, on="workunit_id", how="left").sort("workunit_id")) logger.info(f"Active jobs: {workunit_info_table.height}") - logger.info( - f"Found {workunit_info_table.filter(pl.col('status') == 'ZOMBIE').height} zombie jobs." - ) + logger.info(f"Found {workunit_info_table.filter(pl.col('status') == 'ZOMBIE').height} zombie jobs.") return workunit_info_table.filter(pl.col("status") == "ZOMBIE") -def main() -> None: +@use_client +def main(*, client: Bfabric) -> None: """Checks the status of the slurm jobs in the specified partition, and reports if there are any zombies.""" parser = argparse.ArgumentParser() parser.add_argument("--partition", type=str, default="prx") - parser.add_argument( - "--ssh", type=str, default=None, help="SSH into the given node to obtain list." - ) + parser.add_argument("--ssh", type=str, default=None, help="SSH into the given node to obtain list.") args = parser.parse_args() - client = Bfabric.from_config() zombie_jobs = find_zombie_jobs(client, partition=args.partition, ssh_host=args.ssh) if zombie_jobs.is_empty(): print(json.dumps([])) diff --git a/bfabric_scripts/src/bfabric_scripts/bfabric_upload_resource.py b/bfabric_scripts/src/bfabric_scripts/bfabric_upload_resource.py index e518fc05..0e9a8937 100755 --- a/bfabric_scripts/src/bfabric_scripts/bfabric_upload_resource.py +++ b/bfabric_scripts/src/bfabric_scripts/bfabric_upload_resource.py @@ -36,9 +36,7 @@ def main(*, client: Bfabric) -> None: parser.add_argument("filename", help="filename", type=Path) parser.add_argument("workunitid", help="workunitid", type=int) args = parser.parse_args() - bfabric_upload_resource( - client=client, filename=args.filename, workunit_id=args.workunitid - ) + bfabric_upload_resource(client=client, filename=args.filename, workunit_id=args.workunitid) if __name__ == "__main__": diff --git a/bfabric_scripts/src/bfabric_scripts/bfabric_upload_submitter_executable.py b/bfabric_scripts/src/bfabric_scripts/bfabric_upload_submitter_executable.py index efd35531..26895781 100755 --- a/bfabric_scripts/src/bfabric_scripts/bfabric_upload_submitter_executable.py +++ b/bfabric_scripts/src/bfabric_scripts/bfabric_upload_submitter_executable.py @@ -56,9 +56,7 @@ def main() -> None: choices=["slurm"], help="Valid engines for job handling are: slurm, gridengine", ) - parser.add_argument( - "--name", type=str, help="Name of the submitter", required=False - ) + parser.add_argument("--name", type=str, help="Name of the submitter", required=False) parser.add_argument( "--description", type=str, diff --git a/bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_log.py b/bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_log.py index 865d0840..f1ef6b71 100644 --- a/bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_log.py +++ b/bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_log.py @@ -48,9 +48,7 @@ def write_workunit(client: Bfabric, workunit_id: int, message: str) -> None: f"Expected exactly one external job for workunit {workunit_id}, but found {len(external_jobs)}" ) else: - write_externaljob( - client=client, externaljob_id=external_jobs[0]["id"], message=message - ) + write_externaljob(client=client, externaljob_id=external_jobs[0]["id"], message=message) def write_externaljob(client: Bfabric, externaljob_id: int, message: str) -> None: diff --git a/bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_read.py b/bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_read.py index 4eaa969f..6c9b8b20 100644 --- a/bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_read.py +++ b/bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_read.py @@ -87,9 +87,7 @@ def bfabric_read( console=console_out, ) # _print_query_rich(console_out, query) - _print_table_rich( - client.config, console_out, endpoint, results, output_columns=output_columns - ) + _print_table_rich(client.config, console_out, endpoint, results, output_columns=output_columns) else: raise ValueError(f"output format {output_format} not supported") @@ -111,9 +109,7 @@ def _determine_output_columns( return columns -def _get_results( - client: Bfabric, endpoint: str, query: dict[str, str], limit: int -) -> list[dict[str, Any]]: +def _get_results(client: Bfabric, endpoint: str, query: dict[str, str], limit: int) -> list[dict[str, Any]]: start_time = time.time() results = client.read(endpoint=endpoint, obj=query, max_results=limit) end_time = time.time() @@ -160,9 +156,7 @@ def _print_table_rich( console_out.print(table) -def _determine_output_format( - console_out: Console, output_format: OutputFormat, n_results: int -) -> OutputFormat: +def _determine_output_format(console_out: Console, output_format: OutputFormat, n_results: int) -> OutputFormat: """Returns the format to use, based on the number of results, and whether the output is an interactive console. If the format is already set to a concrete value instead of "auto", it will be returned unchanged. """ diff --git a/bfabric_scripts/src/bfabric_scripts/cli/cli_external_job.py b/bfabric_scripts/src/bfabric_scripts/cli/cli_external_job.py index 7122c339..f4a4b4af 100644 --- a/bfabric_scripts/src/bfabric_scripts/cli/cli_external_job.py +++ b/bfabric_scripts/src/bfabric_scripts/cli/cli_external_job.py @@ -29,9 +29,7 @@ def find_slurm_root() -> str: @app.command @use_client -def submitter( - external_job_id: int, scheduler: Literal["Slurm"] = "Slurm", *, client: Bfabric -) -> None: +def submitter(external_job_id: int, scheduler: Literal["Slurm"] = "Slurm", *, client: Bfabric) -> None: if scheduler != "Slurm": raise NotImplementedError(f"Unsupported scheduler: {scheduler}") slurm_root = find_slurm_root() diff --git a/bfabric_scripts/src/bfabric_scripts/cli/executable/inspect.py b/bfabric_scripts/src/bfabric_scripts/cli/executable/inspect.py index d16ae6af..713c9276 100644 --- a/bfabric_scripts/src/bfabric_scripts/cli/executable/inspect.py +++ b/bfabric_scripts/src/bfabric_scripts/cli/executable/inspect.py @@ -7,9 +7,7 @@ from bfabric_scripts.cli.base import use_client -def get_storage_info( - storage_id: int | None, client: Bfabric -) -> dict[str, str | int] | None: +def get_storage_info(storage_id: int | None, client: Bfabric) -> dict[str, str | int] | None: if storage_id is None: return None storage_info = Storage.find(storage_id, client=client) @@ -20,13 +18,8 @@ def get_storage_info( def inspect_executable(executable_id: int, *, client: Bfabric) -> None: console = Console() executable = Executable.find(executable_id, client=client) - metadata = { - key: executable.get(key) - for key in ("id", "name", "description", "relativepath", "context", "program") - } - metadata["storage"] = get_storage_info( - executable.storage.id if executable.storage else None, client - ) + metadata = {key: executable.get(key) for key in ("id", "name", "description", "relativepath", "context", "program")} + metadata["storage"] = get_storage_info(executable.storage.id if executable.storage else None, client) console.print(Panel("Executable Metadata")) pprint(metadata) diff --git a/bfabric_scripts/src/bfabric_scripts/cli/executable/upload.py b/bfabric_scripts/src/bfabric_scripts/cli/executable/upload.py index 10054128..3869ac5b 100644 --- a/bfabric_scripts/src/bfabric_scripts/cli/executable/upload.py +++ b/bfabric_scripts/src/bfabric_scripts/cli/executable/upload.py @@ -10,9 +10,7 @@ @use_client -def upload_executable( - executable_yaml: Path, *, upload: Path | None = None, client: Bfabric -) -> None: +def upload_executable(executable_yaml: Path, *, upload: Path | None = None, client: Bfabric) -> None: """Uploads an executable defined in the specified YAML to bfabric. :param executable_yaml: Path to the YAML file containing the executable data. @@ -28,9 +26,7 @@ def upload_executable( raise ValueError(msg) executable_data = executable_data["executable"] if upload is not None: - executable_data["base64"] = base64.encodebytes(upload.read_bytes()).decode( - "utf-8" - ) + executable_data["base64"] = base64.encodebytes(upload.read_bytes()).decode("utf-8") # Ensure id is not set if "id" in executable_data: @@ -46,6 +42,4 @@ def upload_executable( console.print("Executable uploaded successfully.") console.print("Executable ID:", executable_id) - console.print( - "Executable URL:", Executable({"id": executable_id}, client=client).web_url - ) + console.print("Executable URL:", Executable({"id": executable_id}, client=client).web_url) diff --git a/bfabric_scripts/src/bfabric_scripts/cli/external_job/upload_submitter_executable.py b/bfabric_scripts/src/bfabric_scripts/cli/external_job/upload_submitter_executable.py index 1848c5f4..cf1c216a 100644 --- a/bfabric_scripts/src/bfabric_scripts/cli/external_job/upload_submitter_executable.py +++ b/bfabric_scripts/src/bfabric_scripts/cli/external_job/upload_submitter_executable.py @@ -9,9 +9,7 @@ def slurm_parameters() -> list[dict[str, str]]: - parameters = [ - {"modifiable": "true", "required": "true", "type": "STRING"} for _ in range(3) - ] + parameters = [{"modifiable": "true", "required": "true", "type": "STRING"} for _ in range(3)] parameters[0]["description"] = "Which Slurm partition should be used." parameters[0]["enumeration"] = ["prx", "mascot"] parameters[0]["key"] = "partition" @@ -58,10 +56,7 @@ def upload_submitter_executable( if engine == "slurm": name = name or "yaml / Slurm executable" - description = ( - description - or "Submitter executable for the bfabric functional test using Slurm." - ) + description = description or "Submitter executable for the bfabric functional test using Slurm." attr["version"] = "1.03" attr["parameter"] = slurm_parameters() else: diff --git a/bfabric_scripts/src/bfabric_scripts/cli/workunit/export_definition.py b/bfabric_scripts/src/bfabric_scripts/cli/workunit/export_definition.py index af435489..230ef8d7 100644 --- a/bfabric_scripts/src/bfabric_scripts/cli/workunit/export_definition.py +++ b/bfabric_scripts/src/bfabric_scripts/cli/workunit/export_definition.py @@ -8,17 +8,13 @@ @use_client -def export_definition( - workunit_id: int, target_path: Path | None = None, *, client: Bfabric -) -> None: +def export_definition(workunit_id: int, target_path: Path | None = None, *, client: Bfabric) -> None: """Exports a workunit_definition.yml file for the specified workunit. :param workunit_id: the workunit ID :param target_path: the target path for the workunit_definition.yml file (default: workunit_definition.yml) """ - target_path = ( - target_path if target_path is not None else Path("workunit_definition.yml") - ) + target_path = target_path if target_path is not None else Path("workunit_definition.yml") workunit_definition = WorkunitDefinition.from_ref(workunit_id, client=client) target_path.parent.mkdir(parents=True, exist_ok=True) data = workunit_definition.model_dump(mode="json") diff --git a/bfabric_scripts/src/bfabric_scripts/cli/workunit/not_available.py b/bfabric_scripts/src/bfabric_scripts/cli/workunit/not_available.py index 7cd026de..1e58db45 100644 --- a/bfabric_scripts/src/bfabric_scripts/cli/workunit/not_available.py +++ b/bfabric_scripts/src/bfabric_scripts/cli/workunit/not_available.py @@ -1,4 +1,6 @@ +from __future__ import annotations from datetime import datetime, timedelta +from typing import Iterable from loguru import logger from rich.console import Console @@ -9,9 +11,7 @@ from bfabric_scripts.cli.base import use_client -def render_output( - workunits_by_status: dict[str, list[Workunit]], client: Bfabric -) -> None: +def render_output(workunits: list[Workunit], client: Bfabric) -> None: """Renders the output as a table.""" table = Table( Column("Application", no_wrap=False), @@ -23,55 +23,75 @@ def render_output( Column("Nodelist", no_wrap=False), ) - workunit_ids = [wu.id for wu_list in workunits_by_status.values() for wu in wu_list] - app_ids = { - wu["application"]["id"] - for wu_list in workunits_by_status.values() - for wu in wu_list - } + workunit_ids = [wu.id for wu in workunits] + app_ids = {wu["application"]["id"] for wu in workunits} - nodelist_params = Parameter.find_by( - {"workunitid": workunit_ids, "key": "nodelist"}, client - ) - nodelist_values = { - param["workunit"]["id"]: param.value for param in nodelist_params.values() - } + nodelist_params = Parameter.find_by({"workunitid": workunit_ids, "key": "nodelist"}, client) + nodelist_values = {param["workunit"]["id"]: param.value for param in nodelist_params.values()} application_values = Application.find_all(ids=sorted(app_ids), client=client) - for status, workunits_all in workunits_by_status.items(): - workunits = [ - x for x in workunits_all if x["createdby"] not in ["gfeeder", "itfeeder"] - ] - status_color = { - "Pending": "yellow", - "Processing": "blue", - "Failed": "red", - }.get(status, "black") - - for wu in workunits: - app = application_values[wu["application"]["id"]] - table.add_row( - f"[link={app.web_url}]A{wu['application']['id']:3} {app['name']}[/link]", - f"[link={wu.web_url}&tab=details]WU{wu['id']}[/link]", - wu["created"], - f"[{status_color}]{status}[/{status_color}]", - wu["createdby"], - wu["name"], - nodelist_values.get(wu.id, "N/A"), - ) + status_colors = { + "PENDING": "yellow", + "PROCESSING": "blue", + "FAILED": "red", + } + + for wu in workunits: + status_color = status_colors.get(wu["status"], "black") + app = application_values[wu["application"]["id"]] + table.add_row( + f"[link={app.web_url}]A{wu['application']['id']:3} {app['name']}[/link]", + f"[link={wu.web_url}&tab=details]WU{wu['id']}[/link]", + wu["created"], + f"[{status_color}]{wu['status']}[/{status_color}]", + wu["createdby"], + wu["name"], + nodelist_values.get(wu.id, "N/A"), + ) console = Console() console.print(table) +def sort_workunits_by(workunits: Iterable[Workunit], key: str) -> list[Workunit]: + if key == "status": + order = ["PENDING", "PROCESSING", "FAILED"] + return sorted(workunits, key=lambda wu: order.index(wu["status"])) + elif key in ("application", "app"): + return sorted(workunits, key=lambda wu: wu["application"]["id"]) + else: + workunits_list = list(workunits) + if workunits_list and key in workunits_list[0]: + return sorted(workunits_list, key=lambda wu: wu[key]) + logger.warning(f"Unknown sort key: {key}") + return list(workunits) + + +def filter_workunits_by_user(workunits: list[Workunit], exclude_user: list[str] | None) -> list[Workunit]: + if exclude_user: + return [wu for wu in workunits if wu["createdby"] not in exclude_user] + return workunits + + @use_client def list_not_available_proteomics_workunits( - *, client: Bfabric, max_age: float = 14.0 + *, + client: Bfabric, + max_age: float = 14.0, + sort_by: str = "status", + exclude_user: list[str] | None = None, + include_user: list[str] | None = None, ) -> None: """Lists not available analysis work units. :param max_age: The maximum age of work units in days. + :param sort_by: The field to sort the output by. + :param exclude_user: List of users to exclude from the output (implicit default: gfeeder, itfeeder) + :param include_user: List of users to include in the output """ + if exclude_user and include_user: + raise ValueError("Cannot provide both include and exclude users") + date_cutoff = datetime.today() - timedelta(days=max_age) console = Console() with console.capture() as capture: @@ -82,10 +102,19 @@ def list_not_available_proteomics_workunits( ) logger.info(capture.get()) - workunits_by_status = {} - for status in ["Pending", "Processing", "Failed"]: - workunits_by_status[status] = Workunit.find_by( - {"status": status, "createdafter": date_cutoff.isoformat()}, client=client - ).values() - - render_output(workunits_by_status, client=client) + extra_query = {} + if include_user: + extra_query["createdby"] = include_user + workunits = Workunit.find_by( + { + "status": ["Pending", "Processing", "Failed"], + "createdafter": date_cutoff.isoformat(), + **extra_query, + }, + client=client, + ).values() + workunits = sort_workunits_by(workunits, sort_by) + if not include_user and not exclude_user: + exclude_user = ["gfeeder", "itfeeder"] + workunits = filter_workunits_by_user(workunits, exclude_user) + render_output(workunits, client=client) diff --git a/bfabric_scripts/src/bfabric_scripts/fgcz_maxquant_scaffold-wrapper.py b/bfabric_scripts/src/bfabric_scripts/fgcz_maxquant_scaffold-wrapper.py index d9286dfe..fa8f461f 100755 --- a/bfabric_scripts/src/bfabric_scripts/fgcz_maxquant_scaffold-wrapper.py +++ b/bfabric_scripts/src/bfabric_scripts/fgcz_maxquant_scaffold-wrapper.py @@ -46,18 +46,12 @@ def __init__(self, yamlfilename=None, zipfilename=None) -> None: try: self.fasta = os.path.basename( - self.config["application"]["parameters"][ - "/fastaFiles/FastaFileInfo/fastaFilePath" - ] + self.config["application"]["parameters"]["/fastaFiles/FastaFileInfo/fastaFilePath"] ) except: raise - L = [ - value - for values in self.config["application"]["input"].values() - for value in values - ] + L = [value for values in self.config["application"]["input"].values() for value in values] self.samples = list(map(lambda x: os.path.basename(x).replace(".raw", ""), L)) @@ -129,9 +123,7 @@ def run(self) -> None: eFastaDatabase.attrib["path"] = f"{os.getcwd()}/{self.fasta}" for s in self.samples: - eExperiment.extend( - self.getBiologicalSample(category=s, InputFile=self.zipfilename) - ) + eExperiment.extend(self.getBiologicalSample(category=s, InputFile=self.zipfilename)) xml.write( "/dev/stdout", @@ -169,7 +161,5 @@ def run(self) -> None: ) (options, args) = parser.parse_args() - driver = FgczMaxQuantScaffold( - yamlfilename=options.yaml_filename, zipfilename=options.zip_filename - ) + driver = FgczMaxQuantScaffold(yamlfilename=options.yaml_filename, zipfilename=options.zip_filename) driver.run() diff --git a/bfabric_scripts/src/bfabric_scripts/fgcz_maxquant_wrapper.py b/bfabric_scripts/src/bfabric_scripts/fgcz_maxquant_wrapper.py index 2318db1d..f0b4d230 100755 --- a/bfabric_scripts/src/bfabric_scripts/fgcz_maxquant_wrapper.py +++ b/bfabric_scripts/src/bfabric_scripts/fgcz_maxquant_wrapper.py @@ -41,9 +41,7 @@ class FgczMaxQuantConfig: def __init__(self, config=None, scratch="/scratch/MAXQUANT/") -> None: if config: self.config = config - self.scratchdir = Path( - f"{scratch}/WU{self.config['job_configuration']['workunit_id']}" - ) + self.scratchdir = Path(f"{scratch}/WU{self.config['job_configuration']['workunit_id']}") if not os.path.isdir(self.scratchdir): print(f"no scratch dir '{self.scratchdir}'.") @@ -99,9 +97,7 @@ def generate_mqpar(self, xml_filename, xml_template) -> None: raise TypeError estring = etree.Element("string") - estring.text = ( - f"{os.path.basename(input).replace('.raw', '').replace('.RAW', '')}" - ) + estring.text = f"{os.path.basename(input).replace('.raw', '').replace('.RAW', '')}" ecount += 1 element.extend(estring) @@ -479,9 +475,7 @@ def run(self) -> None: """ if __name__ == "__main__": - parser = OptionParser( - usage="usage: %prog -y ", version="%prog 1.0" - ) + parser = OptionParser(usage="usage: %prog -y ", version="%prog 1.0") parser.add_option( "-y", diff --git a/docs/changelog.md b/docs/changelog.md index fd516535..ace55085 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -10,6 +10,12 @@ Versioning currently follows `X.Y.Z` where ## \[Unreleased\] +## \[1.13.19\] - 2025-02-06 + +### Fixed + +- Config: Log messages of app runner are shown by default again. + ## \[1.13.18\] - 2025-01-28 ### Changed diff --git a/docs/index.md b/docs/index.md index 8340e54d..bf3c8723 100644 --- a/docs/index.md +++ b/docs/index.md @@ -7,40 +7,60 @@ Several pieces of functionality are available: - General client for all B-Fabric web service operations (CRUD) and configuration management. - A relational API for low-boilerplate read access to the B-Fabric system. - Scripts: Several scripts we use more or less frequently to interact with the system. -- A REST API: A REST API to interact with the B-Fabric system. This allows us to interact with B-Fabric from R using [bfabricShiny](https://github.com/cpanse/bfabricShiny). +- A REST API: A REST API to interact with the B-Fabric system. This allows us to interact with B-Fabric from R + using [bfabricShiny](https://github.com/cpanse/bfabricShiny). Please see below for how to install bfabricPy. ## Installation -The package is not available on PyPI as of now, but can be installed directly from GitHub and a `stable` branch is available for your convenience. +The [bfabric](https://pypi.org/project/bfabric/) and [bfabric-scripts](https://pypi.org/project/bfabric-scripts/) +packages are available on PyPI. +If you want to use the API in your code the `bfabric` package already provides all relevant functionality, whereas +`bfabric-scripts` will provide several command line tools to interact with the B-Fabric system. -If you are only interested in running the command line scripts, installation with `uv tool` is recommended as it will create a separate virtual environment for bfabricPy and make it possible to upgrade your installation later easily. +### Installing the tool + +If you are only interested in running the command line scripts, installation with `uv tool` is recommended as it will +create a separate virtual environment for bfabric-scripts and make it possible to upgrade your installation later +easily. + +```bash +uv tool install -p 3.13 bfabric-scripts +``` + +You can upgrade this installation to the most recent version later with: ```bash -uv tool install -p 3.13 "git+https://github.com/fgcz/bfabricPy.git@stable" +uv tool upgrade bfabric-scripts ``` -If you want to add it to a `pyproject.toml` the syntax for specifying a git dependency is as follows: +### Declaring a package dependency + +If you want to add it to a `pyproject.toml`, simply add bfabric to your dependencies: ```toml [project] dependencies = [ - "bfabric @ git+https://github.com/fgcz/bfabricPy.git@stable" + "bfabric==x.y.z" ] ``` -## Updating +where you replace `x.y.z` with the version you want to use. -If you installed with `uv`, you can update the package to the most recent release with the following command: +If you instead want to install a development version, you can specify the git repository and branch to use: -```bash -uv tool upgrade bfabric +```toml +[project] +dependencies = [ + "bfabric @ git+https://github.com/fgcz/bfabricPy.git@stable&subdirectory=bfabric#egg=bfabric", +] ``` ## Configuration -Create a file as follows: (note: the password is not your login password, but the web service password available on your profile page) +Create a file as follows: (note: the password is not your login password, but the web service password available on your +profile page) ```yaml # ~/.bfabricpy.yml @@ -54,7 +74,8 @@ PRODUCTION: base_url: https://fgcz-bfabric.uzh.ch/bfabric ``` -You can also append an additional config section for the TEST instance which will be used for instance when running the integration tests: +You can also append an additional config section for the TEST instance which will be used for instance when running the +integration tests: ```yaml TEST: @@ -63,6 +84,9 @@ TEST: base_url: https://fgcz-bfabric-test.uzh.ch/bfabric ``` -When you run an application using bfabricPy, and it does not explicitly set the config when calling `Bfabric.from_config`, you can adjust the -environment that is used by setting the environemnt variable `BFABRICPY_CONFIG_ENV` to the name of the config section you want to use. -Command line scripts will log the user and base URL that is used, so you can verify that you are indeed using the correct environment. +When you run an application using bfabricPy, and it does not explicitly set the config when calling +`Bfabric.from_config`, you can adjust the +environment that is used by setting the environemnt variable `BFABRICPY_CONFIG_ENV` to the name of the config section +you want to use. +Command line scripts will log the user and base URL that is used, so you can verify that you are indeed using the +correct environment. diff --git a/noxfile.py b/noxfile.py index 54aaa48f..2941c8a0 100644 --- a/noxfile.py +++ b/noxfile.py @@ -23,12 +23,10 @@ def tests(session): @nox.session(python=["3.13"]) def test_app_runner(session): - # TODO this one has a problem that bfabric gets installed from `@main` (so it could break CI) - session.install("./bfabric") - session.install("./app_runner[test]") - session.install("--upgrade", "./bfabric") + session.install("-e", "./bfabric") + session.install("./bfabric_app_runner[test]") session.run("uv", "pip", "list") - session.run("pytest", "--durations=50", "tests/app_runner") + session.run("pytest", "--durations=50", "tests/bfabric_app_runner") @nox.session @@ -53,12 +51,12 @@ def docs(session): session.install("./bfabric[doc]") session.run("mkdocs", "build", "-d", Path(tmpdir) / "build_bfabricpy") - session.install("./app_runner[doc]") + session.install("./bfabric_app_runner[doc]") session.run( "sphinx-build", "-M", "html", - "app_runner/docs", + "bfabric_app_runner/docs", Path(tmpdir) / "build_app_runner", ) @@ -67,9 +65,7 @@ def docs(session): shutil.rmtree(target_dir) shutil.copytree(Path(tmpdir) / "build_bfabricpy", target_dir) - shutil.copytree( - Path(tmpdir) / "build_app_runner" / "html", target_dir / "app_runner" - ) + shutil.copytree(Path(tmpdir) / "build_app_runner" / "html", target_dir / "app_runner") @nox.session(default=False) diff --git a/pyproject.toml b/pyproject.toml index 43300ffe..5a8eb33c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,3 +7,12 @@ logot_capturer = "logot.loguru.LoguruCapturer" "**/examples/**" = ["ALL"] "**/tests/**" = ["ALL"] "noxfile.py" = ["ALL"] + +[tool.black] +line-length = 120 +target-version = ["py311"] + +[tool.ruff] +line-length = 120 +indent-width = 4 +target-version = "py39" diff --git a/tests/bfabric/config/test_bfabric_auth.py b/tests/bfabric/config/test_bfabric_auth.py index f2624851..8cbeadca 100644 --- a/tests/bfabric/config/test_bfabric_auth.py +++ b/tests/bfabric/config/test_bfabric_auth.py @@ -11,17 +11,11 @@ def example_config_path() -> Path: def test_bfabric_auth_repr() -> None: - assert ( - repr(BfabricAuth(login="login", password="x" * 32)) - == "BfabricAuth(login='login', password=...)" - ) + assert repr(BfabricAuth(login="login", password="x" * 32)) == "BfabricAuth(login='login', password=...)" def test_bfabric_auth_str() -> None: - assert ( - str(BfabricAuth(login="login", password="x" * 32)) - == "BfabricAuth(login='login', password=...)" - ) + assert str(BfabricAuth(login="login", password="x" * 32)) == "BfabricAuth(login='login', password=...)" if __name__ == "__main__": diff --git a/tests/bfabric/config/test_bfabric_client_config.py b/tests/bfabric/config/test_bfabric_client_config.py index 682f408d..c156efc9 100644 --- a/tests/bfabric/config/test_bfabric_client_config.py +++ b/tests/bfabric/config/test_bfabric_client_config.py @@ -30,9 +30,7 @@ def test_bfabric_config_default_params_when_omitted() -> None: def test_bfabric_config_default_params_when_specified() -> None: - config = BfabricClientConfig( - base_url=None, application_ids=None, job_notification_emails=None - ) + config = BfabricClientConfig(base_url=None, application_ids=None, job_notification_emails=None) assert config.base_url == "https://fgcz-bfabric.uzh.ch/bfabric" assert config.application_ids == {} assert config.job_notification_emails == "" @@ -66,9 +64,7 @@ def test_bfabric_config_copy_with_replaced_when_invalid( mock_config.copy_with(base_url="not a url") -def test_bfabric_config_read_yml_bypath_default( - mocker: MockerFixture, example_config_path: Path, logot: Logot -) -> None: +def test_bfabric_config_read_yml_bypath_default(mocker: MockerFixture, example_config_path: Path, logot: Logot) -> None: # Ensure environment variable is not available, and the default is environment is loaded mocker.patch.dict(os.environ, {}, clear=True) @@ -77,16 +73,8 @@ def test_bfabric_config_read_yml_bypath_default( assert auth.password == "01234567890123456789012345678901" assert config.base_url == "https://mega-production-server.uzh.ch/myprod" - logot.assert_logged( - logged.debug( - f"Reading configuration from: {str(example_config_path.absolute())}" - ) - ) - logot.assert_logged( - logged.debug( - "BFABRICPY_CONFIG_ENV not found, using default environment PRODUCTION" - ) - ) + logot.assert_logged(logged.debug(f"Reading configuration from: {str(example_config_path.absolute())}")) + logot.assert_logged(logged.debug("BFABRICPY_CONFIG_ENV not found, using default environment PRODUCTION")) def test_bfabric_config_read_yml_bypath_environment_variable( @@ -100,11 +88,7 @@ def test_bfabric_config_read_yml_bypath_environment_variable( assert auth.password == "012345678901234567890123456789ff" assert config.base_url == "https://mega-test-server.uzh.ch/mytest" - logot.assert_logged( - logged.debug( - f"Reading configuration from: {str(example_config_path.absolute())}" - ) - ) + logot.assert_logged(logged.debug(f"Reading configuration from: {str(example_config_path.absolute())}")) logot.assert_logged(logged.debug("found BFABRICPY_CONFIG_ENV = TEST")) diff --git a/tests/bfabric/config/test_config_file.py b/tests/bfabric/config/test_config_file.py index a2ec11ec..cab1129d 100644 --- a/tests/bfabric/config/test_config_file.py +++ b/tests/bfabric/config/test_config_file.py @@ -69,10 +69,7 @@ def test_config_file_when_auth(data_with_auth): assert len(config.environments) == 1 assert config.environments["PRODUCTION"].config.base_url == "https://example.com/" assert config.environments["PRODUCTION"].auth.login == "test-dummy" - assert ( - config.environments["PRODUCTION"].auth.password - == "00000000001111111111222222222233" - ) + assert config.environments["PRODUCTION"].auth.password == "00000000001111111111222222222233" def test_config_file_when_no_auth(data_no_auth): @@ -91,9 +88,7 @@ def test_config_file_when_multiple(data_multiple): assert config.environments["PRODUCTION"].auth is None assert config.environments["TEST"].config.base_url == "https://test.example.com/" assert config.environments["TEST"].auth.login == "test-dummy" - assert ( - config.environments["TEST"].auth.password == "00000000001111111111222222222233" - ) + assert config.environments["TEST"].auth.password == "00000000001111111111222222222233" def test_config_file_when_non_existent_default(data_no_auth): @@ -117,13 +112,8 @@ def test_get_selected_config_env_when_default(config_with_auth, monkeypatch): def test_get_selected_config(config_with_auth, mocker): - mock_get_config_env = mocker.patch.object( - ConfigFile, "get_selected_config_env", return_value="PRODUCTION" - ) - assert ( - config_with_auth.get_selected_config() - == config_with_auth.environments["PRODUCTION"] - ) + mock_get_config_env = mocker.patch.object(ConfigFile, "get_selected_config_env", return_value="PRODUCTION") + assert config_with_auth.get_selected_config() == config_with_auth.environments["PRODUCTION"] mock_get_config_env.assert_called_once_with(explicit_config_env=None) diff --git a/tests/bfabric/engine/test_engine_suds.py b/tests/bfabric/engine/test_engine_suds.py index 3bd54548..99631b3d 100644 --- a/tests/bfabric/engine/test_engine_suds.py +++ b/tests/bfabric/engine/test_engine_suds.py @@ -32,9 +32,7 @@ def mock_client(mock_suds_service): def test_read(engine_suds, mock_auth, mock_suds_service, mocker): - mocker.patch.object( - engine_suds, "_get_suds_service", return_value=mock_suds_service - ) + mocker.patch.object(engine_suds, "_get_suds_service", return_value=mock_suds_service) mock_convert = mocker.patch.object(engine_suds, "_convert_results") obj = {"field1": "value1"} @@ -52,9 +50,7 @@ def test_read(engine_suds, mock_auth, mock_suds_service, mocker): def test_save(engine_suds, mock_auth, mock_suds_service, mocker): - mocker.patch.object( - engine_suds, "_get_suds_service", return_value=mock_suds_service - ) + mocker.patch.object(engine_suds, "_get_suds_service", return_value=mock_suds_service) mock_convert = mocker.patch.object(engine_suds, "_convert_results") obj = {"field1": "value1"} @@ -70,9 +66,7 @@ def test_save(engine_suds, mock_auth, mock_suds_service, mocker): def test_save_method_not_found(engine_suds, mock_auth, mock_suds_service, mocker): - mocker.patch.object( - engine_suds, "_get_suds_service", return_value=mock_suds_service - ) + mocker.patch.object(engine_suds, "_get_suds_service", return_value=mock_suds_service) mock_suds_service.save.side_effect = MethodNotFound("save") with pytest.raises( @@ -83,9 +77,7 @@ def test_save_method_not_found(engine_suds, mock_auth, mock_suds_service, mocker def test_delete(engine_suds, mock_auth, mock_suds_service, mocker): - mocker.patch.object( - engine_suds, "_get_suds_service", return_value=mock_suds_service - ) + mocker.patch.object(engine_suds, "_get_suds_service", return_value=mock_suds_service) mock_convert = mocker.patch.object(engine_suds, "_convert_results") engine_suds.delete("sample", 123, mock_auth) @@ -96,9 +88,7 @@ def test_delete(engine_suds, mock_auth, mock_suds_service, mocker): def test_delete_empty_list(engine_suds, mock_auth, mock_suds_service, mocker): - mocker.patch.object( - engine_suds, "_get_suds_service", return_value=mock_suds_service - ) + mocker.patch.object(engine_suds, "_get_suds_service", return_value=mock_suds_service) result = engine_suds.delete("sample", [], mock_auth) @@ -109,16 +99,12 @@ def test_delete_empty_list(engine_suds, mock_auth, mock_suds_service, mocker): def test_get_suds_service(engine_suds, mock_client, mocker): - mock_client_init = mocker.patch( - "bfabric.engine.engine_suds.Client", return_value=mock_client - ) + mock_client_init = mocker.patch("bfabric.engine.engine_suds.Client", return_value=mock_client) service = engine_suds._get_suds_service("sample") assert service == mock_client.service - mock_client_init.assert_called_once_with( - "http://example.com/api/sample?wsdl", cache=None - ) + mock_client_init.assert_called_once_with("http://example.com/api/sample?wsdl", cache=None) # Test caching service2 = engine_suds._get_suds_service("sample") diff --git a/tests/bfabric/engine/test_engine_zeep.py b/tests/bfabric/engine/test_engine_zeep.py index 72def595..956d7d85 100644 --- a/tests/bfabric/engine/test_engine_zeep.py +++ b/tests/bfabric/engine/test_engine_zeep.py @@ -77,9 +77,7 @@ def test_save(engine_zeep, mock_auth, mock_zeep_client, mocker): def test_save_method_not_found(engine_zeep, mock_auth, mock_zeep_client, mocker): mocker.patch.object(engine_zeep, "_get_client", return_value=mock_zeep_client) - mock_zeep_client.service.save.side_effect = AttributeError( - "Service has no operation 'save'" - ) + mock_zeep_client.service.save.side_effect = AttributeError("Service has no operation 'save'") with pytest.raises( BfabricRequestError, @@ -111,9 +109,7 @@ def test_delete_empty_list(engine_zeep, mock_auth, mock_zeep_client, mocker): def test_get_client(engine_zeep, mocker): - mock_zeep_client = mocker.patch( - "zeep.Client", return_value=MagicMock(spec=zeep.Client) - ) + mock_zeep_client = mocker.patch("zeep.Client", return_value=MagicMock(spec=zeep.Client)) client = engine_zeep._get_client("sample") @@ -186,9 +182,7 @@ def test_zeep_query_append_skipped(): result_no_overwrite = _zeep_query_append_skipped(query_with_existing, skipped_keys) assert result_no_overwrite == query_with_existing - result_overwrite = _zeep_query_append_skipped( - query_with_existing, skipped_keys, overwrite=True - ) + result_overwrite = _zeep_query_append_skipped(query_with_existing, skipped_keys, overwrite=True) assert result_overwrite == {"key1": zeep.xsd.SkipValue, "key2": zeep.xsd.SkipValue} diff --git a/tests/bfabric/entities/core/test_entity.py b/tests/bfabric/entities/core/test_entity.py index 5ac43236..7ed028fc 100644 --- a/tests/bfabric/entities/core/test_entity.py +++ b/tests/bfabric/entities/core/test_entity.py @@ -83,9 +83,7 @@ def test_find_by_when_found(mocker, mock_client) -> None: assert len(entities) == 1 assert isinstance(entities[1], Entity) assert entities[1].data_dict == {"id": 1, "name": "Test Entity"} - mock_client.read.assert_called_once_with( - "test_endpoint", obj={"id": 1}, max_results=100 - ) + mock_client.read.assert_called_once_with("test_endpoint", obj={"id": 1}, max_results=100) def test_find_by_when_not_found(mocker, mock_client) -> None: @@ -93,9 +91,7 @@ def test_find_by_when_not_found(mocker, mock_client) -> None: mocker.patch.object(Entity, "ENDPOINT", new="test_endpoint") entities = Entity.find_by({"id": 1}, mock_client) assert len(entities) == 0 - mock_client.read.assert_called_once_with( - "test_endpoint", obj={"id": 1}, max_results=100 - ) + mock_client.read.assert_called_once_with("test_endpoint", obj={"id": 1}, max_results=100) def test_get_item(mock_entity) -> None: diff --git a/tests/bfabric/entities/core/test_has_many.py b/tests/bfabric/entities/core/test_has_many.py index 52815bac..77cbe1fe 100644 --- a/tests/bfabric/entities/core/test_has_many.py +++ b/tests/bfabric/entities/core/test_has_many.py @@ -44,9 +44,7 @@ def test_has_many_init(): def test_has_many_get(mock_client): - mock_obj = MockEntity( - data_dict={"test_field": [{"id": 1}, {"id": 2}]}, client=mock_client - ) + mock_obj = MockEntity(data_dict={"test_field": [{"id": 1}, {"id": 2}]}, client=mock_client) has_many = HasMany("MockEntity", bfabric_field="test_field") result = has_many.__get__(mock_obj) @@ -74,9 +72,7 @@ def test_has_many_get_invalid_config(): has_many = HasMany("MockEntity") mock_obj = MockEntity() - with pytest.raises( - ValueError, match="Exactly one of bfabric_field and ids_property must be set" - ): + with pytest.raises(ValueError, match="Exactly one of bfabric_field and ids_property must be set"): has_many.__get__(mock_obj) @@ -109,9 +105,7 @@ def test_has_many_proxy_polars(mocker: MockerFixture, mock_client, mock_proxy): result = mock_proxy.polars assert isinstance(result, DataFrame) - DataFrame.__init__.assert_called_once_with( - [x.data_dict for x in mock_entities.values()] - ) + DataFrame.__init__.assert_called_once_with([x.data_dict for x in mock_entities.values()]) def test_has_many_proxy_getitem(mocker: MockerFixture, mock_client, mock_proxy): diff --git a/tests/bfabric/entities/core/test_has_one.py b/tests/bfabric/entities/core/test_has_one.py index c6b3da0b..aa212c8f 100644 --- a/tests/bfabric/entities/core/test_has_one.py +++ b/tests/bfabric/entities/core/test_has_one.py @@ -37,9 +37,7 @@ def test_init(): def test_get_when_cache_not_exists(mocker, mock_client): mock_obj = MockEntity(data_dict={"test_field": {"id": 1}}, client=mock_client) has_one = HasOne("MockEntity", bfabric_field="test_field") - mock_load_entity = mocker.patch.object( - HasOne, "_load_entity", return_value="mock_entity" - ) + mock_load_entity = mocker.patch.object(HasOne, "_load_entity", return_value="mock_entity") result = has_one.__get__(mock_obj) assert result == "mock_entity" mock_load_entity.assert_called_once_with(obj=mock_obj) @@ -49,9 +47,7 @@ def test_get_when_cache_exists(mocker, mock_client): mock_obj = MockEntity(data_dict={"test_field": {"id": 1}}, client=mock_client) has_one = HasOne("MockEntity", bfabric_field="test_field") mock_obj._HasOne__test_field_cache = "mock_entity" - mock_load_entity = mocker.patch.object( - HasOne, "_load_entity", return_value="mock_entity" - ) + mock_load_entity = mocker.patch.object(HasOne, "_load_entity", return_value="mock_entity") result = has_one.__get__(mock_obj) assert result == "mock_entity" mock_load_entity.assert_not_called() diff --git a/tests/bfabric/entities/test_dataset.py b/tests/bfabric/entities/test_dataset.py index f88a3fa5..19a6059f 100644 --- a/tests/bfabric/entities/test_dataset.py +++ b/tests/bfabric/entities/test_dataset.py @@ -71,17 +71,11 @@ def test_write_csv(mocker: MockFixture, mock_dataset: Dataset) -> None: def test_repr(mock_empty_dataset: Dataset) -> None: - assert ( - repr(mock_empty_dataset) - == "Dataset({'id': 1234, 'attribute': [], 'item': []}, client=None)" - ) + assert repr(mock_empty_dataset) == "Dataset({'id': 1234, 'attribute': [], 'item': []}, client=None)" def test_str(mock_empty_dataset: Dataset) -> None: - assert ( - str(mock_empty_dataset) - == "Dataset({'id': 1234, 'attribute': [], 'item': []}, client=None)" - ) + assert str(mock_empty_dataset) == "Dataset({'id': 1234, 'attribute': [], 'item': []}, client=None)" if __name__ == "__main__": diff --git a/tests/bfabric/entities/test_workunit.py b/tests/bfabric/entities/test_workunit.py index dfa9046c..c0f82f26 100644 --- a/tests/bfabric/entities/test_workunit.py +++ b/tests/bfabric/entities/test_workunit.py @@ -101,13 +101,8 @@ def test_store_output_folder(mocker, mock_workunit) -> None: "name": "my app", }.__getitem__ mocker.patch.object(mock_workunit, "application", mock_application) - mocker.patch.object( - Workunit, "container", mocker.PropertyMock(return_value=mocker.MagicMock(id=12)) - ) - assert ( - Path("xyz12/bfabric/tech/my_app/2024/2024-01/2024-01-02/workunit_30000") - == mock_workunit.store_output_folder - ) + mocker.patch.object(Workunit, "container", mocker.PropertyMock(return_value=mocker.MagicMock(id=12))) + assert Path("xyz12/bfabric/tech/my_app/2024/2024-01/2024-01-02/workunit_30000") == mock_workunit.store_output_folder def test_repr() -> None: diff --git a/tests/bfabric/results/test_result_container.py b/tests/bfabric/results/test_result_container.py index d115265b..6c24aa1e 100644 --- a/tests/bfabric/results/test_result_container.py +++ b/tests/bfabric/results/test_result_container.py @@ -10,9 +10,7 @@ class BfabricTestResultContainer(unittest.TestCase): def setUp(self): self.res1 = ResultContainer([1, 2, 3], total_pages_api=1) self.res2 = ResultContainer([4, 5], total_pages_api=1) - self.res_with_empty = ResultContainer( - [{"a": None, "b": 1, "c": []}, {"a": 2, "b": 3, "c": None}] - ) + self.res_with_empty = ResultContainer([{"a": None, "b": 1, "c": []}, {"a": 2, "b": 3, "c": None}]) def test_str(self): self.assertEqual("[1, 2, 3]", str(self.res1)) @@ -56,9 +54,7 @@ def test_assert_success_when_error(self): self.res1.errors.append("MockedError") with self.assertRaises(RuntimeError) as error: self.res1.assert_success() - self.assertEqual( - "('Query was not successful', ['MockedError'])", str(error.exception) - ) + self.assertEqual("('Query was not successful', ['MockedError'])", str(error.exception)) def test_extend_when_same_lengths(self): res1 = ResultContainer([{"a": 1}, {"a": 2}], total_pages_api=5) @@ -79,31 +75,23 @@ def test_extend_when_different_lengths(self): self.assertEqual(203, len(res3)) self.assertEqual(list(range(200, 400)) + [1, 2, 3], res3.results) self.assertEqual(2, res3.total_pages_api) - self.assertIn( - "Results observed with different total pages counts: 2 != 1", str(error) - ) + self.assertIn("Results observed with different total pages counts: 2 != 1", str(error)) def test_to_list_dict_when_not_drop_empty(self): expected = [{"a": None, "b": 1, "c": []}, {"a": 2, "b": 3, "c": None}] with self.subTest(case="default"): self.assertListEqual(expected, self.res_with_empty.to_list_dict()) with self.subTest(case="explicit"): - self.assertListEqual( - expected, self.res_with_empty.to_list_dict(drop_empty=False) - ) + self.assertListEqual(expected, self.res_with_empty.to_list_dict(drop_empty=False)) def test_to_list_dict_when_drop_empty(self): expected = [{"b": 1}, {"a": 2, "b": 3}] - self.assertListEqual( - expected, self.res_with_empty.to_list_dict(drop_empty=True) - ) + self.assertListEqual(expected, self.res_with_empty.to_list_dict(drop_empty=True)) def test_to_polars(self): res = ResultContainer([{"a": 1, "b": 2}, {"a": 3, "b": 4}]) df = res.to_polars() - polars.testing.assert_frame_equal( - polars.DataFrame({"a": [1, 3], "b": [2, 4]}), df - ) + polars.testing.assert_frame_equal(polars.DataFrame({"a": [1, 3], "b": [2, 4]}), df) if __name__ == "__main__": diff --git a/tests/bfabric/test_bfabric.py b/tests/bfabric/test_bfabric.py index 2c29ed4a..1c375930 100644 --- a/tests/bfabric/test_bfabric.py +++ b/tests/bfabric/test_bfabric.py @@ -55,9 +55,7 @@ def test_from_config_when_explicit_auth(mocker): assert isinstance(client, Bfabric) assert client.config == mock_config assert client.auth == mock_auth - mock_get_system_auth.assert_called_once_with( - config_env="TestingEnv", config_path=None - ) + mock_get_system_auth.assert_called_once_with(config_env="TestingEnv", config_path=None) def test_from_config_when_none_auth(mocker): @@ -74,9 +72,7 @@ def test_from_config_when_none_auth(mocker): assert client.config == mock_config with pytest.raises(ValueError, match="Authentication not available"): _ = client.auth - mock_get_system_auth.assert_called_once_with( - config_env="TestingEnv", config_path=None - ) + mock_get_system_auth.assert_called_once_with(config_env="TestingEnv", config_path=None) def test_from_config_when_engine_suds(mocker): @@ -130,9 +126,7 @@ def test_auth_when_missing(bfabric_instance): def test_auth_when_provided(mock_config, mock_engine): mock_auth = MagicMock(name="mock_auth") - bfabric_instance = Bfabric( - config=mock_config, auth=mock_auth, engine=BfabricAPIEngineType.SUDS - ) + bfabric_instance = Bfabric(config=mock_config, auth=mock_auth, engine=BfabricAPIEngineType.SUDS) assert bfabric_instance.auth == mock_auth @@ -164,9 +158,7 @@ def test_read_when_no_pages_available_and_check(bfabric_instance, mocker): bfabric_instance._auth = mock_auth mock_engine = mocker.patch.object(bfabric_instance, "_engine") - mock_result = MagicMock( - name="mock_result", total_pages_api=0, assert_success=MagicMock() - ) + mock_result = MagicMock(name="mock_result", total_pages_api=0, assert_success=MagicMock()) mock_engine.read.return_value = mock_result result = bfabric_instance.read(endpoint="mock_endpoint", obj="mock_obj") @@ -187,9 +179,7 @@ def test_read_when_pages_available_and_check(bfabric_instance, mocker): mock_auth = MagicMock(name="mock_auth") bfabric_instance._auth = mock_auth - mock_compute_requested_pages = mocker.patch( - "bfabric.bfabric.compute_requested_pages" - ) + mock_compute_requested_pages = mocker.patch("bfabric.bfabric.compute_requested_pages") mock_engine = mocker.patch.object(bfabric_instance, "_engine") mock_page_results = [ @@ -205,9 +195,7 @@ def test_read_when_pages_available_and_check(bfabric_instance, mocker): mock_page_results[1].__getitem__.side_effect = lambda i: [6, 7, 8, 9, 10][i] mock_page_results[2].__getitem__.side_effect = lambda i: [11, 12, 13, 14, 15][i] - mock_engine.read.side_effect = lambda **kwargs: mock_page_results[ - kwargs["page"] - 1 - ] + mock_engine.read.side_effect = lambda **kwargs: mock_page_results[kwargs["page"] - 1] mock_compute_requested_pages.return_value = ([1, 2], 4) result = bfabric_instance.read(endpoint="mock_endpoint", obj="mock_obj") @@ -301,9 +289,7 @@ def test_delete_when_auth_and_check_false(bfabric_instance, mocker): assert result == mock_engine.delete.return_value method_assert_success.assert_not_called() - mock_engine.delete.assert_called_once_with( - endpoint="test_endpoint", id=10, auth=mock_auth - ) + mock_engine.delete.assert_called_once_with(endpoint="test_endpoint", id=10, auth=mock_auth) def test_delete_when_auth_and_check_true(bfabric_instance, mocker): @@ -318,9 +304,7 @@ def test_delete_when_auth_and_check_true(bfabric_instance, mocker): assert result == mock_engine.delete.return_value method_assert_success.assert_called_once() - mock_engine.delete.assert_called_once_with( - endpoint="test_endpoint", id=10, auth=mock_auth - ) + mock_engine.delete.assert_called_once_with(endpoint="test_endpoint", id=10, auth=mock_auth) def test_exists_when_true(bfabric_instance, mocker): @@ -363,9 +347,7 @@ def test_exists_when_false(bfabric_instance, mocker): mock_read = mocker.patch.object(Bfabric, "read") mock_read.return_value.__len__.return_value = 0 - assert not bfabric_instance.exists( - endpoint="test_endpoint", key="key", value="value" - ) + assert not bfabric_instance.exists(endpoint="test_endpoint", key="key", value="value") mock_read.assert_called_once_with( endpoint="test_endpoint", @@ -401,18 +383,14 @@ def test_upload_resource(bfabric_instance, mocker): def test_get_version_message(mock_config, bfabric_instance): mock_config.base_url = "dummy_url" line1, line2 = bfabric_instance._get_version_message() - pattern = ( - r"bfabricPy v\d+\.\d+\.\d+ \(EngineSUDS, dummy_url, U=None, PY=\d\.\d+\.\d+\)" - ) + pattern = r"bfabricPy v\d+\.\d+\.\d+ \(EngineSUDS, dummy_url, U=None, PY=\d\.\d+\.\d+\)" assert re.match(pattern, line1) year = datetime.datetime.now().year assert line2 == f"Copyright (C) 2014-{year} Functional Genomics Center Zurich" def test_log_version_message(mocker, bfabric_instance): - mocker.patch.object( - Bfabric, "_get_version_message", return_value=("line1", "line2") - ) + mocker.patch.object(Bfabric, "_get_version_message", return_value=("line1", "line2")) mock_logger = mocker.patch("bfabric.bfabric.logger") bfabric_instance._log_version_message() assert mock_logger.info.mock_calls == [call("line1"), call("line2")] diff --git a/tests/bfabric/test_bfabric_config.py b/tests/bfabric/test_bfabric_config.py index ec1dfa55..bb1ddd70 100644 --- a/tests/bfabric/test_bfabric_config.py +++ b/tests/bfabric/test_bfabric_config.py @@ -19,9 +19,7 @@ def test_read_yml_bypath_all_fields(example_config_path: Path) -> None: "Proteomics/DUCK_666": 12, } - job_notification_emails_ground_truth = ( - "john.snow@fgcz.uzh.ch billy.the.kid@fgcz.ethz.ch" - ) + job_notification_emails_ground_truth = "john.snow@fgcz.uzh.ch billy.the.kid@fgcz.ethz.ch" assert auth.login == "my_epic_test_login" assert auth.password == "012345678901234567890123456789ff" @@ -36,11 +34,7 @@ def test_read_yml_when_empty_optional(example_config_path: Path, logot: Logot) - assert config.base_url == "https://standby-server.uzh.ch/mystandby" assert config.application_ids == {} assert config.job_notification_emails == "" - logot.assert_logged( - logged.debug( - f"Reading configuration from: {str(example_config_path.absolute())}" - ) - ) + logot.assert_logged(logged.debug(f"Reading configuration from: {str(example_config_path.absolute())}")) if __name__ == "__main__": diff --git a/tests/bfabric/wrapper_creator/test_slurm.py b/tests/bfabric/wrapper_creator/test_slurm.py index cddbfefb..38f793f5 100644 --- a/tests/bfabric/wrapper_creator/test_slurm.py +++ b/tests/bfabric/wrapper_creator/test_slurm.py @@ -15,9 +15,7 @@ def mock_slurm() -> SLURM: @pytest.mark.parametrize("path", ["/tmp/hello/world.txt", Path("/tmp/hello/world.txt")]) -def test_sbatch_when_success( - mocker: MockerFixture, mock_slurm: SLURM, path: Path | str -) -> None: +def test_sbatch_when_success(mocker: MockerFixture, mock_slurm: SLURM, path: Path | str) -> None: mock_is_file = mocker.patch.object(Path, "is_file", return_value=True) mocker.patch("os.environ", new={"x": "y"}) mock_run = mocker.patch( @@ -38,9 +36,7 @@ def test_sbatch_when_success( assert mock_is_file.call_count == 2 -def test_sbatch_when_script_not_exists( - mocker: MockerFixture, mock_slurm: SLURM, logot: Logot -) -> None: +def test_sbatch_when_script_not_exists(mocker: MockerFixture, mock_slurm: SLURM, logot: Logot) -> None: mocker.patch("bfabric.wrapper_creator.slurm.Path", side_effect=lambda x: x) mock_script = mocker.MagicMock(name="script", is_file=lambda: False) result = mock_slurm.sbatch(script=mock_script) @@ -48,14 +44,10 @@ def test_sbatch_when_script_not_exists( logot.assert_logged(logged.error(f"Script not found: {mock_script}")) -def test_sbatch_when_sbatch_not_exists( - mocker: MockerFixture, mock_slurm: SLURM, logot: Logot -) -> None: +def test_sbatch_when_sbatch_not_exists(mocker: MockerFixture, mock_slurm: SLURM, logot: Logot) -> None: mocker.patch("bfabric.wrapper_creator.slurm.Path", side_effect=lambda x: x) mock_script = mocker.MagicMock(name="script", is_file=lambda: True) - mock_sbatch = mocker.patch.object( - mock_slurm, "_sbatch_bin", mocker.MagicMock(is_file=lambda: False) - ) + mock_sbatch = mocker.patch.object(mock_slurm, "_sbatch_bin", mocker.MagicMock(is_file=lambda: False)) result = mock_slurm.sbatch(script=mock_script) assert result is None logot.assert_logged(logged.error(f"sbatch binary not found: {mock_sbatch}")) diff --git a/tests/app_runner/__init__.py b/tests/bfabric_app_runner/__init__.py similarity index 100% rename from tests/app_runner/__init__.py rename to tests/bfabric_app_runner/__init__.py diff --git a/tests/app_runner/app_runner/__init__.py b/tests/bfabric_app_runner/app_runner/__init__.py similarity index 100% rename from tests/app_runner/app_runner/__init__.py rename to tests/bfabric_app_runner/app_runner/__init__.py diff --git a/tests/bfabric_app_runner/app_runner/test_prepare_file_spec.py b/tests/bfabric_app_runner/app_runner/test_prepare_file_spec.py new file mode 100644 index 00000000..55495175 --- /dev/null +++ b/tests/bfabric_app_runner/app_runner/test_prepare_file_spec.py @@ -0,0 +1,190 @@ +from pathlib import Path +from shutil import SameFileError +from subprocess import CalledProcessError + +import pytest +from logot import Logot, logged + +from bfabric_app_runner.input_preparation.prepare_file_spec import ( + prepare_file_spec, + _operation_copy_rsync, + _operation_copy_cp, + _operation_link_symbolic, + _operation_copy_scp, +) +from bfabric_app_runner.specs.inputs.file_copy_spec import FileSpec +from bfabric import Bfabric + + +@pytest.fixture +def mock_subprocess(mocker): + return mocker.patch("subprocess.run") + + +@pytest.fixture +def mock_shutil_copyfile(mocker): + return mocker.patch("shutil.copyfile") + + +@pytest.fixture +def mock_client(mocker): + return mocker.MagicMock(spec=Bfabric) + + +@pytest.fixture +def operation_copy_rsync(mocker): + return mocker.patch( + "bfabric_app_runner.input_preparation.prepare_file_spec._operation_copy_rsync", return_value=False + ) + + +@pytest.fixture +def operation_copy_scp(mocker): + return mocker.patch( + "bfabric_app_runner.input_preparation.prepare_file_spec._operation_copy_scp", return_value=False + ) + + +@pytest.fixture +def operation_copy_cp(mocker): + return mocker.patch("bfabric_app_runner.input_preparation.prepare_file_spec._operation_copy_cp", return_value=False) + + +@pytest.fixture +def operation_link_symbolic(mocker): + return mocker.patch( + "bfabric_app_runner.input_preparation.prepare_file_spec._operation_link_symbolic", return_value=False + ) + + +# TODO should absolute path for filename be mandatory in the future? -> this could also have unwanted side effects + + +def test_prepare_local_copy_when_rsync_success(mock_client, operation_copy_rsync) -> None: + spec = FileSpec.model_validate({"source": {"local": "/source.txt"}, "filename": "destination.txt"}) + operation_copy_rsync.return_value = True + prepare_file_spec(spec=spec, client=mock_client, working_dir=Path("../integration/working_dir"), ssh_user=None) + operation_copy_rsync.assert_called_once_with(spec, Path("../integration/working_dir") / "destination.txt", None) + + +def test_prepare_local_copy_when_fallback_success(mock_client, operation_copy_rsync, operation_copy_cp) -> None: + spec = FileSpec.model_validate({"source": {"local": "/source.txt"}, "filename": "destination.txt"}) + operation_copy_rsync.return_value = False + operation_copy_cp.return_value = True + prepare_file_spec(spec=spec, client=mock_client, working_dir=Path("../integration/working_dir"), ssh_user=None) + operation_copy_rsync.assert_called_once_with(spec, Path("../integration/working_dir") / "destination.txt", None) + operation_copy_cp.assert_called_once_with(spec, Path("../integration/working_dir") / "destination.txt") + + +def test_prepare_local_link_when_success(mock_client, operation_link_symbolic): + spec = FileSpec.model_validate({"source": {"local": "/source.txt"}, "filename": "destination.txt", "link": True}) + operation_link_symbolic.return_value = True + prepare_file_spec(spec=spec, client=mock_client, working_dir=Path("../integration/working_dir"), ssh_user=None) + operation_link_symbolic.assert_called_once_with(spec, Path("../integration/working_dir") / "destination.txt") + + +def test_prepare_remote_copy_when_rsync_success(mock_client, operation_copy_rsync): + spec = FileSpec.model_validate( + {"source": {"ssh": {"host": "host", "path": "/source.txt"}}, "filename": "destination.txt"} + ) + operation_copy_rsync.return_value = True + prepare_file_spec(spec=spec, client=mock_client, working_dir=Path("../integration/working_dir"), ssh_user="user") + operation_copy_rsync.assert_called_once_with(spec, Path("../integration/working_dir") / "destination.txt", "user") + + +def test_prepare_remote_copy_when_fallback_success(mock_client, operation_copy_rsync, operation_copy_scp): + spec = FileSpec.model_validate( + {"source": {"ssh": {"host": "host", "path": "/source.txt"}}, "filename": "destination.txt"} + ) + operation_copy_rsync.return_value = False + operation_copy_scp.return_value = True + prepare_file_spec(spec=spec, client=mock_client, working_dir=Path("../integration/working_dir"), ssh_user="user") + operation_copy_rsync.assert_called_once_with(spec, Path("../integration/working_dir") / "destination.txt", "user") + operation_copy_scp.assert_called_once_with(spec, Path("../integration/working_dir") / "destination.txt", "user") + + +def test_operation_copy_rsync_local(mock_subprocess, logot: Logot): + spec = FileSpec.model_validate({"source": {"local": "/source.txt"}, "filename": "destination.txt"}) + mock_subprocess.return_value.returncode = 0 + result = _operation_copy_rsync(spec=spec, output_path=Path("mock_output.txt"), ssh_user=None) + mock_subprocess.assert_called_once_with(["rsync", "-Pav", "/source.txt", "mock_output.txt"], check=False) + logot.assert_logged(logged.info("rsync -Pav /source.txt mock_output.txt")) + assert result + + +def test_operation_copy_rsync_ssh_default(mock_subprocess, logot: Logot): + spec = FileSpec.model_validate( + {"source": {"ssh": {"host": "host", "path": "/source.txt"}}, "filename": "destination.txt"} + ) + mock_subprocess.return_value.returncode = 0 + result = _operation_copy_rsync(spec=spec, output_path=Path("mock_output.txt"), ssh_user=None) + mock_subprocess.assert_called_once_with(["rsync", "-Pav", "host:/source.txt", "mock_output.txt"], check=False) + logot.assert_logged(logged.info("rsync -Pav host:/source.txt mock_output.txt")) + assert result + + +def test_operation_copy_rsync_ssh_custom_user(mock_subprocess, logot: Logot): + spec = FileSpec.model_validate( + {"source": {"ssh": {"host": "host", "path": "/source.txt"}}, "filename": "destination.txt"} + ) + mock_subprocess.return_value.returncode = 0 + result = _operation_copy_rsync(spec=spec, output_path=Path("mock_output.txt"), ssh_user="user") + mock_subprocess.assert_called_once_with(["rsync", "-Pav", "user@host:/source.txt", "mock_output.txt"], check=False) + logot.assert_logged(logged.info("rsync -Pav user@host:/source.txt mock_output.txt")) + assert result + + +def test_operation_copy_scp(mocker): + mock_scp = mocker.patch("bfabric_app_runner.input_preparation.prepare_file_spec.scp") + spec = FileSpec.model_validate( + {"source": {"ssh": {"host": "host", "path": "/source.txt"}}, "filename": "destination.txt"} + ) + result = _operation_copy_scp(spec=spec, output_path=Path("mock_output.txt"), ssh_user="user") + mock_scp.assert_called_once_with(source="host:/source.txt", target=Path("mock_output.txt"), user="user") + assert result + + +def test_operation_copy_scp_when_error(mocker): + mock_scp = mocker.patch( + "bfabric_app_runner.input_preparation.prepare_file_spec.scp", side_effect=CalledProcessError(1, "scp") + ) + spec = FileSpec.model_validate( + {"source": {"ssh": {"host": "host", "path": "/source.txt"}}, "filename": "destination.txt"} + ) + result = _operation_copy_scp(spec=spec, output_path=Path("mock_output.txt"), ssh_user="user") + mock_scp.assert_called_once_with(source="host:/source.txt", target=Path("mock_output.txt"), user="user") + assert not result + + +def test_operation_copy_cp(mock_shutil_copyfile, logot: Logot): + spec = FileSpec.model_validate({"source": {"local": "/source.txt"}, "filename": "destination.txt"}) + result = _operation_copy_cp(spec=spec, output_path=Path("mock_output.txt")) + mock_shutil_copyfile.assert_called_once_with("/source.txt", "mock_output.txt") + logot.assert_logged(logged.info("cp /source.txt mock_output.txt")) + assert result + + +def test_operation_copy_cp_when_error(mock_shutil_copyfile, logot: Logot): + mock_shutil_copyfile.side_effect = SameFileError + spec = FileSpec.model_validate({"source": {"local": "/source.txt"}, "filename": "destination.txt"}) + result = _operation_copy_cp(spec=spec, output_path=Path("mock_output.txt")) + mock_shutil_copyfile.assert_called_once_with("/source.txt", "mock_output.txt") + logot.assert_logged(logged.info("cp /source.txt mock_output.txt")) + assert not result + + +@pytest.mark.parametrize( + "source,dest,expected_target", + [ + ("/E/source.txt", "/E/dir/destination.txt", "../source.txt"), + ("/X/source.txt", "/E/dir/destination.txt", "../../X/source.txt"), + ("/work/source.txt", "/work/destination.txt", "source.txt"), + ], +) +def test_operation_link_symbolic(mock_subprocess, logot: Logot, source, dest, expected_target): + spec = FileSpec.model_validate({"source": {"local": source}, "filename": "IGNORED", "link": True}) + mock_subprocess.return_value.returncode = 0 + result = _operation_link_symbolic(spec=spec, output_path=Path(dest)) + mock_subprocess.assert_called_once_with(["ln", "-s", expected_target, str(dest)], check=False) + logot.assert_logged(logged.info(f"ln -s {expected_target} {dest}")) + assert result diff --git a/tests/app_runner/app_runner/test_runner.py b/tests/bfabric_app_runner/app_runner/test_runner.py similarity index 86% rename from tests/app_runner/app_runner/test_runner.py rename to tests/bfabric_app_runner/app_runner/test_runner.py index 92ebbe07..d91e73a6 100644 --- a/tests/app_runner/app_runner/test_runner.py +++ b/tests/bfabric_app_runner/app_runner/test_runner.py @@ -4,7 +4,7 @@ import yaml from pydantic import BaseModel -from app_runner.app_runner.runner import Runner, run_app +from bfabric_app_runner.app_runner.runner import Runner, run_app # Mock classes to represent dependencies @@ -87,14 +87,12 @@ def test_runner_run_dispatch(mocker, mock_app_version, mock_bfabric, tmp_path): runner.run_dispatch(workunit_ref, work_dir) - mock_run.assert_called_once_with( - ["mock-command", str(workunit_ref), str(work_dir)], check=True - ) + mock_run.assert_called_once_with(["mock-command", str(workunit_ref), str(work_dir)], check=True) def test_runner_run_prepare_input(mocker, mock_app_version, mock_bfabric, tmp_path): """Test Runner.run_prepare_input method""" - mock_prepare = mocker.patch("app_runner.app_runner.runner.prepare_folder") + mock_prepare = mocker.patch("bfabric_app_runner.app_runner.runner.prepare_folder") runner = Runner(spec=mock_app_version, client=mock_bfabric, ssh_user="test_user") chunk_dir = tmp_path / "chunk" chunk_dir.mkdir() @@ -122,9 +120,7 @@ def test_runner_run_process(mocker, mock_app_version, mock_bfabric, tmp_path): mock_run.assert_called_once_with(["mock-command", str(chunk_dir)], check=True) -def test_run_app_full_workflow( - mocker, mock_app_version, mock_bfabric, mock_workunit_definition, setup_work_dir -): +def test_run_app_full_workflow(mocker, mock_app_version, mock_bfabric, mock_workunit_definition, setup_work_dir): """Test complete run_app workflow""" # Setup mocks mocker.patch( @@ -132,8 +128,8 @@ def test_run_app_full_workflow( return_value=mock_workunit_definition, ) mock_run = mocker.patch("subprocess.run") - mock_prepare = mocker.patch("app_runner.app_runner.runner.prepare_folder") - mock_register = mocker.patch("app_runner.app_runner.runner.register_outputs") + mock_prepare = mocker.patch("bfabric_app_runner.app_runner.runner.prepare_folder") + mock_register = mocker.patch("bfabric_app_runner.app_runner.runner.register_outputs") # Run the app run_app( @@ -166,8 +162,8 @@ def test_run_app_read_only_mode( return_value=mock_workunit_definition, ) mocker.patch("subprocess.run") - mocker.patch("app_runner.app_runner.runner.prepare_folder") - mocker.patch("app_runner.app_runner.runner.register_outputs") + mocker.patch("bfabric_app_runner.app_runner.runner.prepare_folder") + mocker.patch("bfabric_app_runner.app_runner.runner.register_outputs") run_app( app_spec=mock_app_version, @@ -194,8 +190,8 @@ def test_run_app_with_path_workunit_ref( return_value=mock_workunit_definition, ) mocker.patch("subprocess.run") - mocker.patch("app_runner.app_runner.runner.prepare_folder") - mocker.patch("app_runner.app_runner.runner.register_outputs") + mocker.patch("bfabric_app_runner.app_runner.runner.prepare_folder") + mocker.patch("bfabric_app_runner.app_runner.runner.register_outputs") run_app( app_spec=mock_app_version, diff --git a/tests/app_runner/integration/__init__.py b/tests/bfabric_app_runner/integration/__init__.py similarity index 100% rename from tests/app_runner/integration/__init__.py rename to tests/bfabric_app_runner/integration/__init__.py diff --git a/tests/app_runner/integration/test_cli_commands.py b/tests/bfabric_app_runner/integration/test_cli_commands.py similarity index 100% rename from tests/app_runner/integration/test_cli_commands.py rename to tests/bfabric_app_runner/integration/test_cli_commands.py diff --git a/tests/app_runner/specs/__init__.py b/tests/bfabric_app_runner/specs/__init__.py similarity index 100% rename from tests/app_runner/specs/__init__.py rename to tests/bfabric_app_runner/specs/__init__.py diff --git a/tests/app_runner/specs/app/__init__.py b/tests/bfabric_app_runner/specs/app/__init__.py similarity index 100% rename from tests/app_runner/specs/app/__init__.py rename to tests/bfabric_app_runner/specs/app/__init__.py diff --git a/tests/app_runner/specs/app/test_commands_spec.py b/tests/bfabric_app_runner/specs/app/test_commands_spec.py similarity index 94% rename from tests/app_runner/specs/app/test_commands_spec.py rename to tests/bfabric_app_runner/specs/app/test_commands_spec.py index e0eec37c..00bd556b 100644 --- a/tests/app_runner/specs/app/test_commands_spec.py +++ b/tests/bfabric_app_runner/specs/app/test_commands_spec.py @@ -1,7 +1,7 @@ import os from pathlib import Path -from app_runner.specs.app.commands_spec import MountOptions, CommandDocker, CommandShell +from bfabric_app_runner.specs.app.commands_spec import MountOptions, CommandDocker, CommandShell def test_mount_options_default_behavior(tmp_path): @@ -81,9 +81,7 @@ def test_command_shell_with_quotes(): def test_command_shell_complex_command(): """Test complex shell command with multiple arguments and quotes""" - cmd = CommandShell( - command="python3 -c \"import sys; print('Hello from Python')\" --verbose" - ) + cmd = CommandShell(command="python3 -c \"import sys; print('Hello from Python')\" --verbose") result = cmd.to_shell() assert result == [ "python3", @@ -129,9 +127,7 @@ def test_command_docker_with_options(tmp_path): mac_address="00:00:00:00:00:00", custom_args=["--network=host"], hostname="myhost", - mounts=MountOptions( - share_bfabric_config=False - ), # Disable bfabric mount for simpler testing + mounts=MountOptions(share_bfabric_config=False), # Disable bfabric mount for simpler testing ) work_dir = tmp_path / "work" diff --git a/tests/app_runner/specs/app/test_versions.py b/tests/bfabric_app_runner/specs/app/test_versions.py similarity index 93% rename from tests/app_runner/specs/app/test_versions.py rename to tests/bfabric_app_runner/specs/app/test_versions.py index d31b580a..b6ed01d9 100644 --- a/tests/app_runner/specs/app/test_versions.py +++ b/tests/bfabric_app_runner/specs/app/test_versions.py @@ -2,7 +2,7 @@ import pytest -from app_runner.specs.app.app_spec import AppSpec +from bfabric_app_runner.specs.app.app_spec import AppSpec @pytest.fixture() diff --git a/tests/app_runner/specs/app/test_versions.yml b/tests/bfabric_app_runner/specs/app/test_versions.yml similarity index 100% rename from tests/app_runner/specs/app/test_versions.yml rename to tests/bfabric_app_runner/specs/app/test_versions.yml diff --git a/tests/app_runner/specs/test_app_spec.py b/tests/bfabric_app_runner/specs/test_app_spec.py similarity index 88% rename from tests/app_runner/specs/test_app_spec.py rename to tests/bfabric_app_runner/specs/test_app_spec.py index 198dda7d..5c6c5c42 100644 --- a/tests/app_runner/specs/test_app_spec.py +++ b/tests/bfabric_app_runner/specs/test_app_spec.py @@ -1,14 +1,14 @@ import pytest import yaml -from app_runner.specs.app.app_version import AppVersion -from app_runner.specs.app.commands_spec import ( +from bfabric_app_runner.specs.app.app_version import AppVersion +from bfabric_app_runner.specs.app.commands_spec import ( CommandShell, CommandDocker, MountOptions, CommandsSpec, ) -from app_runner.specs.submitter_ref import SubmitterRef +from bfabric_app_runner.specs.submitter_ref import SubmitterRef @pytest.fixture() diff --git a/tests/app_runner/specs/test_config_interpolation.py b/tests/bfabric_app_runner/specs/test_config_interpolation.py similarity index 93% rename from tests/app_runner/specs/test_config_interpolation.py rename to tests/bfabric_app_runner/specs/test_config_interpolation.py index a9665590..f501e5d0 100644 --- a/tests/app_runner/specs/test_config_interpolation.py +++ b/tests/bfabric_app_runner/specs/test_config_interpolation.py @@ -2,7 +2,7 @@ from mako.exceptions import MakoException from pydantic import ValidationError -from app_runner.specs.config_interpolation import ( +from bfabric_app_runner.specs.config_interpolation import ( Variables, VariablesApp, interpolate_config_strings, @@ -11,9 +11,7 @@ @pytest.fixture def basic_variables(): - return Variables( - app=VariablesApp(id="test-app", name="Test Application", version="1.0.0") - ) + return Variables(app=VariablesApp(id="test-app", name="Test Application", version="1.0.0")) def test_variables_app_model(): @@ -93,9 +91,7 @@ def test_invalid_template_syntax(basic_variables): def test_missing_variable(basic_variables): template = "${app.missing}" - with pytest.raises( - Exception - ): # Could be more specific depending on Mako's behavior + with pytest.raises(Exception): # Could be more specific depending on Mako's behavior interpolate_config_strings(template, basic_variables) diff --git a/tests/app_runner/specs/test_inputs_spec.py b/tests/bfabric_app_runner/specs/test_inputs_spec.py similarity index 79% rename from tests/app_runner/specs/test_inputs_spec.py rename to tests/bfabric_app_runner/specs/test_inputs_spec.py index 6a78d514..9a049224 100644 --- a/tests/app_runner/specs/test_inputs_spec.py +++ b/tests/bfabric_app_runner/specs/test_inputs_spec.py @@ -1,9 +1,9 @@ import pytest import yaml -from app_runner.specs.inputs_spec import InputsSpec -from app_runner.specs.inputs.bfabric_dataset_spec import BfabricDatasetSpec -from app_runner.specs.inputs.bfabric_resource_spec import BfabricResourceSpec +from bfabric_app_runner.specs.inputs_spec import InputsSpec +from bfabric_app_runner.specs.inputs.bfabric_dataset_spec import BfabricDatasetSpec +from bfabric_app_runner.specs.inputs.bfabric_resource_spec import BfabricResourceSpec @pytest.fixture() diff --git a/tests/app_runner/specs/test_outputs_spec.py b/tests/bfabric_app_runner/specs/test_outputs_spec.py similarity index 92% rename from tests/app_runner/specs/test_outputs_spec.py rename to tests/bfabric_app_runner/specs/test_outputs_spec.py index 6846adb4..ade870ea 100644 --- a/tests/app_runner/specs/test_outputs_spec.py +++ b/tests/bfabric_app_runner/specs/test_outputs_spec.py @@ -1,7 +1,7 @@ import pytest import yaml -from app_runner.specs.outputs_spec import OutputsSpec, CopyResourceSpec, SaveDatasetSpec +from bfabric_app_runner.specs.outputs_spec import OutputsSpec, CopyResourceSpec, SaveDatasetSpec @pytest.fixture() diff --git a/tests/bfabric_scripts/test_bfabric_list_not_existing_storage_directories.py b/tests/bfabric_scripts/test_bfabric_list_not_existing_storage_directories.py index c32d9d4a..44f2558b 100644 --- a/tests/bfabric_scripts/test_bfabric_list_not_existing_storage_directories.py +++ b/tests/bfabric_scripts/test_bfabric_list_not_existing_storage_directories.py @@ -21,9 +21,7 @@ def test_list_not_existing_storage_directories( # mock a client client = mocker.MagicMock() - client.read.return_value = ResultContainer( - [{"id": 3000}, {"id": 3050}, {"id": 3100}, {"id": 3200}, {"id": 3300}] - ) + client.read.return_value = ResultContainer([{"id": 3000}, {"id": 3050}, {"id": 3100}, {"id": 3200}, {"id": 3300}]) # call the function list_not_existing_storage_dirs(client, tmp_path, cache_path=tmp_path / "cache.json") diff --git a/tests/bfabric_scripts/test_bfabric_save_csv2dataset.py b/tests/bfabric_scripts/test_bfabric_save_csv2dataset.py index af55cf4d..1198112b 100644 --- a/tests/bfabric_scripts/test_bfabric_save_csv2dataset.py +++ b/tests/bfabric_scripts/test_bfabric_save_csv2dataset.py @@ -55,9 +55,7 @@ def test_check_for_invalid_characters_empty_dataframe(): def test_check_for_invalid_characters_non_string_columns(): - data = pl.DataFrame( - {"col1": [1, 2, 3], "col2": [4.5, 5.6, 6.7], "col3": ["abc", "def", "ghi"]} - ) + data = pl.DataFrame({"col1": [1, 2, 3], "col2": [4.5, 5.6, 6.7], "col3": ["abc", "def", "ghi"]}) invalid_characters = "!@#" # Should not raise an exception