From f44ea67e03ac9990863d2b5d8eb6043635bfc3c5 Mon Sep 17 00:00:00 2001 From: ZdenekM Date: Tue, 30 Jan 2024 09:13:36 +0100 Subject: [PATCH] fix(arcor2_build): extra objects were not added into package --- README.md | 2 +- compose-files/fit-demo/docker-compose.yml | 2 +- src/docker/arcor2_build/BUILD | 2 +- src/python/arcor2_build/CHANGELOG.md | 10 ++ src/python/arcor2_build/VERSION | 2 +- src/python/arcor2_build/scripts/build.py | 32 +++++-- .../arcor2_build/tests/test_cross_import.py | 24 ++++- .../arcor2_mocks/scripts/mock_project.py | 93 ++++++++++++++++--- 8 files changed, 137 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 5fcece395..64a95328e 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ The following video by [Kinali](https://www.kinali.cz/en/) shows the use case (o [README](src/python/arcor2_build/README.md) | [CHANGELOG](src/python/arcor2_build/CHANGELOG.md) - - 2024-01-26: [1.3.0](https://github.com/robofit/arcor2/releases/tag/arcor2_build%2F1.3.0) ([docker](https://hub.docker.com/r/arcor2/arcor2_build/tags?page=1&ordering=last_updated&name=1.3.0), [pypi](https://pypi.org/project/arcor2-build/1.3.0/)). + - 2024-01-30: [1.3.1](https://github.com/robofit/arcor2/releases/tag/arcor2_build%2F1.3.1) ([docker](https://hub.docker.com/r/arcor2/arcor2_build/tags?page=1&ordering=last_updated&name=1.3.1), [pypi](https://pypi.org/project/arcor2-build/1.3.1/)). ### arcor2_build_data diff --git a/compose-files/fit-demo/docker-compose.yml b/compose-files/fit-demo/docker-compose.yml index 513382b52..171e289a8 100644 --- a/compose-files/fit-demo/docker-compose.yml +++ b/compose-files/fit-demo/docker-compose.yml @@ -44,7 +44,7 @@ services: - ARCOR2_KINECT_AZURE_URL=http://192.168.104.100:5017 # Run kinect using pants fit-demo-build: - image: arcor2/arcor2_build:1.2.0 + image: arcor2/arcor2_build:1.3.1 container_name: fit-demo-build depends_on: fit-demo-project: diff --git a/src/docker/arcor2_build/BUILD b/src/docker/arcor2_build/BUILD index 117153015..4fe60d3a9 100644 --- a/src/docker/arcor2_build/BUILD +++ b/src/docker/arcor2_build/BUILD @@ -1 +1 @@ -docker_image(name="arcor2_build", repository="arcor2/arcor2_build", image_tags=["1.3.0"]) +docker_image(name="arcor2_build", repository="arcor2/arcor2_build", image_tags=["1.3.1"]) diff --git a/src/python/arcor2_build/CHANGELOG.md b/src/python/arcor2_build/CHANGELOG.md index b52c62e1e..d9b1da8ff 100644 --- a/src/python/arcor2_build/CHANGELOG.md +++ b/src/python/arcor2_build/CHANGELOG.md @@ -2,6 +2,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +## [1.3.1] - 2024-01-30 + +### Fixed + +- Objects specified in `project_objects_ids` were actually missing in the package. + +### Changes + +- Support for `project_objects_ids` was added for `import`. + ## [1.3.0] - 2024-01-26 ### Changed diff --git a/src/python/arcor2_build/VERSION b/src/python/arcor2_build/VERSION index 589268e6f..6261a05bb 100644 --- a/src/python/arcor2_build/VERSION +++ b/src/python/arcor2_build/VERSION @@ -1 +1 @@ -1.3.0 \ No newline at end of file +1.3.1 \ No newline at end of file diff --git a/src/python/arcor2_build/scripts/build.py b/src/python/arcor2_build/scripts/build.py index d9eccfd60..e1c726558 100644 --- a/src/python/arcor2_build/scripts/build.py +++ b/src/python/arcor2_build/scripts/build.py @@ -160,6 +160,7 @@ def _publish(project_id: str, package_name: str) -> RespT: logger.debug(f"Getting additional module {additional_module_id}.") am = ps.get_object_type(additional_module_id) save_type_def(am.source, am.id, tmp_dir, OBJECT_TYPE_MODULE) + zf.writestr(os.path.join(ot_path, humps.depascalize(additional_module_id)) + ".py", am.source) # to allow imports between OTs, all objects listed in scene are downloaded first for scene_obj in scene.objects: @@ -412,6 +413,24 @@ def project_import() -> RespT: prepare_object_types_dir(tmp_dir, OBJECT_TYPE_MODULE) + # read and save additional objects... + if project.project_objects_ids: + for add_obj in project.project_objects_ids: + try: + src = read_str_from_zip(zip_file, f"object_types/{humps.depascalize(add_obj)}.py") + except KeyError: + raise NotFound(f"Additional object {add_obj} is missing in the package.") + objects[add_obj] = ObjectType(add_obj, src) + save_type_def(src, add_obj, tmp_dir, OBJECT_TYPE_MODULE) + + # extra pass through scene objects to save them all - in order to allow imports between OTs + for scene_obj in scene.objects: + try: + obj_type_src = read_str_from_zip(zip_file, f"object_types/{humps.depascalize(scene_obj.type)}.py") + except KeyError: + raise NotFound(f"Object type {scene_obj.type} is missing in the package.") + save_type_def(obj_type_src, scene_obj.type, tmp_dir, OBJECT_TYPE_MODULE) + for scene_obj in scene.objects: obj_type_name = scene_obj.type @@ -462,7 +481,7 @@ def project_import() -> RespT: models[obj_type.id] = model - if not project.has_logic: + if not project.has_logic or update_project_from_script: logger.debug("Importing the main script.") try: @@ -475,13 +494,6 @@ def project_import() -> RespT: except Arcor2Exception: raise InvalidPackage("Invalid code of the main script.") - # case when preparing data from script to decompilation - if update_project_from_script: - try: - src = zip_file.read("script.py").decode("UTF-8") - except KeyError: - raise NotFound("Could not find script.py.") - # check that we are not going to overwrite something if not overwrite_scene: try: @@ -537,8 +549,8 @@ def project_import() -> RespT: if update_project_from_script: logger.debug("Decompiling source...") - project = python_to_json(project, scene, src, object_type) - logger.debug("Decompile was successfull and project was overwritten") + project = python_to_json(project, scene, script, object_type) + logger.debug("Decompilation was successfull and project was overwritten") for model in models.values(): ps.put_model(model) diff --git a/src/python/arcor2_build/tests/test_cross_import.py b/src/python/arcor2_build/tests/test_cross_import.py index f16af693b..fee0ceed6 100644 --- a/src/python/arcor2_build/tests/test_cross_import.py +++ b/src/python/arcor2_build/tests/test_cross_import.py @@ -2,13 +2,14 @@ import subprocess as sp import tempfile import time +import zipfile from typing import Iterator import pytest from arcor2 import rest from arcor2.clients import project_service as ps -from arcor2.data.common import Project, Scene, SceneObject +from arcor2.data.common import Project, ProjectSources, Scene, SceneObject from arcor2.data.object_type import ObjectType from arcor2.helpers import find_free_port @@ -76,7 +77,7 @@ def whatever(): from .additional_module import whatever class ObjectTypeOne(Generic): - pass + _ABSTRACT = False """ ot2 = """ @@ -85,7 +86,7 @@ class ObjectTypeOne(Generic): from .additional_module import whatever class ObjectTypeTwo(Generic): - pass + _ABSTRACT = False """ @@ -104,6 +105,7 @@ def test_cross_import(start_processes: None) -> None: project.project_objects_ids = ["AdditionalModule"] project.has_logic = False ps.update_project(project) + ps.update_project_sources(ProjectSources(project.id, "blah")) with tempfile.TemporaryDirectory() as tmpdirname: path = os.path.join(tmpdirname, "publish.zip") @@ -116,3 +118,19 @@ def test_cross_import(start_processes: None) -> None: "projectId": project.id, }, ) + + with zipfile.ZipFile(path) as zip_file: + ot_dir_list = [name for name in zip_file.namelist() if name.startswith("object_types")] + # there should be three OTs and __init__.py + assert len(ot_dir_list) == 4, f"Strange content of object_types dir: {ot_dir_list}" + + assert {ot.id for ot in ps.get_object_type_ids()} == {"AdditionalModule", "ObjectTypeOne", "ObjectTypeTwo"} + + ps.delete_object_type("AdditionalModule") + ps.delete_object_type("ObjectTypeOne") + ps.delete_object_type("ObjectTypeTwo") + + with open(path, "rb") as fh: + rest.call(rest.Method.PUT, url=f"{build_url}/project/import", files={"executionPackage": fh.read()}) + + assert {ot.id for ot in ps.get_object_type_ids()} == {"AdditionalModule", "ObjectTypeOne", "ObjectTypeTwo"} diff --git a/src/python/arcor2_mocks/scripts/mock_project.py b/src/python/arcor2_mocks/scripts/mock_project.py index c54766774..bd86f06c4 100644 --- a/src/python/arcor2_mocks/scripts/mock_project.py +++ b/src/python/arcor2_mocks/scripts/mock_project.py @@ -1,8 +1,6 @@ #!/usr/bin/env python3 import argparse -import copy -import random from datetime import datetime, timezone import humps @@ -18,6 +16,7 @@ SCENES: dict[str, common.Scene] = {} PROJECTS: dict[str, common.Project] = {} +PROJECT_SOURCES: dict[str, common.ProjectSources] = {} OBJECT_TYPES: dict[str, object_type.ObjectType] = {} BOXES: dict[str, object_type.Box] = {} @@ -76,6 +75,80 @@ def put_project() -> RespT: return jsonify(project.modified.isoformat()) +@app.route("/projects/sources", methods=["PUT"]) +def put_project_sources() -> RespT: + """Add or update project sources. + --- + put: + tags: + - Project + description: Add or update project sources. + requestBody: + content: + application/json: + schema: + $ref: ProjectSources + responses: + 200: + description: Timestamp of last project modification. + 500: + description: "Error types: **General**, **ProjectGeneral**, **NotFound**." + content: + application/json: + schema: + $ref: WebApiError + """ + + if not isinstance(request.json, dict): + raise ProjectGeneral("Body should be a JSON dict containing ProjectSources.") + + project_sources = common.ProjectSources.from_dict(humps.decamelize(request.json)) + + if project_sources.id not in PROJECTS: + raise NotFound(f"Project {project_sources.id} does not exist.") + + PROJECT_SOURCES[project_sources.id] = project_sources + return Response(status=200) + + +@app.route("/projects//sources", methods=["GET"]) +def get_project_sources(id: str) -> RespT: + """Get project sources. + --- + get: + tags: + - Project + summary: Gets project by project id. + parameters: + - name: id + in: path + description: unique ID + required: true + schema: + type: string + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: ProjectSources + 500: + description: "Error types: **General**, **NotFound**." + content: + application/json: + schema: + $ref: WebApiError + """ + + try: + project_sources = PROJECT_SOURCES[id] + except KeyError: + raise NotFound(f"Sources for project {id} not found.") + + return jsonify(humps.camelize(project_sources.to_dict())) + + @app.route("/projects/", methods=["GET"]) def get_project(id: str) -> RespT: """Add or update project. @@ -107,16 +180,11 @@ def get_project(id: str) -> RespT: """ try: - project_copy = copy.deepcopy(PROJECTS[id]) + project = PROJECTS[id] except KeyError: raise NotFound(f"Project {id} was not found.") - random.shuffle(project_copy.action_points) - random.shuffle(project_copy.logic) - random.shuffle(project_copy.object_overrides) - random.shuffle(project_copy.parameters) - - return jsonify(humps.camelize(project_copy.to_dict())) + return jsonify(humps.camelize(project.to_dict())) @app.route("/projects/", methods=["DELETE"]) @@ -274,13 +342,11 @@ def get_scene(id: str) -> RespT: """ try: - scene_copy = copy.deepcopy(SCENES[id]) + scene = SCENES[id] except KeyError: raise NotFound(f"Scene {id} was not found.") - random.shuffle(scene_copy.objects) - - return jsonify(humps.camelize(scene_copy.to_dict())) + return jsonify(humps.camelize(scene.to_dict())) @app.route("/scenes/", methods=["DELETE"]) @@ -881,6 +947,7 @@ def main() -> None: PROJECT_PORT, [ common.Project, + common.ProjectSources, common.Scene, common.IdDesc, object_type.ObjectType,