From daa05c2db7b6a403992c7710b4e67e66aabe07a1 Mon Sep 17 00:00:00 2001 From: "M. Rehan" Date: Wed, 31 Jul 2024 14:58:38 +0500 Subject: [PATCH 1/2] Don't normalize apps versions data while caching --- apps_ci/scripts/catalog_update.py | 3 ++- catalog_reader/app.py | 15 +++++++++++---- catalog_reader/catalog.py | 12 ++++++++---- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/apps_ci/scripts/catalog_update.py b/apps_ci/scripts/catalog_update.py index 3c09d6d..fd8662c 100644 --- a/apps_ci/scripts/catalog_update.py +++ b/apps_ci/scripts/catalog_update.py @@ -26,7 +26,8 @@ def get_trains(location: str) -> typing.Tuple[dict, dict]: catalog_data = {} versions_data = {} for train_name, train_data in retrieve_trains_data( - get_apps_in_trains(trains_to_traverse, location), location, preferred_trains, trains_to_traverse + get_apps_in_trains(trains_to_traverse, location), location, preferred_trains, trains_to_traverse, + normalize_questions=False, )[0].items(): catalog_data[train_name] = {} versions_data[train_name] = {} diff --git a/catalog_reader/app.py b/catalog_reader/app.py index d9ff930..d23a8ee 100644 --- a/catalog_reader/app.py +++ b/catalog_reader/app.py @@ -19,7 +19,7 @@ def get_app_details( - item_location: str, questions_context: typing.Optional[dict] = None, options: typing.Optional[dict] = None + item_location: str, questions_context: typing.Optional[dict] = None, options: typing.Optional[dict] = None, ) -> dict: catalog_path = item_location.rstrip('/').rsplit('/', 2)[0] item = item_location.rsplit('/', 1)[-1] @@ -51,6 +51,7 @@ def get_app_details( item_data.update(get_app_details_impl(item_location, schema, questions_context, { 'retrieve_latest_version': not retrieve_versions, 'default_values_callable': options.get('default_values_callable'), + 'normalize_questions': options.get('normalize_questions', True), })) unhealthy_versions = [] desired_keys_mapping = { @@ -134,7 +135,11 @@ def get_app_details_impl( version_details.update({ 'healthy': True, - **get_app_version_details(version_details['location'], questions_context) + **get_app_version_details( + version_details['location'], questions_context, { + 'normalize_questions': options.get('normalize_questions', True), + }, + ) }) if retrieve_latest_version: break @@ -145,6 +150,7 @@ def get_app_details_impl( def get_app_version_details( version_path: str, questions_context: typing.Optional[dict], options: typing.Optional[dict] = None ) -> dict: + options = options or {} version_data = {'location': version_path, 'required_features': set()} for key, filename, parser in ( ('app_metadata', 'app.yaml', yaml.safe_load), @@ -160,13 +166,14 @@ def get_app_version_details( # We will normalise questions now so that if they have any references, we render them accordingly # like a field referring to available interfaces on the system - normalize_questions(version_data, questions_context or get_default_questions_context()) + if options.get('normalize_questions', True): + normalize_questions(version_data, questions_context or get_default_questions_context()) version_data.update({ 'supported': version_supported(version_data), 'required_features': list(version_data['required_features']), }) - if options and options.get('default_values_callable'): + if options.get('default_values_callable'): version_data['values'] = options['default_values_callable'](version_data) app_metadata = version_data['app_metadata'] diff --git a/catalog_reader/catalog.py b/catalog_reader/catalog.py index 631d680..9e836a7 100644 --- a/catalog_reader/catalog.py +++ b/catalog_reader/catalog.py @@ -8,11 +8,15 @@ from .train_utils import get_train_path, is_train_valid -def app_details(apps: dict, location: str, questions_context: typing.Optional[dict], app_key: str) -> dict: +def app_details( + apps: dict, location: str, questions_context: typing.Optional[dict], app_key: str, normalize_questions: bool = True, +) -> dict: train = apps[app_key] app = app_key.removesuffix(f'_{train}') app_location = os.path.join(get_train_path(location), train, app) - return get_app_details(app_location, questions_context, {'retrieve_versions': True}) + return get_app_details( + app_location, questions_context, {'retrieve_versions': True, 'normalize_questions': normalize_questions}, + ) def retrieve_train_names(location: str, all_trains=True, trains_filter=None) -> list: @@ -37,8 +41,8 @@ def get_apps_in_trains(trains_to_traverse: list, catalog_location: str) -> dict: def retrieve_trains_data( - apps: dict, catalog_location: str, preferred_trains: list, - trains_to_traverse: list, job: typing.Any = None, questions_context: typing.Optional[dict] = None + apps: dict, catalog_location: str, preferred_trains: list, trains_to_traverse: list, job: typing.Any = None, + questions_context: typing.Optional[dict] = None, normalize_questions: bool = True, ) -> typing.Tuple[dict, set]: questions_context = questions_context or get_default_questions_context() trains = { From a5864070cd4e7edb413e8699867d4a37e26ea73e Mon Sep 17 00:00:00 2001 From: "M. Rehan" Date: Wed, 31 Jul 2024 16:52:38 +0500 Subject: [PATCH 2/2] Fix small bug --- catalog_reader/catalog.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/catalog_reader/catalog.py b/catalog_reader/catalog.py index 9e836a7..6183978 100644 --- a/catalog_reader/catalog.py +++ b/catalog_reader/catalog.py @@ -54,7 +54,9 @@ def retrieve_trains_data( total_apps = len(apps) with concurrent.futures.ProcessPoolExecutor(max_workers=(5 if total_apps > 10 else 2)) as exc: for index, result in enumerate(zip(apps, exc.map( - functools.partial(app_details, apps, catalog_location, questions_context), + functools.partial( + app_details, apps, catalog_location, questions_context, normalize_questions=normalize_questions + ), apps, chunksize=(10 if total_apps > 10 else 5) ))): app_key = result[0]