From 273c80a01dcae28661804d3aa292c786b8933277 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexandre=20Bult=C3=A9?= Date: Tue, 10 Sep 2024 10:23:22 +0200 Subject: [PATCH 1/3] add result_ttl conf for transform and migrate --- ecospheres_migrator/app.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/ecospheres_migrator/app.py b/ecospheres_migrator/app.py index 8e61329..f1279a5 100644 --- a/ecospheres_migrator/app.py +++ b/ecospheres_migrator/app.py @@ -23,6 +23,8 @@ app = Flask(__name__) app.config["SECRET_KEY"] = os.getenv("FLASK_SECRET_KEY", "default-secret-key") +app.config["TRANSFORM_TTL"] = 60 * 60 * 24 * 7 # 1 week +app.config["MIGRATE_TTL"] = 60 * 60 * 24 * 7 * 30 # 1 month @app.route("/") @@ -96,7 +98,9 @@ def transform(): abort(400, "Missing `transformation` parameter") migrator = Migrator(url=url, username=username, password=password) selection = migrator.select(query=query) - job = get_queue().enqueue(migrator.transform, transformation, selection) + job = get_queue().enqueue( + migrator.transform, transformation, selection, result_ttl=app.config["TRANSFORM_TTL"] + ) return redirect(url_for("transform_success", job_id=job.id)) @@ -160,7 +164,11 @@ def migrate(job_id: str): abort(400, "Missing `group` parameter") migrator = Migrator(url=url, username=username, password=password) migrate_job = get_queue().enqueue( - migrator.migrate, transform_job.result, overwrite=overwrite, group=group + migrator.migrate, + transform_job.result, + overwrite=overwrite, + group=group, + result_ttl=app.config["MIGRATE_TTL"], ) return redirect(url_for("migrate_success", job_id=migrate_job.id)) From 65fa1b2544d4f8f4b3194a0200270492304b8a01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexandre=20Bult=C3=A9?= Date: Tue, 10 Sep 2024 10:27:20 +0200 Subject: [PATCH 2/3] add url to batches --- ecospheres_migrator/batch.py | 2 ++ ecospheres_migrator/migrator.py | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/ecospheres_migrator/batch.py b/ecospheres_migrator/batch.py index 841ea96..20933d1 100644 --- a/ecospheres_migrator/batch.py +++ b/ecospheres_migrator/batch.py @@ -8,6 +8,7 @@ class BatchRecord: uuid: str template: bool original: str + url: str @dataclass(kw_only=True) @@ -49,6 +50,7 @@ class MigrateBatchRecord: source_content: str target_content: str template: bool + url: str @dataclass(kw_only=True) diff --git a/ecospheres_migrator/migrator.py b/ecospheres_migrator/migrator.py index f960d8c..dbcf295 100644 --- a/ecospheres_migrator/migrator.py +++ b/ecospheres_migrator/migrator.py @@ -67,6 +67,7 @@ def transform(self, transformation: Path, selection: list[Record]) -> Batch: # TODO: check if result != original batch.add( SuccessBatchRecord( + url=self.gn.url, uuid=r.uuid, template=r.template, original=xml_to_string(original), @@ -77,6 +78,7 @@ def transform(self, transformation: Path, selection: list[Record]) -> Batch: except Exception as e: batch.add( FailureBatchRecord( + url=self.gn.url, uuid=r.uuid, template=r.template, original=xml_to_string(original), @@ -100,6 +102,7 @@ def migrate( self.gn.update_record(r.uuid, r.result, template=r.template) migrate_batch.add( SuccessMigrateBatchRecord( + url=self.gn.url, source_uuid=r.uuid, target_uuid=r.uuid, template=r.template, @@ -115,6 +118,7 @@ def migrate( ) migrate_batch.add( SuccessMigrateBatchRecord( + url=self.gn.url, source_uuid=r.uuid, target_uuid=new_record["uuid"], template=r.template, @@ -125,6 +129,7 @@ def migrate( except Exception as e: migrate_batch.add( FailureMigrateBatchRecord( + url=self.gn.url, source_uuid=r.uuid, template=r.template, source_content=r.original, From da18aea5cc8368f64875803a0a8828d8c95e0df7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexandre=20Bult=C3=A9?= Date: Wed, 11 Sep 2024 15:08:22 +0200 Subject: [PATCH 3/3] set both TTLs to 2 months --- ecospheres_migrator/app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ecospheres_migrator/app.py b/ecospheres_migrator/app.py index f1279a5..1473768 100644 --- a/ecospheres_migrator/app.py +++ b/ecospheres_migrator/app.py @@ -23,8 +23,8 @@ app = Flask(__name__) app.config["SECRET_KEY"] = os.getenv("FLASK_SECRET_KEY", "default-secret-key") -app.config["TRANSFORM_TTL"] = 60 * 60 * 24 * 7 # 1 week -app.config["MIGRATE_TTL"] = 60 * 60 * 24 * 7 * 30 # 1 month +app.config["TRANSFORM_TTL"] = 60 * 60 * 24 * 7 * 30 * 2 # 2 months +app.config["MIGRATE_TTL"] = 60 * 60 * 24 * 7 * 30 * 2 # 2 months @app.route("/")