From 8f03e8578d49326ed009670c6619f5e1fc76d066 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20M=C3=BCller?= Date: Tue, 2 Nov 2021 15:01:18 +0100 Subject: [PATCH] ref: mirgrate to package_revise for updating datasets (#28) --- CHANGELOG | 2 ++ dcoraid/api/dataset.py | 17 ++--------------- dcoraid/upload/job.py | 14 -------------- 3 files changed, 4 insertions(+), 29 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 39fe0e6..aeaeedb 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,5 @@ +0.7.7 + - ref: mirgrate to package_revise for updating datasets (#28) 0.7.6 - enh: add "create_task" function for generating custom .dcoraid-task files in Python scripts diff --git a/dcoraid/api/dataset.py b/dcoraid/api/dataset.py index 842b258..1bfc0cb 100644 --- a/dcoraid/api/dataset.py +++ b/dcoraid/api/dataset.py @@ -170,21 +170,8 @@ def resource_add(dataset_id, path, api, resource_name=None, else: # Attempt upload with path.open("rb") as fd: - e = MultipartEncoder(fields={ - 'package_id': dataset_id, - 'name': resource_name, - 'upload': (resource_name, fd)}) - m = MultipartEncoderMonitor(e, monitor_callback) - # perform upload - data = api.post("resource_create", - data=m, - dump_json=False, - headers={"Content-Type": m.content_type}) - if False: - # TODO: Seems like package_revise does not trigger - # IResourceController.after_create! - # https://github.com/DCOR-dev/DCOR-Aid/issues/28 - # https://github.com/ckan/ckan/issues/6472 + # use package_revise to upload the resource + # https://github.com/DCOR-dev/DCOR-Aid/issues/28 e = MultipartEncoder(fields={ 'match__id': dataset_id, 'update__resources__extend': f'[{{"name":"{resource_name}"}}]', diff --git a/dcoraid/upload/job.py b/dcoraid/upload/job.py index 29092b8..0ea26ea 100644 --- a/dcoraid/upload/job.py +++ b/dcoraid/upload/job.py @@ -404,20 +404,6 @@ def task_upload_resources(self): exist_ok=True, monitor_callback=self.monitor_callback) self.paths_uploaded.append(path) - # Workaround until `package_revise` is in `resource_add` - # (https://github.com/DCOR-dev/DCOR-Aid/issues/28): - # Wait for the SHA256 sum to be computed by DCOR. - for _ in range(60): - sha256dict = resource_sha256_sums( - dataset_id=self.dataset_id, - api=self.api) - if sha256dict[resource_name] is not None: - break - time.sleep(10) - else: - warnings.warn( - f"SHA256 sum for {self.dataset_id}: " - + f"{resource_name} not yet computed by DCOR!") self.end_time = time.perf_counter() self.set_state("online") else: