Merge pull request #104 from ACED-IDP/fix/repeatable-push

fixes: repeatable push, G3T_NUM_PARALLEL,  skip validate client_ids

gen3_tracker/git/__init__.py

@@ -312,6 +312,7 @@ def git_files(dry_run=False) -> list[str]:
         to_upload = set()
         for _ in git_logs:
             to_upload.update([_ for _ in _['files'] if _.startswith('MANIFEST')])
+            break
         return list(to_upload)
     return []
 
@@ -597,7 +598,14 @@ def save(self, dvc: DVC) -> str:
             self.manifest.append(to_manifest(dvc))
         return 'OK'
 
-    def commit(self, dry_run=False, profile=None, upload_path=None, bucket_name=None, worker_count=(multiprocessing.cpu_count() - 1)):
+    @staticmethod
+    def default_worker_count():
+        """Get the default worker count."""
+        if 'G3T_NUM_PARALLEL' in os.environ:
+            return int(os.environ.get('G3T_NUM_PARALLEL'))
+        return multiprocessing.cpu_count() - 1
+
+    def commit(self, dry_run=False, profile=None, upload_path=None, bucket_name=None, worker_count=default_worker_count()):
         with open(self.manifest_file_path, 'w') as f:
             json.dump(self.manifest, f)
         if len(self.manifest) > 0:

gen3_tracker/git/cli.py

@@ -254,7 +254,13 @@ def commit(ctx, targets, message, all):
     if all:
         command.append("-a")
 
-    run_command(" ".join(command), dry_run=config.dry_run, no_capture=True)
+    try:
+        run_command(" ".join(command), dry_run=config.dry_run, no_capture=True)
+    except Exception as e:
+        click.secho(str(e), fg=ERROR_COLOR, file=sys.stderr)
+        if config.debug:
+            raise
+        exit(1)
 
 
 @cli.command()

gen3_tracker/meta/validator.py

@@ -72,7 +72,12 @@ def _check_reference(self: Reference, *args, **kwargs):
 
 
 def validate(directory_path: pathlib.Path, project_id=None) -> ValidateDirectoryResult:
-    """Check FHIR data, accumulate results."""
+    """Check FHIR data, accumulate results.
+
+    Args:
+        directory_path: pathlib.Path
+        project_id: str, optional if set, check that the resource id is valid for the project_id
+    """
     exceptions = []
     resources = defaultdict(int)
     # add resources to bundle
@@ -85,7 +90,8 @@ def validate(directory_path: pathlib.Path, project_id=None) -> ValidateDirectory
             continue
 
         try:
-            assert_valid_id(parse_result.resource, project_id)
+            if project_id:
+                assert_valid_id(parse_result.resource, project_id)
         except Exception as e:
             parse_result.exception = e
             exceptions.append(parse_result)

setup.py

@@ -11,7 +11,7 @@
 
 setup(
     name='gen3_tracker',
-    version='0.0.6',
+    version='0.0.7rc1',
     description='A CLI for adding version control to Gen3 data submission projects.',
     long_description=long_description,
     long_description_content_type='text/markdown',

tests/integration/test_end_to_end_workflow.py

@@ -57,6 +57,16 @@ def test_simple_workflow(runner: CliRunner, project_id, tmpdir) -> None:
     # validate the meta files
     run(runner, ["--debug", "meta", "validate"])
 
+    # update the file
+    test_file = pathlib.Path("my-project-data/hello.txt")
+    test_file.parent.mkdir(parents=True, exist_ok=True)
+    test_file.write_text('hello UPDATE\n')
+    # re-add the file
+    run(runner, ["--debug", "add", str(test_file)], expected_files=["MANIFEST/my-project-data/hello.txt.dvc"])
+    run(runner, ["--debug", "meta", "init"], expected_files=["META/DocumentReference.ndjson"])
+    run(runner, ["--debug", "commit", "-am", "updated"])
+    run(runner, ["--debug", "meta", "validate"])
+
     # create a visualisation
     run(runner, ["--debug", "meta", "graph"], expected_files=["meta.html"])
 

tests/unit/meta/test_meta.py

@@ -6,6 +6,7 @@
 
 import gen3_tracker.config
 from gen3_tracker.git import run_command
+from gen3_tracker.meta.validator import validate
 from tests import run
 
 
@@ -63,6 +64,10 @@ def test_assert_object_id_invalid_on_project_id_change(runner: CliRunner, projec
         yaml.dump(config.model_dump(), f)
     run(runner, ["commit", "-m",  "restore-project_id", '.g3t/config.yaml'])
 
+    # ensure we can validate without passing project id
+    results = validate(directory_path="META")
+    assert len(results.exceptions) == 0, "Expected no exceptions."
+
     run(runner, ["--debug", "meta", "validate"], expected_exit_code=0)
     run(runner, ["--debug", "push", "--dry-run"], expected_exit_code=0)
 

tests/unit/test_num_parallel.py

@@ -0,0 +1,12 @@
+import os
+
+from gen3_tracker.git import Gen3ClientRemoteWriter
+
+
+def test_num_parallel():
+    """Test the default worker count for file uploads."""
+    os.environ["G3T_NUM_PARALLEL"] = "9999"
+    assert Gen3ClientRemoteWriter.default_worker_count() == 9999
+    del os.environ["G3T_NUM_PARALLEL"]
+    assert Gen3ClientRemoteWriter.default_worker_count() != 9999
+    assert Gen3ClientRemoteWriter.default_worker_count() > 0