diff --git a/qa-midrc.planx-pla.net/etlMapping.yaml b/qa-midrc.planx-pla.net/etlMapping.yaml index b95601ba1..1b74c63bd 100644 --- a/qa-midrc.planx-pla.net/etlMapping.yaml +++ b/qa-midrc.planx-pla.net/etlMapping.yaml @@ -30,7 +30,7 @@ mappings: path: imaging_studies.mr_series_files fn: count parent_props: - - path: datasets[dataset_submitter_id:submitter_id] + - path: datasets[dataset_submitter_id:submitter_id,license,data_url_doi,data_contributor] nested_props: - name: case_annotations path: annotations @@ -42,6 +42,7 @@ mappings: props: - name: airspace_disease_grading - name: class_covid19_pneumonia + - name: annotation_name - name: midrc_mRALE_score - name: annotation_method - name: annotator_id @@ -63,7 +64,6 @@ mappings: - name: medication_code_system - name: medication_manufacturer - name: medication_name - - name: medication_status - name: medication_type - name: conditions path: conditions @@ -88,13 +88,13 @@ mappings: - name: days_from_study_to_neg_covid_test - name: days_from_study_to_pos_covid_test - name: days_to_study + - name: image_data_modified - name: loinc_code - name: loinc_contrast - name: loinc_long_common_name - name: loinc_method - name: loinc_system - name: study_description - - name: study_location - name: study_modality - name: study_year - name: study_uid @@ -114,6 +114,18 @@ mappings: - name: data_category src: data_category fn: set + - name: data_file_image_data_modified + src: image_data_modified + fn: set + - name: data_file_image_data_modification_name + src: image_data_modification_name + fn: set + - name: data_file_image_data_modification_method + src: image_data_modification_method + fn: set + - name: data_file_count + src: _data_file_id + fn: count - name: midrc_measurement doc_type: measurement type: aggregator @@ -126,6 +138,8 @@ mappings: - name: test_method - name: test_name - name: test_result_text + parent_props: + - path: cases.datasets[dataset_submitter_id:submitter_id,license,data_url_doi,data_contributor] - name: midrc_annotation doc_type: annotation type: aggregator @@ -136,6 +150,7 @@ mappings: - name: case_ids - name: airspace_disease_grading - name: class_covid19_pneumonia + - name: annotation_name - name: midrc_mRALE_score - name: annotation_method - name: annotator_id @@ -153,13 +168,13 @@ mappings: - name: days_from_study_to_neg_covid_test - name: days_from_study_to_pos_covid_test - name: days_to_study + - name: image_data_modified - name: loinc_code - name: loinc_contrast - name: loinc_long_common_name - name: loinc_method - name: loinc_system - name: study_description - - name: study_location - name: study_modality - name: study_year - name: study_year_shifted @@ -170,6 +185,7 @@ mappings: props: - name: airspace_disease_grading - name: class_covid19_pneumonia + - name: annotation_name - name: midrc_mRALE_score - name: annotation_method - name: annotator_id @@ -188,7 +204,7 @@ mappings: path: mr_series_files fn: count parent_props: - - path: cases[sex,race,age_at_index,index_event,zip,covid19_positive,ethnicity].datasets[dataset_submitter_id:submitter_id] + - path: cases[sex,race,age_at_index,index_event,zip,covid19_positive,ethnicity].datasets[dataset_submitter_id:submitter_id,license,data_url_doi,data_contributor] joining_props: - index: data_file join_on: _imaging_study_id @@ -205,6 +221,24 @@ mappings: - name: data_category src: data_category fn: set + - name: data_file_source_node + src: source_node + fn: set + - name: data_file_annotation_name + src: annotation_name + fn: set + - name: data_file_annotation_method + src: annotation_method + fn: set + - name: data_file_image_data_modified + src: image_data_modified + fn: set + - name: data_file_image_data_modification_name + src: image_data_modification_name + fn: set + - name: data_file_image_data_modification_method + src: image_data_modification_method + fn: set - name: midrc_data_file doc_type: data_file type: collector @@ -239,6 +273,9 @@ mappings: - name: echo_time - name: echo_train_length - name: exposure_modulation_type + - name: image_data_modified + - name: image_data_modification_method + - name: image_data_modification_name - name: image_type - name: imaged_nucleus - name: imager_pixel_spacing @@ -246,6 +283,7 @@ mappings: - name: manufacturer - name: manufacturer_model_name - name: modality + - name: number_of_instances - name: number_of_temporal_positions - name: patient_position - name: pixel_spacing @@ -266,6 +304,15 @@ mappings: - name: _dataset_id src: id fn: set + - name: license + src: license + fn: set + - name: data_url_doi + src: data_url_doi + fn: set + - name: data_contributor + src: data_contributor + fn: set annotation: props: - name: _annotation_id @@ -323,9 +370,6 @@ mappings: - name: study_modality src: study_modality fn: set - - name: study_location - src: study_location - fn: set - name: study_year src: study_year fn: set diff --git a/qa-midrc.planx-pla.net/manifest.json b/qa-midrc.planx-pla.net/manifest.json index f11c977be..628dc77f9 100644 --- a/qa-midrc.planx-pla.net/manifest.json +++ b/qa-midrc.planx-pla.net/manifest.json @@ -5,26 +5,27 @@ ], "versions": { "ambassador": "quay.io/datawire/ambassador:1.4.2", - "arborist": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/arborist:2023.12", + "arborist": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/arborist:2024.08", "aws-es-proxy": "quay.io/cdis/aws-es-proxy:v1.3.1", - "dashboard": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/gen3-statics:2023.12", - "dicom-server": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/gen3-orthanc:gen3-0.1.1", - "fence": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/fence:2023.12", + "dashboard": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/gen3-statics:2024.08", + "dicom-server": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/gen3-orthanc:gen3-0.1.2", + "fence": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/fence:10.2.0", "fluentd": "fluent/fluentd-kubernetes-daemonset:v1.15.3-debian-cloudwatch-1.0", - "guppy": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/guppy:2024.04", - "indexd": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/indexd:2023.12", - "manifestservice": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/manifestservice:2023.12", - "metadata": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/metadata-service:2023.12", - "ohif-viewer": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/ohif-viewer:gen3-v3.8.0", - "peregrine": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/peregrine:2023.12", - "portal": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/data-portal:5.24.0", "frontend-framework": "quay.io/cdis/frontend-framework:feat_n3c", - "revproxy": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/nginx:2023.12", - "sheepdog": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/sheepdog:2023.12", - "spark": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/gen3-spark:2023.12", - "ssjdispatcher": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/ssjdispatcher:2023.12", - "tube": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/tube:2023.12", - "wts": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/workspace-token-service:2023.12" + "guppy": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/guppy:2024.08", + "indexd": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/indexd:2024.08", + "manifestservice": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/manifestservice:2024.08", + "metadata": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/metadata-service:2024.08", + "peregrine": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/peregrine:2024.08", + "portal": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/data-portal:5.33.0", + "revproxy": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/nginx:2024.08", + "sheepdog": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/sheepdog:2024.08", + "spark": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/gen3-spark:2024.08", + "ssjdispatcher": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/ssjdispatcher:2024.08", + "tube": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/tube:2024.08", + "wts": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/workspace-token-service:2024.08" + + }, "arborist": { "deployment_version": "2" @@ -82,7 +83,7 @@ "tier_access_limit": "2", "useryaml_s3path": "s3://cdis-gen3-users/qa/user.yaml", "dispatcher_job_num": "10", - "dictionary_url": "http://s3.amazonaws.com/dictionary-artifacts/midrc_dictionary/1.2.5/schema.json", + "dictionary_url": "http://s3.amazonaws.com/dictionary-artifacts/midrc_dictionary/1.3.5/schema.json", "sync_from_dbgap": "False", "frontend_root": "portal", "lb_type": "internal",