Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

updated MIDRC manifest #3002

Merged
merged 2 commits into from
Sep 10, 2024
Merged

updated MIDRC manifest #3002

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
446b04c
updated MIDRC manifest
cgmeyer Sep 10, 2024
9a1f81c
updating MIDRC etl
cgmeyer Sep 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 52 additions & 8 deletions qa-midrc.planx-pla.net/etlMapping.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ mappings:
path: imaging_studies.mr_series_files
fn: count
parent_props:
- path: datasets[dataset_submitter_id:submitter_id]
- path: datasets[dataset_submitter_id:submitter_id,license,data_url_doi,data_contributor]
nested_props:
- name: case_annotations
path: annotations
Expand All @@ -42,6 +42,7 @@ mappings:
props:
- name: airspace_disease_grading
- name: class_covid19_pneumonia
- name: annotation_name
- name: midrc_mRALE_score
- name: annotation_method
- name: annotator_id
Expand All @@ -63,7 +64,6 @@ mappings:
- name: medication_code_system
- name: medication_manufacturer
- name: medication_name
- name: medication_status
- name: medication_type
- name: conditions
path: conditions
Expand All @@ -88,13 +88,13 @@ mappings:
- name: days_from_study_to_neg_covid_test
- name: days_from_study_to_pos_covid_test
- name: days_to_study
- name: image_data_modified
- name: loinc_code
- name: loinc_contrast
- name: loinc_long_common_name
- name: loinc_method
- name: loinc_system
- name: study_description
- name: study_location
- name: study_modality
- name: study_year
- name: study_uid
Expand All @@ -114,6 +114,18 @@ mappings:
- name: data_category
src: data_category
fn: set
- name: data_file_image_data_modified
src: image_data_modified
fn: set
- name: data_file_image_data_modification_name
src: image_data_modification_name
fn: set
- name: data_file_image_data_modification_method
src: image_data_modification_method
fn: set
- name: data_file_count
src: _data_file_id
fn: count
- name: midrc_measurement
doc_type: measurement
type: aggregator
Expand All @@ -126,6 +138,8 @@ mappings:
- name: test_method
- name: test_name
- name: test_result_text
parent_props:
- path: cases.datasets[dataset_submitter_id:submitter_id,license,data_url_doi,data_contributor]
- name: midrc_annotation
doc_type: annotation
type: aggregator
Expand All @@ -136,6 +150,7 @@ mappings:
- name: case_ids
- name: airspace_disease_grading
- name: class_covid19_pneumonia
- name: annotation_name
- name: midrc_mRALE_score
- name: annotation_method
- name: annotator_id
Expand All @@ -153,13 +168,13 @@ mappings:
- name: days_from_study_to_neg_covid_test
- name: days_from_study_to_pos_covid_test
- name: days_to_study
- name: image_data_modified
- name: loinc_code
- name: loinc_contrast
- name: loinc_long_common_name
- name: loinc_method
- name: loinc_system
- name: study_description
- name: study_location
- name: study_modality
- name: study_year
- name: study_year_shifted
Expand All @@ -170,6 +185,7 @@ mappings:
props:
- name: airspace_disease_grading
- name: class_covid19_pneumonia
- name: annotation_name
- name: midrc_mRALE_score
- name: annotation_method
- name: annotator_id
Expand All @@ -188,7 +204,7 @@ mappings:
path: mr_series_files
fn: count
parent_props:
- path: cases[sex,race,age_at_index,index_event,zip,covid19_positive,ethnicity].datasets[dataset_submitter_id:submitter_id]
- path: cases[sex,race,age_at_index,index_event,zip,covid19_positive,ethnicity].datasets[dataset_submitter_id:submitter_id,license,data_url_doi,data_contributor]
joining_props:
- index: data_file
join_on: _imaging_study_id
Expand All @@ -205,6 +221,24 @@ mappings:
- name: data_category
src: data_category
fn: set
- name: data_file_source_node
src: source_node
fn: set
- name: data_file_annotation_name
src: annotation_name
fn: set
- name: data_file_annotation_method
src: annotation_method
fn: set
- name: data_file_image_data_modified
src: image_data_modified
fn: set
- name: data_file_image_data_modification_name
src: image_data_modification_name
fn: set
- name: data_file_image_data_modification_method
src: image_data_modification_method
fn: set
- name: midrc_data_file
doc_type: data_file
type: collector
Expand Down Expand Up @@ -239,13 +273,17 @@ mappings:
- name: echo_time
- name: echo_train_length
- name: exposure_modulation_type
- name: image_data_modified
- name: image_data_modification_method
- name: image_data_modification_name
- name: image_type
- name: imaged_nucleus
- name: imager_pixel_spacing
- name: lossy_image_compression
- name: manufacturer
- name: manufacturer_model_name
- name: modality
- name: number_of_instances
- name: number_of_temporal_positions
- name: patient_position
- name: pixel_spacing
Expand All @@ -266,6 +304,15 @@ mappings:
- name: _dataset_id
src: id
fn: set
- name: license
src: license
fn: set
- name: data_url_doi
src: data_url_doi
fn: set
- name: data_contributor
src: data_contributor
fn: set
annotation:
props:
- name: _annotation_id
Expand Down Expand Up @@ -323,9 +370,6 @@ mappings:
- name: study_modality
src: study_modality
fn: set
- name: study_location
src: study_location
fn: set
- name: study_year
src: study_year
fn: set
Expand Down
37 changes: 19 additions & 18 deletions qa-midrc.planx-pla.net/manifest.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,26 +5,27 @@
],
"versions": {
"ambassador": "quay.io/datawire/ambassador:1.4.2",
"arborist": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/arborist:2023.12",
"arborist": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/arborist:2024.08",
"aws-es-proxy": "quay.io/cdis/aws-es-proxy:v1.3.1",
"dashboard": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/gen3-statics:2023.12",
"dicom-server": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/gen3-orthanc:gen3-0.1.1",
"fence": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/fence:2023.12",
"dashboard": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/gen3-statics:2024.08",
"dicom-server": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/gen3-orthanc:gen3-0.1.2",
"fence": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/fence:10.2.0",
"fluentd": "fluent/fluentd-kubernetes-daemonset:v1.15.3-debian-cloudwatch-1.0",
"guppy": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/guppy:2024.04",
"indexd": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/indexd:2023.12",
"manifestservice": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/manifestservice:2023.12",
"metadata": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/metadata-service:2023.12",
"ohif-viewer": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/ohif-viewer:gen3-v3.8.0",
"peregrine": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/peregrine:2023.12",
"portal": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/data-portal:5.24.0",
"frontend-framework": "quay.io/cdis/frontend-framework:feat_n3c",
"revproxy": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/nginx:2023.12",
"sheepdog": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/sheepdog:2023.12",
"spark": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/gen3-spark:2023.12",
"ssjdispatcher": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/ssjdispatcher:2023.12",
"tube": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/tube:2023.12",
"wts": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/workspace-token-service:2023.12"
"guppy": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/guppy:2024.08",
"indexd": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/indexd:2024.08",
"manifestservice": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/manifestservice:2024.08",
"metadata": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/metadata-service:2024.08",
"peregrine": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/peregrine:2024.08",
"portal": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/data-portal:5.33.0",
"revproxy": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/nginx:2024.08",
"sheepdog": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/sheepdog:2024.08",
"spark": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/gen3-spark:2024.08",
"ssjdispatcher": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/ssjdispatcher:2024.08",
"tube": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/tube:2024.08",
"wts": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/workspace-token-service:2024.08"


},
"arborist": {
"deployment_version": "2"
Expand Down Expand Up @@ -82,7 +83,7 @@
"tier_access_limit": "2",
"useryaml_s3path": "s3://cdis-gen3-users/qa/user.yaml",
"dispatcher_job_num": "10",
"dictionary_url": "http://s3.amazonaws.com/dictionary-artifacts/midrc_dictionary/1.2.5/schema.json",
"dictionary_url": "http://s3.amazonaws.com/dictionary-artifacts/midrc_dictionary/1.3.5/schema.json",
"sync_from_dbgap": "False",
"frontend_root": "portal",
"lb_type": "internal",
Expand Down
Loading