diff --git a/LabPurchasing/resources/queries/labpurchasing/purchases.js b/LabPurchasing/resources/queries/labpurchasing/purchases.js index 602ce37ff..cbcfa7d50 100644 --- a/LabPurchasing/resources/queries/labpurchasing/purchases.js +++ b/LabPurchasing/resources/queries/labpurchasing/purchases.js @@ -4,6 +4,31 @@ var console = require("console"); var triggerHelper = new org.labkey.labpurchasing.LabPurchasingTriggerHelper(LABKEY.Security.currentUser.id, LABKEY.Security.currentContainer.id); function beforeInsert(row, errors){ + beforeUpsert(row, errors) +} + +function beforeUpdate(row, oldRow, errors){ + beforeUpsert(row, errors) +} + +function beforeUpsert(row, errors){ + // Validate requestor: + if (row.requestor) { + if (!isNaN(row.requestor) && !triggerHelper.isValidUserId(row.requestor)) { + errors.requestor = 'Unknown userId for requestor: ' + row.requestor; + } + // Try to resolve strings: + else if (isNaN(row.requestor)) { + var id = triggerHelper.resolveUserId(String(row.requestor)); + if (!id) { + errors.requestor = 'Unknown userId for requestor: ' + row.requestor; + } + else { + row.requestor = id; + } + } + } + // The purpose of this is to allow the user to provide a string value for // vendorId or vendorName, and attempt to resolve this against known vendors: if (!row.vendorId || isNaN(row.vendorId)) { diff --git a/LabPurchasing/src/org/labkey/labpurchasing/LabPurchasingTriggerHelper.java b/LabPurchasing/src/org/labkey/labpurchasing/LabPurchasingTriggerHelper.java index 4e86852b8..169bded0b 100644 --- a/LabPurchasing/src/org/labkey/labpurchasing/LabPurchasingTriggerHelper.java +++ b/LabPurchasing/src/org/labkey/labpurchasing/LabPurchasingTriggerHelper.java @@ -112,4 +112,16 @@ public void sendNotifications(List rowIds) { _log.error("Unable to send purchasing email", e); } } + + public boolean isValidUserId(int userId) + { + return UserManager.getUser(userId) != null; + } + + public Integer resolveUserId(String userNameOrEmail) + { + User u = UserManager.getUserByDisplayName(userNameOrEmail); + + return u == null ? null : u.getUserId(); + } } diff --git a/mGAP/resources/etls/prime-seq.xml b/mGAP/resources/etls/prime-seq.xml index 25a9ac50a..8b2bcecc9 100644 --- a/mGAP/resources/etls/prime-seq.xml +++ b/mGAP/resources/etls/prime-seq.xml @@ -3,6 +3,13 @@ PRIMe-Seq ETLs Syncs Anonymized Data PRIMe-Seq to mGAP + + + + + + + Copy to local table @@ -190,6 +197,14 @@ + + + + + + + + diff --git a/mGAP/src/org/labkey/mgap/columnTransforms/AbstractVariantTransform.java b/mGAP/src/org/labkey/mgap/columnTransforms/AbstractVariantTransform.java index 9490c39e6..5345e4fc3 100644 --- a/mGAP/src/org/labkey/mgap/columnTransforms/AbstractVariantTransform.java +++ b/mGAP/src/org/labkey/mgap/columnTransforms/AbstractVariantTransform.java @@ -1,6 +1,5 @@ package org.labkey.mgap.columnTransforms; -import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.labkey.api.collections.CaseInsensitiveHashMap; import org.labkey.api.data.Results; @@ -21,13 +20,12 @@ import org.labkey.api.query.QueryService; import org.labkey.api.util.FileUtil; import org.labkey.api.util.PageFlowUtil; +import org.labkey.mgap.etl.EtlQueueManager; import org.labkey.mgap.mGAPManager; import java.io.File; -import java.io.IOException; import java.net.URI; import java.sql.SQLException; -import java.util.Arrays; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -107,7 +105,7 @@ protected Integer getOrCreateOutputFile(Object dataFileUrl, Object folderName, S { if (dataFileUrl == null) { - throw new IllegalArgumentException("DataFileUrl was null."); + throw new IllegalArgumentException("DataFileUrl was null"); } URI uri = new URI(String.valueOf(dataFileUrl)); @@ -215,20 +213,8 @@ protected File doFileCopy(File f, File subdir, String name) throws PipelineJobEx if (doCopy) { - getStatusLogger().info("copying file locally: " + localCopy.getPath()); - if (localCopy.exists()) - { - localCopy.delete(); - } - - try - { - FileUtils.copyFile(f, localCopy); - } - catch (IOException e) - { - throw new PipelineJobException(e); - } + getStatusLogger().info("queueing file copy: " + localCopy.getPath()); + EtlQueueManager.get().queueFileCopy(getContainerUser().getContainer(), f, localCopy); } File index = new File(f.getPath() + ".tbi"); @@ -243,15 +229,8 @@ protected File doFileCopy(File f, File subdir, String name) throws PipelineJobEx if (!indexLocal.exists()) { - getStatusLogger().info("copying index locally: " + indexLocal.getPath()); - try - { - FileUtils.copyFile(index, indexLocal); - } - catch (IOException e) - { - throw new PipelineJobException(e); - } + getStatusLogger().info("queueing copy of index: " + indexLocal.getPath()); + EtlQueueManager.get().queueFileCopy(getContainerUser().getContainer(), index, indexLocal); } } diff --git a/mGAP/src/org/labkey/mgap/columnTransforms/ExpDataTransform.java b/mGAP/src/org/labkey/mgap/columnTransforms/ExpDataTransform.java deleted file mode 100644 index 0c6a66ba0..000000000 --- a/mGAP/src/org/labkey/mgap/columnTransforms/ExpDataTransform.java +++ /dev/null @@ -1,50 +0,0 @@ -package org.labkey.mgap.columnTransforms; - -import org.labkey.api.di.columnTransform.ColumnTransform; -import org.labkey.api.exp.api.DataType; -import org.labkey.api.exp.api.ExpData; -import org.labkey.api.exp.api.ExperimentService; - -import java.io.File; -import java.net.URI; -import java.net.URISyntaxException; - -/** - * Created by bimber on 5/1/2017. - */ -public class ExpDataTransform extends ColumnTransform -{ - @Override - protected Object doTransform(Object inputValue) - { - if (null == inputValue) - return null; - - try - { - URI uri = new URI(String.valueOf(inputValue)); - File f = new File(uri); - if (!f.exists()) - { - getStatusLogger().error("File not found: " + uri); - } - - ExpData d = ExperimentService.get().getExpDataByURL(String.valueOf(inputValue), getContainerUser().getContainer()); - if (d == null) - { - d = ExperimentService.get().createData(getContainerUser().getContainer(), new DataType("Variant Catalog")); - d.setDataFileURI(uri); - d.setName(f.getName()); - d.save(getContainerUser().getUser()); - } - - return d.getRowId(); - } - catch (URISyntaxException e) - { - getStatusLogger().error("Error syncing file: " + inputValue, e); - } - - return null; - } -} diff --git a/mGAP/src/org/labkey/mgap/columnTransforms/LuceneIndexTransform.java b/mGAP/src/org/labkey/mgap/columnTransforms/LuceneIndexTransform.java index 14d4f8c03..35e9787db 100644 --- a/mGAP/src/org/labkey/mgap/columnTransforms/LuceneIndexTransform.java +++ b/mGAP/src/org/labkey/mgap/columnTransforms/LuceneIndexTransform.java @@ -4,6 +4,7 @@ import org.jetbrains.annotations.Nullable; import org.labkey.api.pipeline.PipelineJobException; import org.labkey.api.sequenceanalysis.run.SimpleScriptWrapper; +import org.labkey.mgap.etl.EtlQueueManager; import java.io.File; import java.io.IOException; @@ -26,12 +27,7 @@ protected File doFileCopy(File f, File subdir, @Nullable String name) throws Pip // NOTE: lucene is a special case since the DB tracks one file, but we need this whole folder: File sourceDir = f.getParentFile(); File targetDir = new File(subdir, "LuceneIndex"); - - // NOTE: rsync should no-op if there are no source changes - getStatusLogger().info("Copying lucene index dir to: " + targetDir.getPath()); - new SimpleScriptWrapper(getStatusLogger()).execute(Arrays.asList( - "rsync", "-r", "-a", "--delete", "--no-owner", "--no-group", "--chmod=D2770,F660", sourceDir.getPath(), targetDir.getPath() - )); + EtlQueueManager.get().queueRsyncCopy(getContainerUser().getContainer(), sourceDir, targetDir); return new File(targetDir, sourceDir.getName() + "/" + f.getName()); } diff --git a/mGAP/src/org/labkey/mgap/etl/ClearEtlWorkQueueTask.java b/mGAP/src/org/labkey/mgap/etl/ClearEtlWorkQueueTask.java new file mode 100644 index 000000000..4ee9f0e1d --- /dev/null +++ b/mGAP/src/org/labkey/mgap/etl/ClearEtlWorkQueueTask.java @@ -0,0 +1,43 @@ +package org.labkey.mgap.etl; + +import org.apache.xmlbeans.XmlException; +import org.jetbrains.annotations.NotNull; +import org.labkey.api.di.TaskRefTask; +import org.labkey.api.pipeline.PipelineJob; +import org.labkey.api.pipeline.PipelineJobException; +import org.labkey.api.pipeline.RecordedActionSet; +import org.labkey.api.writer.ContainerUser; + +import java.util.List; +import java.util.Map; + +public class ClearEtlWorkQueueTask implements TaskRefTask +{ + private ContainerUser _containerUser = null; + + @Override + public RecordedActionSet run(@NotNull PipelineJob job) throws PipelineJobException + { + EtlQueueManager.get().clearQueue(_containerUser.getContainer(), job.getLogger()); + + return new RecordedActionSet(); + } + + @Override + public List getRequiredSettings() + { + return null; + } + + @Override + public void setSettings(Map settings) throws XmlException + { + + } + + @Override + public void setContainerUser(ContainerUser containerUser) + { + _containerUser = containerUser; + } +} diff --git a/mGAP/src/org/labkey/mgap/etl/EtlQueueManager.java b/mGAP/src/org/labkey/mgap/etl/EtlQueueManager.java new file mode 100644 index 000000000..950454383 --- /dev/null +++ b/mGAP/src/org/labkey/mgap/etl/EtlQueueManager.java @@ -0,0 +1,122 @@ +package org.labkey.mgap.etl; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.logging.log4j.Logger; +import org.labkey.api.data.Container; +import org.labkey.api.pipeline.PipelineJobException; +import org.labkey.api.sequenceanalysis.run.SimpleScriptWrapper; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class EtlQueueManager +{ + private static EtlQueueManager _instance = new EtlQueueManager(); + + private Map>> _pendingFileCopy = new HashMap<>(); + + private Map>> _pendingRsyncCopy = new HashMap<>(); + + private EtlQueueManager() + { + + } + + public static EtlQueueManager get() + { + return _instance; + } + + public void clearQueue(Container container, Logger log) + { + if (_pendingFileCopy.containsKey(container) && !_pendingFileCopy.get(container).isEmpty()) + { + log.error("The file copy queue was not empty!"); + } + + if (_pendingRsyncCopy.containsKey(container) && !_pendingRsyncCopy.get(container).isEmpty()) + { + log.error("The rsync copy queue was not empty!"); + } + + _pendingFileCopy.clear(); + _pendingRsyncCopy.clear(); + } + + public void performQueuedWork(Container container, Logger log) + { + List> queue = _pendingFileCopy.get(container); + if (queue != null && !queue.isEmpty()) + { + queue.forEach(x -> copyFile(x.getLeft(), x.getRight(), log)); + } + _pendingFileCopy.clear(); + + List> rsyncQueue = _pendingRsyncCopy.get(container); + if (rsyncQueue != null && !rsyncQueue.isEmpty()) + { + rsyncQueue.forEach(x -> doRsyncCopy(x.getLeft(), x.getRight(), log)); + } + _pendingRsyncCopy.clear(); + } + + public void queueFileCopy(Container c, File source, File destination) + { + if (!_pendingFileCopy.containsKey(c)) + { + _pendingFileCopy.put(c, new ArrayList<>()); + } + + _pendingFileCopy.get(c).add(Pair.of(source, destination)); + } + + public void queueRsyncCopy(Container c, File source, File destination) + { + if (!_pendingRsyncCopy.containsKey(c)) + { + _pendingRsyncCopy.put(c, new ArrayList<>()); + } + + _pendingRsyncCopy.get(c).add(Pair.of(source, destination)); + } + + private void doRsyncCopy(File sourceDir, File destination, Logger log) + { + // NOTE: rsync should no-op if there are no source changes + log.info("Performing rsync from: " + sourceDir.getPath() + " to " + destination.getPath()); + try + { + new SimpleScriptWrapper(log).execute(Arrays.asList( + "rsync", "-r", "-a", "--delete", "--no-owner", "--no-group", "--chmod=D2770,F660", sourceDir.getPath(), destination.getPath() + )); + } + catch (PipelineJobException e) + { + log.error("Error running rsync", e); + } + } + + private void copyFile(File source, File destination, Logger log) + { + if (destination.exists()) + { + destination.delete(); + } + + try + { + log.info("Copying file: " + source.getPath()); + FileUtils.copyFile(source, destination); + } + catch (IOException e) + { + log.error("Error copying file", e); + } + } +} diff --git a/mGAP/src/org/labkey/mgap/etl/PerformQueuedEtlWorkTask.java b/mGAP/src/org/labkey/mgap/etl/PerformQueuedEtlWorkTask.java new file mode 100644 index 000000000..712b53970 --- /dev/null +++ b/mGAP/src/org/labkey/mgap/etl/PerformQueuedEtlWorkTask.java @@ -0,0 +1,42 @@ +package org.labkey.mgap.etl; + +import org.apache.xmlbeans.XmlException; +import org.jetbrains.annotations.NotNull; +import org.labkey.api.di.TaskRefTask; +import org.labkey.api.pipeline.PipelineJob; +import org.labkey.api.pipeline.PipelineJobException; +import org.labkey.api.pipeline.RecordedActionSet; +import org.labkey.api.writer.ContainerUser; + +import java.util.List; +import java.util.Map; + +public class PerformQueuedEtlWorkTask implements TaskRefTask +{ + private ContainerUser _containerUser = null; + + @Override + public RecordedActionSet run(@NotNull PipelineJob job) throws PipelineJobException + { + EtlQueueManager.get().performQueuedWork(_containerUser.getContainer(), job.getLogger()); + return new RecordedActionSet(); + } + + @Override + public List getRequiredSettings() + { + return null; + } + + @Override + public void setSettings(Map settings) throws XmlException + { + + } + + @Override + public void setContainerUser(ContainerUser containerUser) + { + _containerUser = containerUser; + } +} diff --git a/mGAP/src/org/labkey/mgap/pipeline/IndexVariantsForMgapStep.java b/mGAP/src/org/labkey/mgap/pipeline/IndexVariantsForMgapStep.java index c2aad3905..9c4f6207a 100644 --- a/mGAP/src/org/labkey/mgap/pipeline/IndexVariantsForMgapStep.java +++ b/mGAP/src/org/labkey/mgap/pipeline/IndexVariantsForMgapStep.java @@ -4,14 +4,19 @@ import org.apache.commons.lang3.StringUtils; import org.jetbrains.annotations.Nullable; import org.json.JSONObject; +import org.labkey.api.collections.CaseInsensitiveHashMap; import org.labkey.api.data.Container; import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.TableInfo; import org.labkey.api.data.TableSelector; import org.labkey.api.jbrowse.JBrowseService; import org.labkey.api.pipeline.PipelineJob; import org.labkey.api.pipeline.PipelineJobException; +import org.labkey.api.query.BatchValidationException; import org.labkey.api.query.FieldKey; +import org.labkey.api.query.InvalidKeyException; import org.labkey.api.query.QueryService; +import org.labkey.api.query.QueryUpdateServiceException; import org.labkey.api.security.User; import org.labkey.api.sequenceanalysis.SequenceOutputFile; import org.labkey.api.sequenceanalysis.pipeline.AbstractVariantProcessingStepProvider; @@ -28,8 +33,11 @@ import org.labkey.mgap.mGAPSchema; import java.io.File; +import java.sql.SQLException; import java.util.Arrays; +import java.util.Collections; import java.util.List; +import java.util.Map; public class IndexVariantsForMgapStep extends AbstractCommandPipelineStep implements VariantProcessingStep { @@ -93,8 +101,61 @@ public Output processVariants(File inputVCF, File outputDirectory, ReferenceGeno throw new PipelineJobException("Unable to find file: " + idx.getPath()); } - output.addSequenceOutput(idx, "mGAP Lucene Index: " + releaseVersion, CATEGORY, null, null, genome.getGenomeId(), "Fields indexed: " + infoFieldsRaw); + output.addSequenceOutput(idx, "mGAP Lucene Index: " + releaseVersion, CATEGORY, null, null, genome.getGenomeId(), "Fields indexed: " + infoFields.size()); return output; } + + @Override + public void complete(PipelineJob job, List inputs, List outputsCreated, SequenceAnalysisJobSupport support) throws PipelineJobException + { + String releaseVersion = getProvider().getParameterByName("releaseVersion").extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), String.class); + + List of = outputsCreated.stream().filter(x -> CATEGORY.equals(x.getCategory())).toList(); + if (of.size() != 1) + { + throw new PipelineJobException("Expected a single output, found: " + of.size()); + } + + Container target = job.getContainer().isWorkbook() ? job.getContainer().getParent() : job.getContainer(); + TableInfo ti = QueryService.get().getUserSchema(job.getUser(), target, mGAPSchema.NAME).getTable(mGAPSchema.TABLE_VARIANT_CATALOG_RELEASES); + TableSelector ts = new TableSelector(ti, PageFlowUtil.set("rowid", "container"), new SimpleFilter(FieldKey.fromString("version"), releaseVersion), null); + if (ts.exists()) + { + if (ts.getRowCount() > 1) + { + throw new IllegalStateException("More than one row found matching: " + releaseVersion); + } + + job.getLogger().info("Updating release record"); + Map row = new CaseInsensitiveHashMap<>(ts.getMap()); + if (!row.containsKey("rowid") || row.get("rowid") == null) + { + job.getLogger().error("Missing rowId, found: "); + for (String key : row.keySet()) + { + job.getLogger().debug(key + ": " + row.get(key)); + } + + throw new IllegalStateException("Missing rowId from release record"); + } + + row.put("luceneIndex", of.get(0).getRowid()); + + try + { + BatchValidationException bve = new BatchValidationException(); + Map oldKeys = new CaseInsensitiveHashMap<>(Map.of("rowid", row.get("rowid"))); + ti.getUpdateService().updateRows(job.getUser(), target, Collections.singletonList(row), Collections.singletonList(oldKeys), bve, null, null); + } + catch (BatchValidationException | InvalidKeyException | QueryUpdateServiceException | SQLException e) + { + throw new PipelineJobException(e); + } + } + else + { + job.getLogger().info("No release record found, will not update"); + } + } } diff --git a/mcc/resources/etls/snprc-datasets.xml b/mcc/resources/etls/snprc-datasets.xml index 981a4855b..401ebba93 100644 --- a/mcc/resources/etls/snprc-datasets.xml +++ b/mcc/resources/etls/snprc-datasets.xml @@ -136,6 +136,10 @@ weight objectid + + + + diff --git a/mcc/resources/queries/study/demographics.js b/mcc/resources/queries/study/demographics.js index c1dc87412..1b0016228 100644 --- a/mcc/resources/queries/study/demographics.js +++ b/mcc/resources/queries/study/demographics.js @@ -110,6 +110,10 @@ function onUpsert(helper, scriptErrors, row, oldRow){ idToMccAlias[row.sire] = row.sireMccAlias; } } + + if (!row.date) { + row.date = new Date(); + } } function onComplete(event, errors, helper){ diff --git a/mcc/resources/views/begin.html b/mcc/resources/views/begin.html index bc85b030f..14d820175 100644 --- a/mcc/resources/views/begin.html +++ b/mcc/resources/views/begin.html @@ -7,7 +7,7 @@ return; Ext4.get(webpart.wrapperDivId).update( - 'This is the future home of the BRAIN Initiative Marmoset Coordinating Center. This site is currently under active development - please come back soon!' + + 'This is the home of the BRAIN Initiative Marmoset Coordinating Center. Please use the links below to access the main website features:' + '

' + '' + '
' + diff --git a/mcc/resources/web/mcc/exampleData/MCC_Data_Template.xlsx b/mcc/resources/web/mcc/exampleData/MCC_Data_Template.xlsx index ae2622e21..916878789 100644 Binary files a/mcc/resources/web/mcc/exampleData/MCC_Data_Template.xlsx and b/mcc/resources/web/mcc/exampleData/MCC_Data_Template.xlsx differ diff --git a/mcc/resources/web/mcc/panel/MccClinicalSnapshotPanel.js b/mcc/resources/web/mcc/panel/MccClinicalSnapshotPanel.js new file mode 100644 index 000000000..f50b1f896 --- /dev/null +++ b/mcc/resources/web/mcc/panel/MccClinicalSnapshotPanel.js @@ -0,0 +1,77 @@ +EHR.reports.clinicalHistoryPanelXtype = 'mcc-snapshotpanel'; + +Ext4.define('MCC.panel.SnapshotPanel', { + extend: 'EHR.panel.SnapshotPanel', + alias: 'widget.mcc-snapshotpanel', + + showLocationDuration: false, + + minWidth: 800, + + initComponent: function () { + + this.defaultLabelWidth = 120; + this.callParent(); + }, + + getBaseItems: function(){ + return [{ + xtype: 'container', + border: false, + defaults: { + border: false + }, + items: [{ + xtype: 'container', + html: 'Summary:
' + },{ + bodyStyle: 'padding-bottom: 20px;', + layout: 'column', + defaults: { + border: false + }, + items: [{ + xtype: 'container', + columnWidth: 0.25, + defaults: { + labelWidth: this.defaultLabelWidth, + style: 'margin-right: 20px;' + }, + items: [{ + xtype: 'displayfield', + fieldLabel: 'Status', + name: 'calculated_status' + },{ + xtype: 'displayfield', + fieldLabel: 'Gender', + name: 'gender' + },{ + xtype: 'displayfield', + fieldLabel: 'Species', + name: 'species' + }] + },{ + xtype: 'container', + columnWidth: 0.25, + defaults: { + labelWidth: this.defaultLabelWidth, + style: 'margin-right: 20px;' + }, + items: [{ + xtype: 'displayfield', + fieldLabel: 'Age', + name: 'age' + }, { + xtype: 'displayfield', + fieldLabel: 'Source', + name: 'source' + },{ + xtype: 'displayfield', + fieldLabel: 'Weights', + name: 'weights' + }] + }] + }] + }]; + } +}); \ No newline at end of file diff --git a/mcc/resources/web/mcc/panel/MccImportPanel.js b/mcc/resources/web/mcc/panel/MccImportPanel.js index 43c0bc4d1..705e85744 100644 --- a/mcc/resources/web/mcc/panel/MccImportPanel.js +++ b/mcc/resources/web/mcc/panel/MccImportPanel.js @@ -33,7 +33,8 @@ Ext4.define('MCC.panel.MccImportPanel', { allowRowSpan: false, alwaysShow: true, transform: 'animalId', - allowBlank: false + allowBlank: false, + expectInImport: true },{ name: 'alternateIds', labels: ['Alternate Ids', 'previous Ids'], @@ -46,7 +47,8 @@ Ext4.define('MCC.panel.MccImportPanel', { labels: ['Source Colony', 'Source'], allowRowSpan: false, allowBlank: true, - alwaysShow: true + alwaysShow: true, + expectInImport: true },{ name: 'shippingDestination', labels: ['Shipping Destination'], @@ -65,13 +67,15 @@ Ext4.define('MCC.panel.MccImportPanel', { labels: ['Birth', 'DOB', 'DOB (mm/dd/yyyy)'], allowRowSpan: false, allowBlank: true, - transform: 'genericDate' + transform: 'genericDate', + expectInImport: true },{ name: 'gender', labels: ['Sex'], allowRowSpan: false, allowBlank: true, - transform: 'sex' + transform: 'sex', + expectInImport: true },{ name: 'status', labels: ['Status'], @@ -84,26 +88,32 @@ Ext4.define('MCC.panel.MccImportPanel', { labels: ['Dam', 'maternal ID'], allowRowSpan: false, allowBlank: true, - transform: 'damOrSire' + alwaysShow: true, + transform: 'damOrSire', + expectInImport: true },{ name: 'sire', labels: ['Sire', 'paternal ID'], allowRowSpan: false, allowBlank: true, - transform: 'damOrSire' + alwaysShow: true, + transform: 'damOrSire', + expectInImport: true },{ name: 'weight', - labels: ['Weight (g)', 'Weight (grams)'], + labels: ['Weight (g)', 'Weight (grams)', 'current weight (g)', 'current weight (grams)'], allowRowSpan: false, allowBlank: true, - transform: 'weight' + transform: 'weight', + expectInImport: true },{ name: 'weightDate', labels: ['Date of Weight', 'date of weight', 'date of weight (mm/dd/yyyy)'], alwaysShow: true, allowRowSpan: false, transform: 'genericDate', - allowBlank: true + allowBlank: true, + expectInImport: true },{ name: 'date', labels: ['Observation Date', 'date'], @@ -117,13 +127,16 @@ Ext4.define('MCC.panel.MccImportPanel', { alwaysShow: false, allowRowSpan: false, allowBlank: false, - transform: 'u24' + transform: 'u24', + expectInImport: true },{ name: 'availability', - labels: ['Available to Transfer', 'available to transfer'], + // NOTE: availalble was a typo in one generation of the input templates: + labels: ['Available to Transfer', 'available to transfer', 'availalble to transfer'], allowRowSpan: false, allowBlank: true, - transform: 'available' + transform: 'available', + expectInImport: true },{ name: 'breedingPartnerId', labels: ['Breeding Partner Id'], @@ -134,25 +147,29 @@ Ext4.define('MCC.panel.MccImportPanel', { labels: ['Current Housing Status'], allowRowSpan: false, allowBlank: true, - transform: 'housingStatus' + transform: 'housingStatus', + expectInImport: true },{ name: 'infantHistory', labels: ['Infant History'], allowRowSpan: false, alwaysShow: true, - transform: 'infantHistory' + transform: 'infantHistory', + expectInImport: true },{ name: 'fertilityStatus', labels: ['Fertility Status'], allowRowSpan: false, alwaysShow: true, - transform: 'fertilityStatus' + transform: 'fertilityStatus', + expectInImport: true },{ name: 'medicalHistory', labels: ['Medical History'], allowRowSpan: false, allowBlank: true, - transform: 'medicalHistory' + transform: 'medicalHistory', + expectInImport: true },{ name: 'currentUsage', labels: ['Usage (Current)', 'usage (current)'], @@ -251,6 +268,10 @@ Ext4.define('MCC.panel.MccImportPanel', { sex: function(val, panel, row) { val = panel.stripLeadingNumbers(val); + if (val && val.toLowerCase() === 'tbd') { + val = 'unknown'; + } + val = panel.enforceAllowableValues(val, ['male', 'female', 'unknown'], row); return(val); @@ -328,6 +349,14 @@ Ext4.define('MCC.panel.MccImportPanel', { row.errors.push('Suspicious weight value'); } + return val; + }, + + alternateIds: function(val) { + if (val) { + val = val.split(/[ ]*[;,]+[ ]*/g).join(',') + } + return val; } }, @@ -433,6 +462,16 @@ Ext4.define('MCC.panel.MccImportPanel', { var rows = LDK.Utils.CSVToArray(text, '\t'); var colArray = this.parseHeader(rows.shift()); + var foundColsNames = colArray.map(x => x.name) + var missingExpectedCols = this.COLUMNS.filter(x => x.expectInImport).map(x => x.name).filter(x => foundColsNames.indexOf(x) === -1) + if (missingExpectedCols.length) { + // now convert from name to label: + var colNameToLabel = {} + this.COLUMNS.forEach(x => colNameToLabel[x.name] = x.labels[0]) + Ext4.Msg.alert('Error', 'The following columns were expected but not found:
' + missingExpectedCols.map(x => colNameToLabel[x]).join('
')) + return null; + } + var errorsMsgs = []; var parsedRows = this.parseRows(colArray, rows, errorsMsgs); @@ -490,7 +529,7 @@ Ext4.define('MCC.panel.MccImportPanel', { else { row.objectId = existingRecord.objectid; - var fields = ['birth', 'dam', 'sire', 'source', 'alternateIds']; + var fields = ['birth', 'dam', 'sire', 'source']; for (var idx in fields) { var fn = fields[idx]; @@ -498,10 +537,18 @@ Ext4.define('MCC.panel.MccImportPanel', { if (fn === 'birth' && existingRecord[fn]) { existingRecord[fn] = Ext4.Date.format(LDK.ConvertUtils.parseDate(existingRecord[fn]), 'Y-m-d'); } + if (row[fn] && existingRecord[fn] && row[fn] !== existingRecord[fn]) { row.errors.push('Does not match existing row for ' + fn + ': ' + existingRecord[fn]); } } + + // The goal of this is to take the union of the existing/new aliases: + if (row.alternateIds && existingRecord.alternateIds) { + row.alternateIds = row.alternateIds.split(/[ ]*[;,]+[ ]*/g) + existingRecord.alternateIds = existingRecord.alternateIds.split(/[ ]*[;,]+[ ]*/g) + row.alternateIds = Ext4.unique(row.alternateIds.concat(existingRecord.alternateIds)).sort().join(',') + } } if (existingRecord['Id/death/date']) { @@ -1215,7 +1262,7 @@ Ext4.define('MCC.panel.MccImportPanel', { commands.push({ command: 'insert', schemaName: 'study', - queryName: 'departures', + queryName: 'departure', rows: departureInserts }); } diff --git a/mcc/src/org/labkey/mcc/MccModule.java b/mcc/src/org/labkey/mcc/MccModule.java index c7c4fe169..a970d6a08 100644 --- a/mcc/src/org/labkey/mcc/MccModule.java +++ b/mcc/src/org/labkey/mcc/MccModule.java @@ -31,7 +31,9 @@ import org.labkey.api.security.roles.RoleManager; import org.labkey.api.util.SystemMaintenance; import org.labkey.api.view.WebPartFactory; +import org.labkey.api.view.template.ClientDependency; import org.labkey.api.writer.ContainerUser; +import org.labkey.mcc.ehr.NoOpClinicalHistorySource; import org.labkey.mcc.query.MarkShippedButton; import org.labkey.mcc.query.MccEhrCustomizer; import org.labkey.mcc.query.RenameIdButton; @@ -125,6 +127,9 @@ private void registerEHRResources() EHRService.get().registerMoreActionsButton(new MarkShippedButton(), "study", "demographics"); EHRService.get().registerMoreActionsButton(new RenameIdButton(), "study", "demographics"); LDKService.get().registerQueryButton(new ShowEditUIButton(this, MccSchema.NAME, MccSchema.TABLE_CENSUS), MccSchema.NAME, MccSchema.TABLE_CENSUS); + + EHRService.get().registerHistoryDataSource(new NoOpClinicalHistorySource("Case Opened")); + EHRService.get().registerClientDependency(ClientDependency.supplierFromPath("mcc/panel/MccClinicalSnapshotPanel.js"), this); } @Override diff --git a/mcc/src/org/labkey/mcc/ehr/NoOpClinicalHistorySource.java b/mcc/src/org/labkey/mcc/ehr/NoOpClinicalHistorySource.java new file mode 100644 index 000000000..13629ad94 --- /dev/null +++ b/mcc/src/org/labkey/mcc/ehr/NoOpClinicalHistorySource.java @@ -0,0 +1,44 @@ +package org.labkey.mcc.ehr; + +import org.jetbrains.annotations.NotNull; +import org.labkey.api.data.Container; +import org.labkey.api.data.Results; +import org.labkey.api.data.TableInfo; +import org.labkey.api.ehr.history.AbstractDataSource; +import org.labkey.api.ehr.history.HistoryRow; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.security.User; +import org.labkey.mcc.MccModule; + +import java.sql.SQLException; +import java.util.Collections; +import java.util.Date; +import java.util.List; + +public class NoOpClinicalHistorySource extends AbstractDataSource +{ + public NoOpClinicalHistorySource(String categoryText) + { + super(null, null, categoryText, ModuleLoader.getInstance().getModule(MccModule.class)); + + } + + @Override + protected TableInfo getTableInfo(Container c, User u) + { + throw new UnsupportedOperationException("This should never be called"); + } + + @Override + @NotNull + public List getRows(Container c, User u, final String subjectId, Date minDate, Date maxDate, boolean redacted) + { + return Collections.emptyList(); + } + + @Override + protected String getHtml(Container c, Results rs, boolean redacted) throws SQLException + { + return null; + } +} diff --git a/mcc/test/src/org/labkey/test/tests/mcc/MccTest.java b/mcc/test/src/org/labkey/test/tests/mcc/MccTest.java index b1da3960d..89d94101c 100644 --- a/mcc/test/src/org/labkey/test/tests/mcc/MccTest.java +++ b/mcc/test/src/org/labkey/test/tests/mcc/MccTest.java @@ -68,7 +68,7 @@ public void testMccModule() throws Exception testAnimalImportAndTransfer(); } - private static final String ANIMAL_DATA_HEADER = "animal ID\tprevious IDs\tsource\t\"DOB\n(MM/DD/YYYY)\"\tsex\tmaternal ID\tpaternal ID\t\"weight(grams)\"\t\"date of weight\n(MM/DD/YY)\"\tU24 status\tavailable to transfer\tcurrent housing status\tinfant history\tfertility status\tmedical history\n"; + private static final String ANIMAL_DATA_HEADER = "animal ID\tprevious IDs\tsource\t\"DOB\n(MM/DD/YYYY)\"\tsex\tmaternal ID\tpaternal ID\t\"weight (grams)\"\t\"date of weight\n(MM/DD/YY)\"\tU24 status\tavailable to transfer\tcurrent housing status\tinfant history\tfertility status\tmedical history\n"; private static final String ANIMAL_DATA1 = "12345\t\t\t7/10/2011\t0 - male\t23456\t23453\t382.8\t5/19/2021\t0 - not assigned to U24 breeding colony\t0 - not available for transfer\t1 - natal family unit\t3 - successful rearing of offspring\t2 - successful offspring produced\t0 - naive animal\n"; diff --git a/primeseq/resources/etls/mcc.xml b/primeseq/resources/etls/mcc.xml new file mode 100644 index 000000000..2277dbbc5 --- /dev/null +++ b/primeseq/resources/etls/mcc.xml @@ -0,0 +1,32 @@ + + + MCC_Data + MCC Demographics and Pedigree Data + + + Copy to target + + + Id + dam + sire + gender + + + + + + + + + + + + + + + + + + +