diff --git a/GenotypeAssays/src/org/labkey/genotypeassays/AlignmentDisplayColumn.java b/GenotypeAssays/src/org/labkey/genotypeassays/AlignmentDisplayColumn.java deleted file mode 100644 index 816b45278..000000000 --- a/GenotypeAssays/src/org/labkey/genotypeassays/AlignmentDisplayColumn.java +++ /dev/null @@ -1,95 +0,0 @@ -package org.labkey.genotypeassays; - -import org.apache.commons.lang3.StringUtils; -import org.jetbrains.annotations.NotNull; -import org.labkey.api.data.ColumnInfo; -import org.labkey.api.data.CompareType; -import org.labkey.api.data.DataColumn; -import org.labkey.api.data.DisplayColumn; -import org.labkey.api.data.DisplayColumnFactory; -import org.labkey.api.data.RenderContext; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.query.FieldKey; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.view.template.ClientDependency; - -import java.io.IOException; -import java.io.Writer; -import java.util.Arrays; -import java.util.Set; - -/** - * Created by bimber on 9/8/2016. - */ -public class AlignmentDisplayColumn extends DataColumn -{ - public AlignmentDisplayColumn(ColumnInfo columnInfo) - { - super(columnInfo); - } - - public static class Factory implements DisplayColumnFactory - { - public Factory() - { - - } - - @Override - public DisplayColumn createRenderer(ColumnInfo colInfo) - { - return new AlignmentDisplayColumn(colInfo); - } - } - - @Override - public void renderGridCellContents(RenderContext ctx, Writer out) throws IOException - { - Object key = getValue(ctx); - if (key != null) - { - out.write(""); - out.write("Edit Alignments"); - SimpleFilter filter = new SimpleFilter(); - filter.addUrlFilters(ctx.getSortFilterURLHelper(), ctx.getCurrentRegion().getName()); - FieldKey pctFk = new FieldKey(getBoundColumn().getFieldKey().getParent(), "percent_from_locus"); - Object filterVal = null; - String filterOperator = null; - for (SimpleFilter.FilterClause fc : filter.getClauses()) - { - if (fc.getFieldKeys().contains(pctFk)) - { - if (fc instanceof CompareType.CompareClause && fc.getParamVals() != null) - { - filterOperator = ((CompareType.CompareClause)fc).getCompareType().getPreferredUrlKey(); - filterVal = StringUtils.join(Arrays.asList(fc.getParamVals()), ";"); - } - } - } - - String lineages = ctx.get(new FieldKey(getBoundColumn().getFieldKey().getParent(), "lineages"), String.class); - Integer analysisId = ctx.get(new FieldKey(getBoundColumn().getFieldKey().getParent(), "analysis_id"), Integer.class); - if (lineages != null) - { - out.write("
"); - out.write("Edit Lineage"); - } - } - } - - @Override - public void addQueryFieldKeys(Set keys) - { - super.addQueryFieldKeys(keys); - - keys.add(new FieldKey(getBoundColumn().getFieldKey().getParent(), "lineages")); - keys.add(new FieldKey(getBoundColumn().getFieldKey().getParent(), "analysis_id")); - } - - @Override - public @NotNull - Set getClientDependencies() - { - return PageFlowUtil.set(ClientDependency.fromPath("SequenceAnalysis/sequenceAnalysis"), ClientDependency.fromPath("genotypeassays/window/EditAlignmentsWindow.js")); - } -} diff --git a/LabPurchasing/resources/queries/labpurchasing/purchases.js b/LabPurchasing/resources/queries/labpurchasing/purchases.js index dfb59f997..602ce37ff 100644 --- a/LabPurchasing/resources/queries/labpurchasing/purchases.js +++ b/LabPurchasing/resources/queries/labpurchasing/purchases.js @@ -3,6 +3,30 @@ var console = require("console"); var triggerHelper = new org.labkey.labpurchasing.LabPurchasingTriggerHelper(LABKEY.Security.currentUser.id, LABKEY.Security.currentContainer.id); +function beforeInsert(row, errors){ + // The purpose of this is to allow the user to provide a string value for + // vendorId or vendorName, and attempt to resolve this against known vendors: + if (!row.vendorId || isNaN(row.vendorId)) { + var vendorName = row.vendorName || row.vendorId; + if (vendorName) { + LABKEY.Query.selectRows({ + schemaName: 'labpurchasing', + queryName: 'vendors', + columns: 'rowid', + filterArray: [LABKEY.Filter.create('vendorName', vendorName)], + sort: '-rowid', + maxRows: 1, + scope: this, + success: function(results) { + if (results.rows.length) { + row.vendorId = results.rows[0].rowId; + } + } + }) + } + } +} + function afterInsert(row, errors){ afterUpsert(row, null, errors); } diff --git a/LabPurchasing/resources/queries/labpurchasing/referenceItems.js b/LabPurchasing/resources/queries/labpurchasing/referenceItems.js index 4848cdbcc..8a6d70e73 100644 --- a/LabPurchasing/resources/queries/labpurchasing/referenceItems.js +++ b/LabPurchasing/resources/queries/labpurchasing/referenceItems.js @@ -19,9 +19,6 @@ function beforeInsert(row, errors){ if (results.rows.length) { row.vendorId = results.rows[0].rowId; } - else { - console.error("Unable to resolve vendor: " + vendorName); - } } }) } diff --git a/MccColony/resources/queries/study/demographics/.qview.xml b/MccColony/resources/queries/study/demographics/.qview.xml index 7cc677499..a35d07baa 100644 --- a/MccColony/resources/queries/study/demographics/.qview.xml +++ b/MccColony/resources/queries/study/demographics/.qview.xml @@ -27,6 +27,46 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/MccColony/resources/queries/study/kinship/.qview.xml b/MccColony/resources/queries/study/kinship/.qview.xml new file mode 100644 index 000000000..71e0ae624 --- /dev/null +++ b/MccColony/resources/queries/study/kinship/.qview.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/PMR/build.gradle b/PMR/build.gradle index 40493795c..14abecf43 100644 --- a/PMR/build.gradle +++ b/PMR/build.gradle @@ -8,6 +8,7 @@ dependencies { BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:DiscvrLabKeyModules:Studies", depProjectConfig: "published", depExtension: "module") BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:dataintegration", depProjectConfig: "published", depExtension: "module") BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:DiscvrLabKeyModules:discvrcore", depProjectConfig: "published", depExtension: "module") + BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:onprcEHRModules:GeneticsCore", depProjectConfig: "published", depExtension: "module") BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:DiscvrLabKeyModules:Studies", depProjectConfig: "apiJarFile") BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:dataintegration", depProjectConfig: "apiJarFile") diff --git a/PMR/resources/etls/KinshipDataStaging.xml b/PMR/resources/etls/KinshipDataStaging.xml new file mode 100644 index 000000000..d0b0b1e47 --- /dev/null +++ b/PMR/resources/etls/KinshipDataStaging.xml @@ -0,0 +1,14 @@ + + + KinshipDataStaging + Prepare PRIMe-seq Kinship Data for Import into PRIMe + + + + + + + + + + diff --git a/PMR/resources/folderTypes/PMR.folderType.xml b/PMR/resources/folderTypes/PMR.folderType.xml index 791e44bd1..95eb08a07 100644 --- a/PMR/resources/folderTypes/PMR.folderType.xml +++ b/PMR/resources/folderTypes/PMR.folderType.xml @@ -64,6 +64,7 @@ PMR EHR + GeneticsCore PMR \ No newline at end of file diff --git a/PMR/src/org/labkey/pmr/etl/TriggerRemoteGeneticsImportStep.java b/PMR/src/org/labkey/pmr/etl/TriggerRemoteGeneticsImportStep.java new file mode 100644 index 000000000..305d57dab --- /dev/null +++ b/PMR/src/org/labkey/pmr/etl/TriggerRemoteGeneticsImportStep.java @@ -0,0 +1,169 @@ +package org.labkey.pmr.etl; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.hc.client5.http.classic.methods.HttpPost; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.core5.http.HttpResponse; +import org.apache.hc.core5.http.HttpStatus; +import org.apache.xmlbeans.XmlException; +import org.jetbrains.annotations.NotNull; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.di.DataIntegrationService; +import org.labkey.api.di.TaskRefTask; +import org.labkey.api.module.Module; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.module.ModuleProperty; +import org.labkey.api.pipeline.PipeRoot; +import org.labkey.api.pipeline.PipelineJob; +import org.labkey.api.pipeline.PipelineJobException; +import org.labkey.api.pipeline.PipelineService; +import org.labkey.api.pipeline.RecordedActionSet; +import org.labkey.api.writer.ContainerUser; +import org.labkey.remoteapi.CommandException; +import org.labkey.remoteapi.CommandResponse; +import org.labkey.remoteapi.PostCommand; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class TriggerRemoteGeneticsImportStep implements TaskRefTask +{ + protected final Map _settings = new CaseInsensitiveHashMap<>(); + protected ContainerUser _containerUser; + + private enum Settings + { + remoteSource() + } + + @Override + public RecordedActionSet run(@NotNull PipelineJob job) throws PipelineJobException + { + // First find the last successful pipeline iteration: + Module ehr = ModuleLoader.getInstance().getModule("ehr"); + Module geneticsCore = ModuleLoader.getInstance().getModule("GeneticsCore"); + + ModuleProperty mp = ehr.getModuleProperties().get("EHRStudyContainer"); + String ehrContainerPath = StringUtils.trimToNull(mp.getEffectiveValue(_containerUser.getContainer())); + if (ehrContainerPath == null) + { + throw new PipelineJobException("EHRStudyContainer has not been set"); + } + + Container ehrContainer = ContainerManager.getForPath(ehrContainerPath); + if (ehrContainer == null) + { + throw new PipelineJobException("Invalid container: " + ehrContainerPath); + } + + if (!_containerUser.getContainer().equals(ehrContainer)) + { + throw new PipelineJobException("This ETL can only be run from the EHRStudyContainer"); + } + + ModuleProperty mp2 = geneticsCore.getModuleProperties().get("KinshipDataPath"); + String pipeDirPath = StringUtils.trimToNull(mp2.getEffectiveValue(ehrContainer)); + if (pipeDirPath == null) + { + throw new PipelineJobException("Must provide the filepath to import data using the KinshipDataPath module property"); + } + + File targetPipelineDir = new File(pipeDirPath); + if (!targetPipelineDir.exists()) + { + targetPipelineDir.mkdirs(); + } + + // Then copy the file to the expected folder: + PipeRoot pr = PipelineService.get().getPipelineRootSetting(ehrContainer); + if (pr == null) + { + throw new PipelineJobException("Unable to find pipeline root for: " + ehrContainer); + } + + File sourceDir = new File(pr.getRootPath(), "/kinship/EHR Kinship Calculation"); + if (!sourceDir.exists()) + { + throw new PipelineJobException("Unable to find source pipeline dir: " + sourceDir.getPath()); + } + + copyReplaceFile(sourceDir, targetPipelineDir, "kinship.txt"); + copyReplaceFile(sourceDir, targetPipelineDir, "inbreeding.txt"); + + // Then ping the main server to import this file: + DataIntegrationService.RemoteConnection rc = DataIntegrationService.get().getRemoteConnection(_settings.get(Settings.remoteSource.name()), _containerUser.getContainer(), job.getLogger()); + if (rc == null) + { + throw new PipelineJobException("Unable to find remote connection: " + _settings.get(Settings.remoteSource.name())); + } + + try + { + KinshipCommand command = new KinshipCommand(); + command.execute(rc.connection, rc.remoteContainer); + } + catch (CommandException | IOException e) + { + throw new PipelineJobException(e); + } + + return new RecordedActionSet(); + } + + private static class KinshipCommand extends PostCommand + { + public KinshipCommand() + { + super("geneticscore", "importGeneticsData"); + } + } + + private void copyReplaceFile(File sourceDir, File targetDir, String filename) throws PipelineJobException + { + File sourceFile = new File(sourceDir, filename); + if (!sourceFile.exists()) + { + throw new PipelineJobException("File does not exist: " + sourceFile.getPath()); + } + + File destFile = new File(targetDir, filename); + if (destFile.exists()) + { + destFile.delete(); + } + + try + { + FileUtils.copyFile(sourceFile, destFile); + } + catch (IOException e) + { + throw new PipelineJobException(e); + } + } + + @Override + public List getRequiredSettings() + { + return Collections.unmodifiableList(Arrays.asList(Settings.remoteSource.name())); + } + + @Override + public void setSettings(Map settings) throws XmlException + { + _settings.putAll(settings); + } + + @Override + public void setContainerUser(ContainerUser containerUser) + { + _containerUser = containerUser; + } +} diff --git a/PMR/test/sampledata/PMR/testPedigree.txt b/PMR/test/sampledata/PMR/testPedigree.txt new file mode 100644 index 000000000..866dbab16 --- /dev/null +++ b/PMR/test/sampledata/PMR/testPedigree.txt @@ -0,0 +1,10 @@ +99991 999911 2 Cynomolgus +999910 99993 99998 2 Cynomolgus +99992 999912 1 Cynomolgus +99993 999912 2 Rhesus +99994 999913 999914 1 Cynomolgus +99995 99991 99992 2 Cynomolgus +99996 99991 99992 1 Cynomolgus +99997 99993 99992 1 Cynomolgus +99998 99995 99992 1 Cynomolgus +99999 99995 99994 2 Cynomolgus diff --git a/PMR/test/src/org/labkey/test/tests/pmr/PMRTest.java b/PMR/test/src/org/labkey/test/tests/pmr/PMRTest.java index 028d7b755..6dd2d5746 100644 --- a/PMR/test/src/org/labkey/test/tests/pmr/PMRTest.java +++ b/PMR/test/src/org/labkey/test/tests/pmr/PMRTest.java @@ -16,25 +16,52 @@ package org.labkey.test.tests.pmr; +import au.com.bytecode.opencsv.CSVReader; +import org.apache.hc.client5.http.classic.methods.HttpUriRequest; +import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.labkey.remoteapi.Command; +import org.labkey.remoteapi.CommandResponse; +import org.labkey.remoteapi.PostCommand; +import org.labkey.remoteapi.query.SelectRowsCommand; +import org.labkey.remoteapi.query.TruncateTableCommand; +import org.labkey.serverapi.reader.Readers; import org.labkey.test.BaseWebDriverTest; +import org.labkey.test.Locator; +import org.labkey.test.Locators; import org.labkey.test.ModulePropertyValue; +import org.labkey.test.TestFileUtils; import org.labkey.test.TestTimeoutException; import org.labkey.test.WebTestHelper; import org.labkey.test.categories.External; import org.labkey.test.categories.LabModule; +import org.labkey.test.tests.di.ETLHelper; +import org.labkey.test.util.Ext4Helper; +import org.labkey.test.util.PipelineStatusTable; +import org.labkey.test.util.RReportHelper; +import org.labkey.test.util.RemoteConnectionHelper; import org.labkey.test.util.SqlserverOnlyTest; +import org.labkey.test.util.ehr.EHRClientAPIHelper; +import org.labkey.test.util.ehr.EHRTestHelper; +import java.io.File; +import java.net.URI; import java.util.Arrays; import java.util.Collections; +import java.util.Date; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Set; @Category({External.class, LabModule.class}) public class PMRTest extends BaseWebDriverTest implements SqlserverOnlyTest { + private final ETLHelper _etlHelper = new ETLHelper(this, getProjectName()); + @Override protected void doCleanup(boolean afterTest) throws TestTimeoutException { @@ -48,6 +75,10 @@ public static void setupProject() throws Exception init.doSetup(); } + private File getKinshipPath() + { + return new File(TestFileUtils.getDefaultFileRoot(getProjectName()), "kinshipEtlDir"); + } private void doSetup() { @@ -55,10 +86,18 @@ private void doSetup() setModuleProperties(Arrays.asList( new ModulePropertyValue("EHR", "/" + getProjectName(), "EHRStudyContainer", "/" + getProjectName()), - new ModulePropertyValue("EHR", "/" + getProjectName(), "EHRAdminUser", getCurrentUser()) + new ModulePropertyValue("EHR", "/" + getProjectName(), "EHRAdminUser", getCurrentUser()), + new ModulePropertyValue("GeneticsCore", "/" + getProjectName(), "KinshipDataPath", getKinshipPath().getPath()) )); importStudy(getProjectName()); + populateLookups(); + + RemoteConnectionHelper rconnHelper = new RemoteConnectionHelper(this); + rconnHelper.goToManageRemoteConnections(); + rconnHelper.createConnection("EHR_ClinicalSource", WebTestHelper.getBaseURL(), getProjectName()); + + new RReportHelper(this).ensureRConfig(); goToHome(); } @@ -70,6 +109,16 @@ private void importStudy(String containerPath) waitForPipelineJobsToComplete(1, "Study import", false, MAX_WAIT_SECONDS * 2500); } + private void populateLookups() + { + beginAt(getProjectName() + "/pmr-populateData.view"); + waitAndClick(Ext4Helper.Locators.ext4Button("Populate Lookup Sets")); + waitForElement(Locator.tagWithText("div", "Populating lookup_sets...")); + waitForElement(Locator.tagWithText("div", "Populate Complete")); + + waitAndClick(Ext4Helper.Locators.ext4Button("Populate All")); + waitForElement(Locator.tagWithText("div", "Populate Complete")); + } @Before public void preTest() @@ -78,9 +127,92 @@ public void preTest() } @Test - public void testPMRModule() + public void testPMRModule() throws Exception + { + testKinshipEtl(); + } + + private void testKinshipEtl() throws Exception + { + createTestPedigreeData(); + + // Calculate genetics, which will output results in the projects file root: + beginAt(getProjectName() + "/ehr-ehrAdmin.view"); + waitAndClickAndWait(Locator.tagContainingText("a", "Genetics Calculations")); + _ext4Helper.checkCheckbox(Ext4Helper.Locators.checkbox(this, "Kinship validation?:")); + _ext4Helper.checkCheckbox(Ext4Helper.Locators.checkbox(this, "Allow Import During Business Hours?:")); + Locator loc = Locator.inputByIdContaining("numberfield"); + waitForElement(loc); + setFormElement(loc, "23"); + click(Ext4Helper.Locators.ext4Button("Save Settings")); + waitAndClick(Ext4Helper.Locators.ext4Button("OK")); + waitAndClickAndWait(Ext4Helper.Locators.ext4Button("Run Now")); + waitAndClickAndWait(Locator.lkButton("OK")); + waitForPipelineJobsToComplete(2, "EHR Kinship Calculation", false); + + // Verify data imported, and then delete from the DB + SelectRowsCommand select1 = new SelectRowsCommand("ehr", "kinship"); + Assert.assertEquals("Incorrect number of kinship rows", 136, select1.execute(getApiHelper().getConnection(), getProjectName()).getRowCount().intValue()); + + new TruncateTableCommand("ehr", "kinship").execute(getApiHelper().getConnection(), getProjectName()); + Assert.assertEquals("Incorrect number of kinship rows", 0, select1.execute(getApiHelper().getConnection(), getProjectName()).getRowCount().intValue()); + + // Kick off ETL to stage data. This should also kick off a separate pipeline job to import, using geneticscore-importGeneticsData.view + _etlHelper.runETL("{PMR}/KinshipDataStaging"); + goToDataPipeline(); + waitForPipelineJobsToComplete(4, "ETL Job: Import PRIMe-seq Kinship Data", false); + + Assert.assertEquals("Incorrect number of kinship rows after ETL", 136, select1.execute(getApiHelper().getConnection(), getProjectName()).getRowCount().intValue()); + } + + private void createTestPedigreeData() throws Exception + { + // Create dummy data: + Set demographicsAdded = new HashSet<>(); + final Date d = new Date(); + + File testPedigree = TestFileUtils.getSampleData("PMR/testPedigree.txt"); + try (CSVReader reader = new CSVReader(Readers.getReader(testPedigree), '\t')) + { + String[] line; + while ((line = reader.readNext()) != null) + { + if (!demographicsAdded.contains(line[0])) + { + getApiHelper().insertRow("study", "demographics", Map.of("Id", line[0], "species", line[4], "gender", ("1".equals(line[3]) ? "m" : "f"), "date", d, "QCStateLabel", "Completed"), false); + demographicsAdded.add(line[0]); + } + + // dam + if (!line[1].isEmpty()) + { + if (!demographicsAdded.contains(line[1])) + { + getApiHelper().insertRow("study", "demographics", Map.of("Id", line[1], "species", line[4], "gender", "f", "date", d, "QCStateLabel", "Completed"), false); + demographicsAdded.add(line[1]); + } + + getApiHelper().insertRow("study", "parentage", Map.of("Id", line[0], "parent", line[1], "relationship", "Dam", "method", "Genetic", "date", d, "QCStateLabel", "Completed"), false); + } + + // sire + if (!line[2].isEmpty()) + { + if (!demographicsAdded.contains(line[2])) + { + getApiHelper().insertRow("study", "demographics", Map.of("Id", line[2], "species", line[4], "gender", "m", "date", d, "QCStateLabel", "Completed"), false); + demographicsAdded.add(line[2]); + } + + getApiHelper().insertRow("study", "parentage", Map.of("Id", line[0], "parent", line[2], "relationship", "Sire", "method", "Genetic", "date", d, "QCStateLabel", "Completed"), false); + } + } + } + } + + private EHRClientAPIHelper getApiHelper() { - _containerHelper.enableModule("PMR"); + return new EHRClientAPIHelper(this, getProjectName()); } @Override diff --git a/flowassays/resources/queries/flowassays/populations.js b/flowassays/resources/queries/flowassays/populations.js index 2902908d0..cf24cdf21 100644 --- a/flowassays/resources/queries/flowassays/populations.js +++ b/flowassays/resources/queries/flowassays/populations.js @@ -16,7 +16,7 @@ function beforeInsert(row, errors){ validateName(row, errors); } -function beforeUpdate(row, errors){ +function beforeUpdate(row, oldRow, errors){ validateName(row, errors); } diff --git a/hivrc/build.gradle b/hivrc/build.gradle new file mode 100644 index 000000000..1c2afe179 --- /dev/null +++ b/hivrc/build.gradle @@ -0,0 +1,17 @@ +import org.labkey.gradle.util.BuildUtils +import org.labkey.gradle.util.ExternalDependency + +repositories { + mavenCentral() +} + +dependencies { + BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:LabDevKitModules:laboratory", depProjectConfig: "apiJarFile") + BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:LabDevKitModules:LDK", depProjectConfig: "apiJarFile") + + BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:LabDevKitModules:laboratory", depProjectConfig: "published", depExtension: "module") + + BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:LabDevKitModules:LDK", depProjectConfig: "published", depExtension: "module") + BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:LabDevKitModules:laboratory", depProjectConfig: "published", depExtension: "module") + BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:DiscvrLabKeyModules:discvrcore", depProjectConfig: "published", depExtension: "module") +} diff --git a/hivrc/module.properties b/hivrc/module.properties new file mode 100644 index 000000000..0136192fd --- /dev/null +++ b/hivrc/module.properties @@ -0,0 +1,6 @@ +ModuleClass: org.labkey.hivrc.HivrcModule +Label: BMGF HIVRC +Description: This is designed to manage analyses and data collected as part of the BMGF HIVRC +License: Apache 2.0 +LicenseURL: http://www.apache.org/licenses/LICENSE-2.0 +ManageVersion: false diff --git a/hivrc/resources/folderTypes/HIVRC Analysis.folderType.xml b/hivrc/resources/folderTypes/HIVRC Analysis.folderType.xml new file mode 100644 index 000000000..8e4f41dc4 --- /dev/null +++ b/hivrc/resources/folderTypes/HIVRC Analysis.folderType.xml @@ -0,0 +1,34 @@ + + HIVRC Analyses + Workbook template for HIVRC analyses + + + HIVRC Analysis Header + body + + + + + Pipeline Files + body + + + + Messages + body + + + + + + HIVRC + laboratory + Pipeline + Experiment + Query + Core + + Core + true + true + \ No newline at end of file diff --git a/hivrc/resources/folderTypes/HIVRC.folderType.xml b/hivrc/resources/folderTypes/HIVRC.folderType.xml new file mode 100644 index 000000000..a57250222 --- /dev/null +++ b/hivrc/resources/folderTypes/HIVRC.folderType.xml @@ -0,0 +1,59 @@ + + HIVRC Overview + The default folder layout for an HIVRC project + + + overview + Overview + + + + + + + HIVRC Data and Analyses + body + + + + + datasets + Datasets + + + datasets + + + + + + + HIVRC Datasets + body + + + + + analyses + Analyses + + + analyses + + + + + + + HIVRC Analyses + body + + + + + + hivrc + + hivrc + true + \ No newline at end of file diff --git a/hivrc/resources/queries/laboratory/workbooks.query.xml b/hivrc/resources/queries/laboratory/workbooks.query.xml new file mode 100644 index 000000000..fbee6d1d8 --- /dev/null +++ b/hivrc/resources/queries/laboratory/workbooks.query.xml @@ -0,0 +1,15 @@ + + + + + HIVRC Analyses + workbookId + + + Analysis # + + +
+ + + \ No newline at end of file diff --git a/hivrc/resources/schemas/dbscripts/postgresql/hivrc-0.00-23.000.sql b/hivrc/resources/schemas/dbscripts/postgresql/hivrc-0.00-23.000.sql new file mode 100644 index 000000000..5230cdf8c --- /dev/null +++ b/hivrc/resources/schemas/dbscripts/postgresql/hivrc-0.00-23.000.sql @@ -0,0 +1 @@ +CREATE SCHEMA hivrc; \ No newline at end of file diff --git a/hivrc/resources/schemas/dbscripts/postgresql/hivrc-23.000-23.001.sql b/hivrc/resources/schemas/dbscripts/postgresql/hivrc-23.000-23.001.sql new file mode 100644 index 000000000..6b94a880a --- /dev/null +++ b/hivrc/resources/schemas/dbscripts/postgresql/hivrc-23.000-23.001.sql @@ -0,0 +1,17 @@ +CREATE TABLE hivrc.datasets ( + rowid SERIAL, + label varchar(1000), + description varchar(4000), + + datatype varchar(1000), + dataowner varchar(1000), + fileid int, + + container entityid, + created timestamp, + createdby int, + modified timestamp, + modifiedby int, + + constraint PK_datasets PRIMARY KEY (rowid) +); \ No newline at end of file diff --git a/hivrc/resources/schemas/dbscripts/sqlserver/hivrc-0.00-23.000.sql b/hivrc/resources/schemas/dbscripts/sqlserver/hivrc-0.00-23.000.sql new file mode 100644 index 000000000..ff0f8ccc9 --- /dev/null +++ b/hivrc/resources/schemas/dbscripts/sqlserver/hivrc-0.00-23.000.sql @@ -0,0 +1,2 @@ +CREATE SCHEMA hivrc; +GO \ No newline at end of file diff --git a/hivrc/resources/schemas/dbscripts/sqlserver/hivrc-23.000-23.001.sql b/hivrc/resources/schemas/dbscripts/sqlserver/hivrc-23.000-23.001.sql new file mode 100644 index 000000000..42b28c1eb --- /dev/null +++ b/hivrc/resources/schemas/dbscripts/sqlserver/hivrc-23.000-23.001.sql @@ -0,0 +1,17 @@ +CREATE TABLE hivrc.datasets ( + rowid int IDENTITY(1,1), + label varchar(1000), + description varchar(4000), + + datatype varchar(1000), + dataowner varchar(1000), + fileid int, + + container entityid, + created datetime, + createdby int, + modified datetime, + modifiedby int, + + constraint PK_datasets PRIMARY KEY (rowid) +); \ No newline at end of file diff --git a/hivrc/resources/schemas/hivrc.xml b/hivrc/resources/schemas/hivrc.xml new file mode 100644 index 000000000..04f6daa40 --- /dev/null +++ b/hivrc/resources/schemas/hivrc.xml @@ -0,0 +1,61 @@ + + + + Subjects + + DETAILED + + + true + false + false + false + Row Id + + + Label + + + Description + + + Data Type + + + Data Owner + + + File ID + + exp + data + RowId + + + + true + + + true + + + false + false + false + true + true + + + true + + + false + false + false + true + true + + +
+ \ No newline at end of file diff --git a/hivrc/resources/views/begin.html b/hivrc/resources/views/begin.html new file mode 100644 index 000000000..4f908505e --- /dev/null +++ b/hivrc/resources/views/begin.html @@ -0,0 +1,9 @@ +This is a prototype for a web-based system to manage BMGF datasets and analyses. Currently, the system is primarily designed to facilitate the work of informaticians/analysts, +aiding in discoverability of data, access to data, and documentation/sharing of analyses. There are two components to this system: +
+
+
    +
  1. This system will contain a registry for input datasets. This provides a central table of inputs, each of which can be identified by ID.
    2. + +
  2. The system also tracks analyses. Each analysis is loosely defined as one unit of work, such as validation of a given biomarker candidate in one cohort. Creating an analysis provides a folder to save any relevant files. The minimum requirement for an analysis is to upload a document (such as a markdown file) that serves to document the details of what was performed.
    3. +
\ No newline at end of file diff --git a/hivrc/resources/views/begin.view.xml b/hivrc/resources/views/begin.view.xml new file mode 100644 index 000000000..3ea006296 --- /dev/null +++ b/hivrc/resources/views/begin.view.xml @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/hivrc/resources/views/begin.webpart.xml b/hivrc/resources/views/begin.webpart.xml new file mode 100644 index 000000000..5c585de57 --- /dev/null +++ b/hivrc/resources/views/begin.webpart.xml @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/hivrc/resources/views/hivrcAnalyses.html b/hivrc/resources/views/hivrcAnalyses.html new file mode 100644 index 000000000..eda36eaef --- /dev/null +++ b/hivrc/resources/views/hivrcAnalyses.html @@ -0,0 +1,115 @@ + diff --git a/hivrc/resources/views/hivrcAnalyses.view.xml b/hivrc/resources/views/hivrcAnalyses.view.xml new file mode 100644 index 000000000..3dcb2a095 --- /dev/null +++ b/hivrc/resources/views/hivrcAnalyses.view.xml @@ -0,0 +1,5 @@ + + + + + diff --git a/hivrc/resources/views/hivrcAnalyses.webpart.xml b/hivrc/resources/views/hivrcAnalyses.webpart.xml new file mode 100644 index 000000000..06b52f4d0 --- /dev/null +++ b/hivrc/resources/views/hivrcAnalyses.webpart.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/hivrc/resources/views/hivrcDatasets.html b/hivrc/resources/views/hivrcDatasets.html new file mode 100644 index 000000000..ef9472881 --- /dev/null +++ b/hivrc/resources/views/hivrcDatasets.html @@ -0,0 +1,22 @@ + + diff --git a/hivrc/resources/views/hivrcDatasets.view.xml b/hivrc/resources/views/hivrcDatasets.view.xml new file mode 100644 index 000000000..007700474 --- /dev/null +++ b/hivrc/resources/views/hivrcDatasets.view.xml @@ -0,0 +1,5 @@ + + + + + diff --git a/hivrc/resources/views/hivrcDatasets.webpart.xml b/hivrc/resources/views/hivrcDatasets.webpart.xml new file mode 100644 index 000000000..7c8c99040 --- /dev/null +++ b/hivrc/resources/views/hivrcDatasets.webpart.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/hivrc/resources/web/hivrc/panel/AnalysisHeaderPanel.js b/hivrc/resources/web/hivrc/panel/AnalysisHeaderPanel.js new file mode 100644 index 000000000..4ca67d407 --- /dev/null +++ b/hivrc/resources/web/hivrc/panel/AnalysisHeaderPanel.js @@ -0,0 +1,46 @@ +/** + * @param description + * @param materials + * @param methods + * @param results + */ +Ext4.define('HIVRC.panel.AnalysisHeaderPanel', { + extend: 'Laboratory.panel.WorkbookHeaderPanel', + panelTitle: 'Analysis Details', + + getPanelItems: function() { + return [{ + html: 'Analysis Description:', + style: 'font-weight: bold;' + }, + this.getFieldCfg('description', this.description) + ,{ + html: 'Results:', + style: 'font-weight: bold;padding-top: 10px;' + }, + this.getFieldCfg('results', this.results) + ,{ + html: 'Tags:', + style: 'font-weight: bold;padding-top: 10px;' + }, + this.getTagFieldCfg() + ] + }, + + onUpdate: function(){ + var values = {}; + Ext4.each(['description', 'results'], function(text){ + values[text] = this.down('#' + text).getValue(); + }, this); + + values.tags = this.tags; + values.forceTagUpdate = true; + + LABKEY.Ajax.request({ + url: LABKEY.ActionURL.buildURL('laboratory', 'updateWorkbook'), + method: 'POST', + params: values, + failure: LDK.Utils.getErrorCallback() + }); + } +}); \ No newline at end of file diff --git a/hivrc/src/org/labkey/hivrc/HivrcController.java b/hivrc/src/org/labkey/hivrc/HivrcController.java new file mode 100644 index 000000000..624163ff2 --- /dev/null +++ b/hivrc/src/org/labkey/hivrc/HivrcController.java @@ -0,0 +1,66 @@ +package org.labkey.hivrc; + +import org.jetbrains.annotations.NotNull; +import org.labkey.api.action.ConfirmAction; +import org.labkey.api.action.SimpleViewAction; +import org.labkey.api.action.SpringActionController; +import org.labkey.api.module.Module; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.module.ModuleProperty; +import org.labkey.api.security.RequiresPermission; +import org.labkey.api.security.permissions.AdminPermission; +import org.labkey.api.security.permissions.ReadPermission; +import org.labkey.api.util.HtmlString; +import org.labkey.api.util.URLHelper; +import org.labkey.api.view.HtmlView; +import org.labkey.api.view.JspView; +import org.labkey.api.view.NavTree; +import org.springframework.validation.BindException; +import org.springframework.validation.Errors; +import org.springframework.web.servlet.ModelAndView; + +public class HivrcController extends SpringActionController +{ + private static final DefaultActionResolver _actionResolver = new DefaultActionResolver(HivrcController.class); + + public static final String NAME = "hivrc"; + + public HivrcController() + { + setActionResolver(_actionResolver); + } + + @RequiresPermission(AdminPermission.class) + public static class SetupDefaultsAction extends ConfirmAction + { + @Override + public ModelAndView getConfirmView(Object o, BindException errors) throws Exception + { + setTitle("Set up HIVRC"); + + return new HtmlView(HtmlString.unsafe("This will ensure various settings required by HIVRC are configured in this folder. Do you want to continue?")); + } + + @Override + public boolean handlePost(Object o, BindException errors) throws Exception + { + Module m = ModuleLoader.getInstance().getModule("laboratory"); + ModuleProperty mp = m.getModuleProperties().get("DefaultWorkbookFolderType"); + mp.saveValue(getUser(), getContainer(), "HIVRC Analyses"); + + return true; + } + + @Override + public void validateCommand(Object o, Errors errors) + { + + } + + @Override + public @NotNull URLHelper getSuccessURL(Object o) + { + return getContainer().getStartURL(getUser()); + } + } +} diff --git a/hivrc/src/org/labkey/hivrc/HivrcManager.java b/hivrc/src/org/labkey/hivrc/HivrcManager.java new file mode 100644 index 000000000..0407b5be5 --- /dev/null +++ b/hivrc/src/org/labkey/hivrc/HivrcManager.java @@ -0,0 +1,61 @@ +package org.labkey.hivrc; + +import org.labkey.api.data.Container; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.query.FieldKey; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.hivrc.query.AnalysisModel; + +public class HivrcManager +{ + private static final HivrcManager _instance = new HivrcManager(); + + private HivrcManager() + { + + } + + public static HivrcManager get() + { + return _instance; + } + + public AnalysisModel getAnalysisModel(Container c, boolean createIfNotPresent) + { + if (!c.isWorkbook()) + { + return null; + } + + TableInfo ti = HivrcSchema.getInstance().getLaboratorySchema().getTable(HivrcSchema.TABLE_WORKBOOKS); + TableSelector ts = new TableSelector(ti, new SimpleFilter(FieldKey.fromString("container"), c.getId()), null); + AnalysisModel[] arr = ts.getArray(AnalysisModel.class); + AnalysisModel m = arr.length == 0 ? null : arr[0]; + if (m == null) + { + if (createIfNotPresent) + { + m = AnalysisModel.createNew(c); + } + else + { + return null; + } + } + + if (m.getContainer() == null) + { + m.setContainer(c.getId()); + } + + TableInfo ti2 = HivrcSchema.getInstance().getLaboratorySchema().getTable(HivrcSchema.TABLE_WORKBOOK_TAGS); + TableSelector ts2 = new TableSelector(ti2, PageFlowUtil.set("tag"), new SimpleFilter(FieldKey.fromString("container"), c.getId()), null); + String[] arr2 = ts2.getArray(String.class); + m.setTags(arr2); + + return m; + } +} \ No newline at end of file diff --git a/hivrc/src/org/labkey/hivrc/HivrcModule.java b/hivrc/src/org/labkey/hivrc/HivrcModule.java new file mode 100644 index 000000000..d5a945e4b --- /dev/null +++ b/hivrc/src/org/labkey/hivrc/HivrcModule.java @@ -0,0 +1,104 @@ +package org.labkey.hivrc; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.data.Container; +import org.labkey.api.ldk.ExtendedSimpleModule; +import org.labkey.api.module.Module; +import org.labkey.api.module.ModuleContext; +import org.labkey.api.query.DefaultSchema; +import org.labkey.api.query.QuerySchema; +import org.labkey.api.util.HtmlString; +import org.labkey.api.view.BaseWebPartFactory; +import org.labkey.api.view.HtmlView; +import org.labkey.api.view.JspView; +import org.labkey.api.view.Portal; +import org.labkey.api.view.ViewContext; +import org.labkey.api.view.WebPartFactory; +import org.labkey.api.view.WebPartView; +import org.labkey.hivrc.query.AnalysisModel; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Set; + +public class HivrcModule extends ExtendedSimpleModule +{ + public static final String NAME = "HIVRC"; + + @Override + public String getName() + { + return NAME; + } + + @Override + public @Nullable Double getSchemaVersion() + { + return 23.001; + } + + @Override + public boolean hasScripts() + { + return true; + } + + @Override + @NotNull + protected Collection createWebPartFactories() + { + return Arrays.asList( + new BaseWebPartFactory("HIVRC Analysis Header") + { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) + { + if (!portalCtx.getContainer().isWorkbook()) + { + return new HtmlView(HtmlString.of("This container is not a workbook")); + } + + AnalysisModel model = HivrcManager.get().getAnalysisModel(portalCtx.getContainer(), true); + if (model == null) + { + model = AnalysisModel.createNew(portalCtx.getContainer()); + } + + JspView view = new JspView<>("/org/labkey/hivrc/view/analysisHeader.jsp", model); + view.setTitle("HIVRC Analysis Summary"); + view.setFrame(WebPartView.FrameType.NONE); + + return view; + } + + @Override + public boolean isAvailable(Container c, String scope, String location) + { + return false; + } + } + ); + } + + @Override + protected void init() + { + addController(HivrcController.NAME, HivrcController.class); + } + + @Override + @NotNull + public Collection getSummary(Container c) + { + return Collections.emptyList(); + } + + @Override + @NotNull + public Set getSchemaNames() + { + return Collections.singleton(HivrcSchema.NAME); + } +} \ No newline at end of file diff --git a/hivrc/src/org/labkey/hivrc/HivrcSchema.java b/hivrc/src/org/labkey/hivrc/HivrcSchema.java new file mode 100644 index 000000000..ee11da04e --- /dev/null +++ b/hivrc/src/org/labkey/hivrc/HivrcSchema.java @@ -0,0 +1,40 @@ +package org.labkey.hivrc; + +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbSchemaType; +import org.labkey.api.data.dialect.SqlDialect; + +public class HivrcSchema +{ + public static final String LABORATORY = "laboratory"; + public static final String TABLE_WORKBOOKS = "workbooks"; + public static final String TABLE_WORKBOOK_TAGS = "workbook_tags"; + + private static final HivrcSchema _instance = new HivrcSchema(); + public static final String NAME = "hivrc"; + + public static HivrcSchema getInstance() + { + return _instance; + } + + private HivrcSchema() + { + + } + + public DbSchema getSchema() + { + return DbSchema.get(NAME, DbSchemaType.Module); + } + + public DbSchema getLaboratorySchema() + { + return DbSchema.get(LABORATORY, DbSchemaType.Module); + } + + public SqlDialect getSqlDialect() + { + return getSchema().getSqlDialect(); + } +} diff --git a/hivrc/src/org/labkey/hivrc/query/AnalysisModel.java b/hivrc/src/org/labkey/hivrc/query/AnalysisModel.java new file mode 100644 index 000000000..bc6dfe788 --- /dev/null +++ b/hivrc/src/org/labkey/hivrc/query/AnalysisModel.java @@ -0,0 +1,140 @@ +package org.labkey.hivrc.query; + +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.query.ValidationException; +import org.labkey.api.security.User; + +/** + * User: bimber + * Date: 3/30/13 + * Time: 11:58 AM + */ +public class AnalysisModel +{ + private Integer _workbookId; + private String _containerId; + private String _materials; + private String _methods; + private String _results; + private String _exptGroup; + private String[] _tags; + + public AnalysisModel() + { + + } + + public Integer getWorkbookId() + { + return _workbookId; + } + + public void setWorkbookId(Integer workbookId) + { + _workbookId = workbookId; + } + + public String getContainer() + { + return _containerId; + } + + public void setContainer(String container) + { + _containerId = container; + } + + public String getMaterials() + { + return _materials; + } + + public void setMaterials(String materials) + { + _materials = materials; + } + + public String getMethods() + { + return _methods; + } + + public void setMethods(String methods) + { + _methods = methods; + } + + public String getResults() + { + return _results; + } + + public void setResults(String results) + { + _results = results; + } + + public String getExptGroup() + { + return _exptGroup; + } + + public void setExptGroup(String exptGroup) + { + _exptGroup = exptGroup; + } + + private Container _getContainer() + { + if (_containerId == null) + throw new IllegalArgumentException("The containerId has not been set"); + + Container c = ContainerManager.getForId(_containerId); + if (c == null) + throw new IllegalArgumentException("Unknown container: " + _containerId); + + return c; + } + + public void setDescription(String description, User u) + { + try + { + ContainerManager.updateDescription(_getContainer(), description, u); + } + catch (ValidationException e) + { + throw new IllegalArgumentException(e.getMessage()); + } + } + + public String getDescription() + { + return _getContainer().getDescription(); + } + + public String[] getTags() + { + return _tags; + } + + public void setTags(String[] tags) + { + _tags = tags; + } + + public static AnalysisModel createNew(Container c) + { + if (!c.isWorkbook()) + { + throw new IllegalArgumentException("Container is not a workbook: " + c.getPath()); + } + + AnalysisModel model = new AnalysisModel(); + model.setWorkbookId(Integer.parseInt(c.getName())); + model.setContainer(c.getId()); + + return model; + } +} diff --git a/hivrc/src/org/labkey/hivrc/view/analysisHeader.jsp b/hivrc/src/org/labkey/hivrc/view/analysisHeader.jsp new file mode 100644 index 000000000..c4641d294 --- /dev/null +++ b/hivrc/src/org/labkey/hivrc/view/analysisHeader.jsp @@ -0,0 +1,75 @@ +<%@ page import="org.apache.commons.lang3.StringUtils" %> +<%@ page import="org.labkey.api.view.HttpView" %> +<%@ page import="org.labkey.api.view.JspView" %> +<%@ page import="org.labkey.api.view.template.ClientDependencies" %> +<%@ page import="org.labkey.hivrc.query.AnalysisModel" %> +<%@ page import="java.util.Arrays" %> +<%@ page extends="org.labkey.api.jsp.JspBase" %> +<%! + @Override + public void addClientDependencies(ClientDependencies dependencies) + { + dependencies.add("Ext4"); + dependencies.add("laboratory.context"); + dependencies.add("laboratory/panel/WorkbookHeaderPanel.js"); + dependencies.add("hivrc/panel/AnalysisHeaderPanel.js"); + dependencies.add("editInPlaceElement.css"); + dependencies.add("editInPlaceElement.js"); + } +%> +<% + JspView me = (JspView) HttpView.currentView(); + AnalysisModel model = me.getModelBean(); + Integer workbookId = model.getWorkbookId(); + String wpId = "wp_" + me.getWebPartRowId(); +%> + + + +
>
+ + diff --git a/mGAP/resources/queries/mGAP/combinedPedigree.query.xml b/mGAP/resources/queries/mGAP/combinedPedigree.query.xml new file mode 100644 index 000000000..c5171b024 --- /dev/null +++ b/mGAP/resources/queries/mGAP/combinedPedigree.query.xml @@ -0,0 +1,14 @@ + + + + + mGAP Combined Pedigree + + + true + + +
+ + + \ No newline at end of file diff --git a/mGAP/resources/queries/mGAP/combinedPedigree.sql b/mGAP/resources/queries/mGAP/combinedPedigree.sql new file mode 100644 index 000000000..55512965a --- /dev/null +++ b/mGAP/resources/queries/mGAP/combinedPedigree.sql @@ -0,0 +1,22 @@ +SELECT + s.subjectname, + s.gender, + s.mother as dam, + s.father as sire, + s.species, + s.geographic_origin, + +FROM laboratory.subjects s + +UNION ALL + +SELECT + d.subjectname, + d.gender, + d.dam, + d.sire, + d.species, + null as geographic_origin + +FROM mgap.demographics d +WHERE d.subjectname NOT IN (SELECT DISTINCT s.subjectname FROM laboratory.subjects s) \ No newline at end of file diff --git a/mGAP/resources/schemas/dbscripts/postgresql/mgap-16.71-16.72.sql b/mGAP/resources/schemas/dbscripts/postgresql/mgap-16.71-16.72.sql new file mode 100644 index 000000000..c8d436f98 --- /dev/null +++ b/mGAP/resources/schemas/dbscripts/postgresql/mgap-16.71-16.72.sql @@ -0,0 +1 @@ +ALTER TABLE mGAP.geneticMeasurements ADD dataid int; \ No newline at end of file diff --git a/mGAP/resources/schemas/dbscripts/postgresql/mgap-16.72-16.73.sql b/mGAP/resources/schemas/dbscripts/postgresql/mgap-16.72-16.73.sql new file mode 100644 index 000000000..52699de10 --- /dev/null +++ b/mGAP/resources/schemas/dbscripts/postgresql/mgap-16.72-16.73.sql @@ -0,0 +1,2 @@ +ALTER TABLE mGAP.demographics ADD sire varchar(1000); +ALTER TABLE mGAP.demographics ADD dam varchar(1000); \ No newline at end of file diff --git a/mGAP/resources/schemas/dbscripts/sqlserver/mgap-16.71-16.72.sql b/mGAP/resources/schemas/dbscripts/sqlserver/mgap-16.71-16.72.sql new file mode 100644 index 000000000..c8d436f98 --- /dev/null +++ b/mGAP/resources/schemas/dbscripts/sqlserver/mgap-16.71-16.72.sql @@ -0,0 +1 @@ +ALTER TABLE mGAP.geneticMeasurements ADD dataid int; \ No newline at end of file diff --git a/mGAP/resources/schemas/dbscripts/sqlserver/mgap-16.72-16.73.sql b/mGAP/resources/schemas/dbscripts/sqlserver/mgap-16.72-16.73.sql new file mode 100644 index 000000000..52699de10 --- /dev/null +++ b/mGAP/resources/schemas/dbscripts/sqlserver/mgap-16.72-16.73.sql @@ -0,0 +1,2 @@ +ALTER TABLE mGAP.demographics ADD sire varchar(1000); +ALTER TABLE mGAP.demographics ADD dam varchar(1000); \ No newline at end of file diff --git a/mGAP/resources/schemas/mgap.xml b/mGAP/resources/schemas/mgap.xml index 08f52c43e..24bfd6332 100644 --- a/mGAP/resources/schemas/mgap.xml +++ b/mGAP/resources/schemas/mgap.xml @@ -1060,6 +1060,12 @@ + + Sire + + + Dam + Center @@ -1132,6 +1138,17 @@ Comment + + File + false + false + false + + exp + data + RowId + + true diff --git a/mGAP/resources/views/demographics.html b/mGAP/resources/views/demographics.html index d1f38e4d2..8ed56996f 100644 --- a/mGAP/resources/views/demographics.html +++ b/mGAP/resources/views/demographics.html @@ -8,7 +8,7 @@ LDK.Utils.getReadOnlyQWP({ frame: 'none', title: 'mGAP Demographics', - schemaName: 'study', + schemaName: 'mgap', queryName: 'demographics', showRecordSelectors: false, showDetailsColumn: false diff --git a/mGAP/resources/web/mGAP/DownloadWindow.js b/mGAP/resources/web/mGAP/DownloadWindow.js index a3698564a..69ff0a824 100644 --- a/mGAP/resources/web/mGAP/DownloadWindow.js +++ b/mGAP/resources/web/mGAP/DownloadWindow.js @@ -2,7 +2,7 @@ Ext4.define('mGAP.window.DownloadWindow', { extend: 'Ext.window.Window', statics: { - buttonHandler: function(releaseId, el){ + buttonHandler: function(releaseId){ Ext4.create('mGAP.window.DownloadWindow', { releaseId: releaseId }); diff --git a/mGAP/src/org/labkey/mgap/buttons/ReleaseButton.java b/mGAP/src/org/labkey/mgap/buttons/ReleaseButton.java index 47a3c437b..8f9dfbead 100644 --- a/mGAP/src/org/labkey/mgap/buttons/ReleaseButton.java +++ b/mGAP/src/org/labkey/mgap/buttons/ReleaseButton.java @@ -14,6 +14,7 @@ public ReleaseButton(Module owner) ClientDependency.supplierFromPath("laboratory.context"), ClientDependency.supplierFromPath("/SequenceAnalysis/window/OutputHandlerWindow.js"), ClientDependency.supplierFromPath("/mGAP/window/ReleaseWindow.js"), - ClientDependency.supplierFromPath("sequenceanalysis/field/GenomeFileSelectorField.js"))); + ClientDependency.supplierFromPath("sequenceanalysis/field/GenomeFileSelectorField.js"), + ClientDependency.supplierFromPath("sequenceanalysis/field/SequenceOutputFileSelectorField.js"))); } } diff --git a/mGAP/src/org/labkey/mgap/jbrowse/mGAPFieldCustomizer.java b/mGAP/src/org/labkey/mgap/jbrowse/mGAPFieldCustomizer.java index b1ab44524..679d967ac 100644 --- a/mGAP/src/org/labkey/mgap/jbrowse/mGAPFieldCustomizer.java +++ b/mGAP/src/org/labkey/mgap/jbrowse/mGAPFieldCustomizer.java @@ -92,7 +92,7 @@ public List getPromotedFilters(Collection indexedFields, Contain List ret = new ArrayList<>(); if (indexedFields.contains("IMPACT")) { - ret.add("Protein Coding Variants|IMPACT,does not equal,MODIFIER"); + ret.add("Protein Coding Variants|IMPACT,does not equal,MODIFIER&IMPACT,is not empty,"); } if (indexedFields.contains("OG")) diff --git a/mGAP/src/org/labkey/mgap/mGAPDemographicsProvider.java b/mGAP/src/org/labkey/mgap/mGAPDemographicsProvider.java new file mode 100644 index 000000000..b59a6ec3d --- /dev/null +++ b/mGAP/src/org/labkey/mgap/mGAPDemographicsProvider.java @@ -0,0 +1,34 @@ +package org.labkey.mgap; + +import org.jetbrains.annotations.Nullable; +import org.labkey.api.laboratory.DemographicsProvider; +import org.labkey.api.module.ModuleLoader; + +public class mGAPDemographicsProvider extends DemographicsProvider +{ + public mGAPDemographicsProvider() + { + super(ModuleLoader.getInstance().getModule(mGAPModule.class), mGAPSchema.NAME, "combinedPedigree", "subjectname"); + } + + @Nullable + @Override + public String getMotherField() + { + return "dam"; + } + + @Nullable + @Override + public String getFatherField() + { + return "sire"; + } + + @Nullable + @Override + public String getSexField() + { + return "gender"; + } +} diff --git a/mGAP/src/org/labkey/mgap/mGAPDemographicsSource.java b/mGAP/src/org/labkey/mgap/mGAPDemographicsSource.java index 97047cd49..584db3e88 100644 --- a/mGAP/src/org/labkey/mgap/mGAPDemographicsSource.java +++ b/mGAP/src/org/labkey/mgap/mGAPDemographicsSource.java @@ -24,7 +24,6 @@ public class mGAPDemographicsSource implements DemographicsSource { - @Override public Map> resolveSubjects(List subjects, Container c, User u) { diff --git a/mGAP/src/org/labkey/mgap/mGAPModule.java b/mGAP/src/org/labkey/mgap/mGAPModule.java index 6bdc5af3c..affb5cbf7 100644 --- a/mGAP/src/org/labkey/mgap/mGAPModule.java +++ b/mGAP/src/org/labkey/mgap/mGAPModule.java @@ -27,6 +27,7 @@ import org.labkey.api.data.TableSelector; import org.labkey.api.data.UpgradeCode; import org.labkey.api.jbrowse.JBrowseService; +import org.labkey.api.laboratory.LaboratoryService; import org.labkey.api.ldk.ExtendedSimpleModule; import org.labkey.api.ldk.LDKService; import org.labkey.api.ldk.buttons.ShowBulkEditButton; @@ -46,6 +47,7 @@ import org.labkey.mgap.jbrowse.mGAPLuceneDetector; import org.labkey.mgap.pipeline.AnnotationStep; import org.labkey.mgap.pipeline.GenerateMgapTracksStep; +import org.labkey.mgap.pipeline.GeographicOriginStep; import org.labkey.mgap.pipeline.GroupCompareStep; import org.labkey.mgap.pipeline.IndexVariantsForMgapStep; import org.labkey.mgap.pipeline.RemoveAnnotationsForMgapStep; @@ -74,7 +76,7 @@ public String getName() @Override public Double getSchemaVersion() { - return 16.71; + return 16.73; } @Override @@ -101,6 +103,8 @@ public void doStartupAfterSpringConfig(ModuleContext moduleContext) JBrowseService.get().registerGroupsProvider(new mGAPGroupsProvider()); JBrowseService.get().registerLuceneIndexDetector(new mGAPLuceneDetector()); + LaboratoryService.get().registerDemographicsProvider(new mGAPDemographicsProvider()); + SystemMaintenance.addTask(new mGapMaintenanceTask()); new PipelineStartup(); @@ -138,6 +142,7 @@ public PipelineStartup() SequencePipelineService.get().registerPipelineStep(new GenerateMgapTracksStep.Provider()); SequencePipelineService.get().registerPipelineStep(new IndexVariantsForMgapStep.Provider()); SequencePipelineService.get().registerPipelineStep(new mGapReleaseAlleleFreqStep.Provider()); + SequencePipelineService.get().registerPipelineStep(new GeographicOriginStep.Provider()); _hasRegistered = true; } diff --git a/mGAP/src/org/labkey/mgap/mGAPSchema.java b/mGAP/src/org/labkey/mgap/mGAPSchema.java index 9a3ed5aa8..b9a06a5fe 100644 --- a/mGAP/src/org/labkey/mgap/mGAPSchema.java +++ b/mGAP/src/org/labkey/mgap/mGAPSchema.java @@ -38,6 +38,7 @@ public class mGAPSchema public static final String TABLE_DEMOGRAPHICS = "demographics"; public static final String TABLE_SUBJECT_SOURCE = "subjectsSource"; public static final String TABLE_SEQUENCE_DATASETS = "sequenceDatasets"; + public static final String TABLE_GENETIC_MEASUREMENTS = "geneticMeasurements"; public static mGAPSchema getInstance() diff --git a/mGAP/src/org/labkey/mgap/mGapMaintenanceTask.java b/mGAP/src/org/labkey/mgap/mGapMaintenanceTask.java index f3c45f906..85a425231 100644 --- a/mGAP/src/org/labkey/mgap/mGapMaintenanceTask.java +++ b/mGAP/src/org/labkey/mgap/mGapMaintenanceTask.java @@ -216,6 +216,12 @@ private void inspectReleaseFolder(String releaseId, File baseDir, Container c, U { expectedFiles.add(new File(f.getPath() + ".tbi")); } + + if ("sitesOnlyVcfId".equals(field)) + { + checkSymlink(log, f, releaseId, commandsToRun); + checkSymlink(log, new File(f.getPath() + ".tbi"), releaseId, commandsToRun); + } } }); diff --git a/mGAP/src/org/labkey/mgap/pipeline/AnnotationStep.java b/mGAP/src/org/labkey/mgap/pipeline/AnnotationStep.java index b5edd6da6..96dd56b56 100644 --- a/mGAP/src/org/labkey/mgap/pipeline/AnnotationStep.java +++ b/mGAP/src/org/labkey/mgap/pipeline/AnnotationStep.java @@ -112,7 +112,7 @@ public Provider() } @Override - public PipelineStep create(PipelineContext context) + public AnnotationStep create(PipelineContext context) { return new AnnotationStep(this, context); } diff --git a/mGAP/src/org/labkey/mgap/pipeline/GenerateMgapTracksStep.java b/mGAP/src/org/labkey/mgap/pipeline/GenerateMgapTracksStep.java index 5b28df51a..df573c6ec 100644 --- a/mGAP/src/org/labkey/mgap/pipeline/GenerateMgapTracksStep.java +++ b/mGAP/src/org/labkey/mgap/pipeline/GenerateMgapTracksStep.java @@ -86,7 +86,7 @@ public Provider() } @Override - public PipelineStep create(PipelineContext context) + public GenerateMgapTracksStep create(PipelineContext context) { return new GenerateMgapTracksStep(this, context); } diff --git a/primeseq/src/org/labkey/primeseq/pipeline/GeographicOriginStep.java b/mGAP/src/org/labkey/mgap/pipeline/GeographicOriginStep.java similarity index 61% rename from primeseq/src/org/labkey/primeseq/pipeline/GeographicOriginStep.java rename to mGAP/src/org/labkey/mgap/pipeline/GeographicOriginStep.java index f1634cd48..b6da9bf1c 100644 --- a/primeseq/src/org/labkey/primeseq/pipeline/GeographicOriginStep.java +++ b/mGAP/src/org/labkey/mgap/pipeline/GeographicOriginStep.java @@ -1,29 +1,48 @@ -package org.labkey.primeseq.pipeline; +package org.labkey.mgap.pipeline; +import au.com.bytecode.opencsv.CSVReader; import htsjdk.samtools.util.Interval; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.Nullable; import org.json.JSONObject; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.data.TableInfo; +import org.labkey.api.pipeline.PipelineJob; import org.labkey.api.pipeline.PipelineJobException; +import org.labkey.api.query.BatchValidationException; +import org.labkey.api.query.DuplicateKeyException; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.QueryUpdateServiceException; +import org.labkey.api.reader.Readers; +import org.labkey.api.sequenceanalysis.SequenceOutputFile; import org.labkey.api.sequenceanalysis.pipeline.AbstractPipelineStep; import org.labkey.api.sequenceanalysis.pipeline.AbstractVariantProcessingStepProvider; +import org.labkey.api.sequenceanalysis.pipeline.PedigreeToolParameterDescriptor; import org.labkey.api.sequenceanalysis.pipeline.PipelineContext; import org.labkey.api.sequenceanalysis.pipeline.PipelineStepProvider; import org.labkey.api.sequenceanalysis.pipeline.ReferenceGenome; +import org.labkey.api.sequenceanalysis.pipeline.SequenceAnalysisJobSupport; import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService; import org.labkey.api.sequenceanalysis.pipeline.ToolParameterDescriptor; import org.labkey.api.sequenceanalysis.pipeline.VariantProcessingStep; import org.labkey.api.sequenceanalysis.pipeline.VariantProcessingStepOutputImpl; import org.labkey.api.sequenceanalysis.run.AbstractDiscvrSeqWrapper; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.mgap.mGAPSchema; import java.io.File; +import java.io.IOException; +import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Map; public class GeographicOriginStep extends AbstractPipelineStep implements VariantProcessingStep { - public GeographicOriginStep(PipelineStepProvider provider, PipelineContext ctx) + public static final String CATEGORY = "Macaque Geographic Origin Scores"; + + public GeographicOriginStep(PipelineStepProvider provider, PipelineContext ctx) { super(provider, ctx); } @@ -31,7 +50,7 @@ public GeographicOriginStep(PipelineStepProvider provider, PipelineContext ctx) private static final String INDIAN_MARKERS = "indianMarkers"; private static final String CHINESE_MARKERS = "chineseMarkers"; - public static class Provider extends AbstractVariantProcessingStepProvider implements VariantProcessingStep.RequiresPedigree + public static class Provider extends AbstractVariantProcessingStepProvider implements RequiresPedigree { public Provider() { @@ -43,8 +62,10 @@ public Provider() ToolParameterDescriptor.createExpDataParam(CHINESE_MARKERS, "Chinese-origin Markers", "This is the ID of a VCF file with markers of chinese-origin.", "ldk-expdatafield", new JSONObject() {{ put("allowBlank", false); - }}, null) - ), null, "https://bimberlab.github.io/DISCVRSeq/"); + }}, null), + ToolParameterDescriptor.create("storeResults", "Store Results", "If checked, the results will be stored in the database", "checkbox", null, null), + new PedigreeToolParameterDescriptor() + ), PageFlowUtil.set(PedigreeToolParameterDescriptor.getClientDependencyPath()), "https://bimberlab.github.io/DISCVRSeq/"); } @Override @@ -54,6 +75,54 @@ public GeographicOriginStep create(PipelineContext ctx) } } + @Override + public void complete(PipelineJob job, List inputs, List outputsCreated, SequenceAnalysisJobSupport support) throws PipelineJobException + { + // TODO: store in DB + for (SequenceOutputFile so : outputsCreated) + { + if (!CATEGORY.equals(so.getCategory())) + { + continue; + } + + List> toInsert = new ArrayList<>(); + try (CSVReader reader = new CSVReader(Readers.getReader(so.getFile()), '\t')) + { + String[] line; + while ((line = reader.readNext()) != null) + { + // SampleName ReferenceName MarkersWithData MarkersNoData FractionWithData UniqueContigs CumulativeAF MinPossible MaxPossible Score + Map row = new CaseInsensitiveHashMap<>(); + row.put("sampleName", line[0]); + row.put("measurementName", line[1]); + row.put("measurementValue", line[9]); + row.put("dataId", so.getDataId()); + + toInsert.add(row); + } + } + catch (IOException e) + { + throw new PipelineJobException(e); + } + + if (!toInsert.isEmpty()) + { + TableInfo ti = QueryService.get().getUserSchema(job.getUser(), job.getContainer(), mGAPSchema.NAME).getTable(mGAPSchema.TABLE_GENETIC_MEASUREMENTS); + BatchValidationException bve = new BatchValidationException(); + try + { + ti.getUpdateService().insertRows(job.getUser(), job.getContainer(), toInsert, bve, null, null); + } + catch (DuplicateKeyException | BatchValidationException | QueryUpdateServiceException | SQLException e) + { + throw new PipelineJobException(e); + } + } + } + } + @Override public Output processVariants(File inputVCF, File outputDirectory, ReferenceGenome genome, @Nullable List intervals) throws PipelineJobException { @@ -97,7 +166,7 @@ public Output processVariants(File inputVCF, File outputDirectory, ReferenceGeno output.addInput(genome.getWorkingFastaFile(), "Reference Genome"); output.addOutput(outputTable, "Macaque Geographic Origin Scores"); - output.addSequenceOutput(outputTable, "Macaque Geographic Origin Scores for: " + inputVCF.getName(), "Macaque Geographic Origin Scores", null, null, genome.getGenomeId(), null); + output.addSequenceOutput(outputTable, "Macaque Geographic Origin Scores for: " + inputVCF.getName(), CATEGORY, null, null, genome.getGenomeId(), null); return output; } diff --git a/mGAP/src/org/labkey/mgap/pipeline/RemoveAnnotationsForMgapStep.java b/mGAP/src/org/labkey/mgap/pipeline/RemoveAnnotationsForMgapStep.java index 4b83e59ac..54f58e519 100644 --- a/mGAP/src/org/labkey/mgap/pipeline/RemoveAnnotationsForMgapStep.java +++ b/mGAP/src/org/labkey/mgap/pipeline/RemoveAnnotationsForMgapStep.java @@ -45,7 +45,7 @@ public Provider() } @Override - public PipelineStep create(PipelineContext context) + public RemoveAnnotationsForMgapStep create(PipelineContext context) { return new RemoveAnnotationsForMgapStep(this, context); } diff --git a/mGAP/src/org/labkey/mgap/pipeline/RemoveAnnotationsStep.java b/mGAP/src/org/labkey/mgap/pipeline/RemoveAnnotationsStep.java index 378a2ea3d..71a87dead 100644 --- a/mGAP/src/org/labkey/mgap/pipeline/RemoveAnnotationsStep.java +++ b/mGAP/src/org/labkey/mgap/pipeline/RemoveAnnotationsStep.java @@ -60,7 +60,7 @@ public Provider() } @Override - public PipelineStep create(PipelineContext context) + public RemoveAnnotationsStep create(PipelineContext context) { return new RemoveAnnotationsStep(this, context); } diff --git a/mGAP/src/org/labkey/mgap/pipeline/RenameSamplesForMgapStep.java b/mGAP/src/org/labkey/mgap/pipeline/RenameSamplesForMgapStep.java index 527817b36..5751ae9a1 100644 --- a/mGAP/src/org/labkey/mgap/pipeline/RenameSamplesForMgapStep.java +++ b/mGAP/src/org/labkey/mgap/pipeline/RenameSamplesForMgapStep.java @@ -67,7 +67,7 @@ public Provider() } @Override - public PipelineStep create(PipelineContext context) + public RenameSamplesForMgapStep create(PipelineContext context) { return new RenameSamplesForMgapStep(this, context); } diff --git a/mGAP/src/org/labkey/mgap/query/VariantReleaseDisplayColumnFactory.java b/mGAP/src/org/labkey/mgap/query/VariantReleaseDisplayColumnFactory.java index c96f8a8ed..8b7b61c7c 100644 --- a/mGAP/src/org/labkey/mgap/query/VariantReleaseDisplayColumnFactory.java +++ b/mGAP/src/org/labkey/mgap/query/VariantReleaseDisplayColumnFactory.java @@ -10,11 +10,11 @@ import org.labkey.api.query.DetailsURL; import org.labkey.api.query.FieldKey; import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.view.HttpView; import org.labkey.api.view.template.ClientDependency; import java.io.IOException; import java.io.Writer; -import java.util.Collections; import java.util.Set; /** @@ -45,13 +45,22 @@ private FieldKey getBoundKey(String colName) return new FieldKey(getBoundColumn().getFieldKey().getParent(), colName); } + private boolean _clickHandlerRegistered = false; + @Override public void renderGridCellContents(RenderContext ctx, Writer out) throws IOException { Integer rowId = ctx.get(getBoundKey("rowid"), Integer.class); if (rowId != null) { - out.write("Download"); + out.write("Download"); + + if (!_clickHandlerRegistered) + { + HttpView.currentPageConfig().addHandlerForQuerySelector("a.vrdc-row", "click", "mGAP.window.DownloadWindow.buttonHandler(this.attributes.getNamedItem('data-rowid').value); return false;"); + + _clickHandlerRegistered = true; + } } String jbrowseId = ctx.get(getBoundKey("jbrowseId"), String.class); diff --git a/mcc/package-lock.json b/mcc/package-lock.json index 1cdfcb4d0..68e5f5e0e 100644 --- a/mcc/package-lock.json +++ b/mcc/package-lock.json @@ -6694,9 +6694,9 @@ } }, "node_modules/follow-redirects": { - "version": "1.15.4", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.4.tgz", - "integrity": "sha512-Cr4D/5wlrb0z9dgERpUL3LrmPKVDsETIJhaCMeDfuFYcqa5bldGV6wBsAN6X/vxlXQtFBMrXdXxdL8CbDTGniw==", + "version": "1.15.6", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", + "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==", "dev": true, "funding": [ { @@ -13906,9 +13906,9 @@ } }, "node_modules/webpack-dev-middleware": { - "version": "5.3.3", - "resolved": "https://registry.npmjs.org/webpack-dev-middleware/-/webpack-dev-middleware-5.3.3.tgz", - "integrity": "sha512-hj5CYrY0bZLB+eTO+x/j67Pkrquiy7kWepMHmUMoPsmcUaeEnQJqFzHJOyxgWlq746/wUuA64p9ta34Kyb01pA==", + "version": "5.3.4", + "resolved": "https://registry.npmjs.org/webpack-dev-middleware/-/webpack-dev-middleware-5.3.4.tgz", + "integrity": "sha512-BVdTqhhs+0IfoeAf7EoH5WE+exCmqGerHfDM0IL096Px60Tq2Mn9MAbnaGUe6HiMa41KMCYF19gyzZmBcq/o4Q==", "dev": true, "dependencies": { "colorette": "^2.0.10", diff --git a/mcc/resources/data/parentageRelationship.tsv b/mcc/resources/data/parentageRelationship.tsv new file mode 100644 index 000000000..23efed649 --- /dev/null +++ b/mcc/resources/data/parentageRelationship.tsv @@ -0,0 +1,5 @@ +value +Dam +Sire +Surrogate Dam +Foster Dam \ No newline at end of file diff --git a/mcc/resources/data/reports.tsv b/mcc/resources/data/reports.tsv new file mode 100644 index 000000000..5e3587aa8 --- /dev/null +++ b/mcc/resources/data/reports.tsv @@ -0,0 +1,14 @@ +reportname category reporttype reporttitle visible containerpath schemaname queryname viewname report datefieldname todayonly queryhaslocation sort_order QCStateLabelFieldName description +birth General query Birth Records TRUE study birth date FALSE FALSE qcstate/publicdata Birth records +weight General query Weights TRUE study weight date FALSE FALSE qcstate/publicdata This report contains a summary of the animal\'s weight +Flags General query Flags TRUE study flags StartDate FALSE FALSE qcstate/publicdata Animal attribute flags +demographics General query Demographics TRUE study demographics FALSE FALSE qcstate/publicdata This report displays the demographics data about each animal including species, sex and birth +death General query Death Records TRUE study deaths date FALSE FALSE qcstate/publicdata Death records +departure General query Shipments TRUE study departure date FALSE FALSE qcstate/publicdata Displays departure dates +obs General query Observations TRUE study clinical_observations date FALSE FALSE qcstate/publicdata This report displays observations for the selected animal +pedigreePlot Genetics report Pedigree Plot TRUE study pedigree module:EHR/schemas/study/Pedigree/Pedigree.r FALSE FALSE qcstate/publicdata This report will generate a pedigree plot for the selected animal +offspring General query Offspring TRUE study demographicsOffspring FALSE FALSE qcstate/publicdata This report displays pedigree data for animals, including parents, grandparents, siblings and offspring +kinship Genetics query Kinship TRUE study kinship FALSE FALSE qcstate/publicdata This report shows the kinship coefficient between every animal in the colony. The kinship coefficient is a measure of relatedness between two individuals. It represents the probability that two genes, sampled at random from each individual are identical (e.g. the kinship coefficient between a parent and an offspring is 0.25). +parentage Genetics query Parentage TRUE study demographicsParents FALSE FALSE qcstate/publicdata This report shows information about the parentage of each animal, drawing from genetic data and observations +geneticData Genetics query Genomic Datasets TRUE study genomicDatasets FALSE FALSE qcstate/publicdata This report summarizes available genomic datasets +mostRecentObservations General query Most Recent Observations TRUE study mostRecentObservations FALSE FALSE This report summarizes the most recent observations per animal diff --git a/mcc/resources/etls/mcc-genomics.xml b/mcc/resources/etls/mcc-genomics.xml new file mode 100644 index 000000000..4e567a553 --- /dev/null +++ b/mcc/resources/etls/mcc-genomics.xml @@ -0,0 +1,18 @@ + + + MCC_Genomic + MCC Genomic Data + + + + + + + + + + + + + + diff --git a/mcc/resources/etls/mcc.xml b/mcc/resources/etls/mcc.xml index b54a3327b..b4f41a9c4 100644 --- a/mcc/resources/etls/mcc.xml +++ b/mcc/resources/etls/mcc.xml @@ -29,6 +29,8 @@ infant_history fertility_status medical_history + usage_current + usage_future date_of_observations colony dam @@ -60,6 +62,75 @@ + + Copy to target + + + Id + date + weight + objectid + + + + + + + + + + Copy to target + + + Id + date + datatype + sra_accession + objectid + + + + + + + + + + Copy to target + + + Id + date + category + observation + objectid + + + + + + + + + + Copy to target + + + Id + yearNo + startdate + enddate + centerName + totalBreedingPairs + totalLivingOffspring + survivalRates + marmosetsShipped + + + + + + diff --git a/mcc/resources/folderTypes/MCC Colony.folderType.xml b/mcc/resources/folderTypes/MCC Colony.folderType.xml index 6c88300ce..741997139 100644 --- a/mcc/resources/folderTypes/MCC Colony.folderType.xml +++ b/mcc/resources/folderTypes/MCC Colony.folderType.xml @@ -26,6 +26,36 @@ + + animalHistory + Animal History + + + animalHistory + + + + + MCC Animal History + body + + + + + census + Census + + + census + + + + + MCC Census + body + + + MCC diff --git a/mcc/resources/folderTypes/MCC Data.folderType.xml b/mcc/resources/folderTypes/MCC Data.folderType.xml new file mode 100644 index 000000000..7d409cf2b --- /dev/null +++ b/mcc/resources/folderTypes/MCC Data.folderType.xml @@ -0,0 +1,83 @@ + + MCC Data + The default folder layout for the MCC data aggregation folder + + + MCC Menu + menu + + + + + mcc + Overview + + + overview + + + + + + + + MCC Dashboard Webpart + body + + + + + animalData + All Animals + + + animalData + + + + + MCC Demographics + body + + + + + animalHistory + Animal Detail + + + animalHistory + + + + + MCC Animal History + body + + + + + census + Census + + + census + + + + + MCC Census + body + + + + + + MCC + Study + Query + Pipeline + Core + + Core + \ No newline at end of file diff --git a/mcc/resources/queries/mcc/aggregatedDemographics.sql b/mcc/resources/queries/mcc/aggregatedDemographics.sql index 1a059ca63..4f481572f 100644 --- a/mcc/resources/queries/mcc/aggregatedDemographics.sql +++ b/mcc/resources/queries/mcc/aggregatedDemographics.sql @@ -25,6 +25,9 @@ SELECT o.infant_history, o.fertility_status, o.medical_history, + null as usage_current, + null as usage_future, + null as breeding_partner_id, o.date_of_observations, d.container @@ -36,10 +39,10 @@ LEFT JOIN (SELECT o."current_housing_status::observation" as current_housing_status, o."infant_history::observation" as infant_history, o."fertility_status::observation" as fertility_status, - o."medical_history::observation" as medical_history, + o."medical_history::observation" as medical_history FROM "/data/Colonies/SNPRC/".study.mostRecentObservationsPivoted o ) o ON (o.Id = d.Id) -WHERE (d.excludeFromCensus IS NULL or d.excludeFromCensus = false) and d.calculated_status NOT IN ('Unknown', 'Other') +WHERE (d.excludeFromCensus IS NULL or d.excludeFromCensus = false) and d.calculated_status NOT IN ('Other') UNION ALL @@ -70,6 +73,9 @@ SELECT o.infant_history, o.fertility_status, o.medical_history, + null as usage_current, + null as usage_future, + null as breeding_partner_id, o.date_of_observations, d.container @@ -112,6 +118,9 @@ SELECT o.infant_history, o.fertility_status, o.medical_history, + o.usage_current, + o.usage_future, + o.breeding_partner_id, o.date_of_observations, d.container @@ -124,6 +133,9 @@ FROM "/data/Colonies/UCSD/".study.demographics d o."infant_history::observation" as infant_history, o."fertility_status::observation" as fertility_status, o."medical_history::observation" as medical_history, + o."usage_current::observation" as usage_current, + o."usage_current::observation" as usage_future, + o."breeding_partner_id::observation" as breeding_partner_id FROM "/data/Colonies/UCSD/".study.mostRecentObservationsPivoted o ) o ON (o.Id = d.Id) WHERE (d.excludeFromCensus IS NULL or d.excludeFromCensus = false) @@ -154,6 +166,9 @@ SELECT o.infant_history, o.fertility_status, o.medical_history, + o.usage_current, + o.usage_future, + o.breeding_partner_id, o.date_of_observations, d.container @@ -166,6 +181,9 @@ FROM "/data/Colonies/Other/".study.demographics d o."infant_history::observation" as infant_history, o."fertility_status::observation" as fertility_status, o."medical_history::observation" as medical_history, + o."usage_current::observation" as usage_current, + o."usage_current::observation" as usage_future, + o."breeding_partner_id::observation" as breeding_partner_id FROM "/data/Colonies/Other/".study.mostRecentObservationsPivoted o ) o ON (o.Id = d.Id) WHERE (d.excludeFromCensus IS NULL or d.excludeFromCensus = false) \ No newline at end of file diff --git a/mcc/resources/queries/mcc/aggregatedDemographics/.qview.xml b/mcc/resources/queries/mcc/aggregatedDemographics/.qview.xml index a5b17c161..834f038ee 100644 --- a/mcc/resources/queries/mcc/aggregatedDemographics/.qview.xml +++ b/mcc/resources/queries/mcc/aggregatedDemographics/.qview.xml @@ -21,6 +21,9 @@ + + + diff --git a/mcc/resources/queries/mcc/aggregatedKinship.sql b/mcc/resources/queries/mcc/aggregatedKinship.sql deleted file mode 100644 index c5e82fba2..000000000 --- a/mcc/resources/queries/mcc/aggregatedKinship.sql +++ /dev/null @@ -1,57 +0,0 @@ -SELECT - d.Id.mccAlias.externalAlias as Id, - d.Id as originalId, - d.date, - d.Id2MccAlias.externalAlias as Id2, - d.Id2 as originalId2, - d.kinship, - d.relationship, - d.objectid, - d.container - -FROM "/data/Colonies/SNPRC/".study.kinship d - -UNION ALL - -SELECT - d.Id.mccAlias.externalAlias as Id, - d.Id as originalId, - d.date, - d.Id2MccAlias.externalAlias as Id2, - d.Id2 as originalId2, - d.kinship, - d.relationship, - d.objectid, - d.container - -FROM "/data/Colonies/WNPRC/".study.kinship d - -UNION ALL - -SELECT - d.Id.mccAlias.externalAlias as Id, - d.Id as originalId, - d.date, - d.Id2MccAlias.externalAlias as Id2, - d.Id2 as originalId2, - d.kinship, - d.relationship, - d.objectid, - d.container - -FROM "/data/Colonies/UCSD/".study.kinship d - -UNION ALL - -SELECT - d.Id.mccAlias.externalAlias as Id, - d.Id as originalId, - d.date, - d.Id2MccAlias.externalAlias as Id2, - d.Id2 as originalId2, - d.kinship, - d.relationship, - d.objectid, - d.container - -FROM "/data/Colonies/Other/".study.kinship d \ No newline at end of file diff --git a/mcc/resources/queries/mcc/genomicDatasetsSource.query.xml b/mcc/resources/queries/mcc/genomicDatasetsSource.query.xml new file mode 100644 index 000000000..a2ebcc6c6 --- /dev/null +++ b/mcc/resources/queries/mcc/genomicDatasetsSource.query.xml @@ -0,0 +1,14 @@ + + + + + MCC Genomic Datasets Source + + + MCC ID + + +
+ + + \ No newline at end of file diff --git a/mcc/resources/queries/mcc/genomicDatasetsSource.sql b/mcc/resources/queries/mcc/genomicDatasetsSource.sql new file mode 100644 index 000000000..1036baafd --- /dev/null +++ b/mcc/resources/queries/mcc/genomicDatasetsSource.sql @@ -0,0 +1,10 @@ +SELECT + + r.subjectid as Id, + r.created as date, + r.application as datatype, + r.sraRuns as sra_accession + +FROM sequenceanalysis.sequence_readsets r + +WHERE r.subjectid LIKE 'MCC%' \ No newline at end of file diff --git a/mcc/resources/queries/mcc/requestScores/.qview.xml b/mcc/resources/queries/mcc/requestScores/.qview.xml index 81956a4f5..3899ec233 100644 --- a/mcc/resources/queries/mcc/requestScores/.qview.xml +++ b/mcc/resources/queries/mcc/requestScores/.qview.xml @@ -5,6 +5,8 @@ + + diff --git a/mcc/resources/queries/study/assignment/.qview.xml b/mcc/resources/queries/study/assignment/.qview.xml index 4c0c2fdb7..59c0fd9f0 100644 --- a/mcc/resources/queries/study/assignment/.qview.xml +++ b/mcc/resources/queries/study/assignment/.qview.xml @@ -10,7 +10,7 @@ - + diff --git a/mcc/resources/queries/study/birth/.qview.xml b/mcc/resources/queries/study/birth/.qview.xml index f1449329f..cbf21f1dc 100644 --- a/mcc/resources/queries/study/birth/.qview.xml +++ b/mcc/resources/queries/study/birth/.qview.xml @@ -11,7 +11,7 @@ - + diff --git a/mcc/resources/queries/study/clinremarks/.qview.xml b/mcc/resources/queries/study/clinremarks/.qview.xml index 78d39cdfc..b76188c72 100644 --- a/mcc/resources/queries/study/clinremarks/.qview.xml +++ b/mcc/resources/queries/study/clinremarks/.qview.xml @@ -6,8 +6,8 @@ - + diff --git a/mcc/resources/queries/study/deaths.query.xml b/mcc/resources/queries/study/deaths.query.xml index 25e309e6e..06a89d4ce 100644 --- a/mcc/resources/queries/study/deaths.query.xml +++ b/mcc/resources/queries/study/deaths.query.xml @@ -15,11 +15,12 @@ Time of Death - Type of Death + Cause of Death ehr_lookups death_cause value + @@ -28,6 +29,7 @@ ehr_lookups death_manner value + diff --git a/mcc/resources/queries/study/deaths/.qview.xml b/mcc/resources/queries/study/deaths/.qview.xml index fa7420819..a66d41d01 100644 --- a/mcc/resources/queries/study/deaths/.qview.xml +++ b/mcc/resources/queries/study/deaths/.qview.xml @@ -5,7 +5,7 @@ - + diff --git a/mcc/resources/queries/study/demographics.query.xml b/mcc/resources/queries/study/demographics.query.xml index 7e2e2457b..3de9b0e1a 100644 --- a/mcc/resources/queries/study/demographics.query.xml +++ b/mcc/resources/queries/study/demographics.query.xml @@ -32,6 +32,12 @@ Medical History + + Usage (Current) + + + Usage (Future) + Date of Observations diff --git a/mcc/resources/queries/study/demographics/.qview.xml b/mcc/resources/queries/study/demographics/.qview.xml index 8c208d409..8950e4cd5 100644 --- a/mcc/resources/queries/study/demographics/.qview.xml +++ b/mcc/resources/queries/study/demographics/.qview.xml @@ -33,6 +33,8 @@ + + diff --git a/mcc/resources/queries/study/demographicsParents.query.xml b/mcc/resources/queries/study/demographicsParents.query.xml new file mode 100644 index 000000000..80993d8cc --- /dev/null +++ b/mcc/resources/queries/study/demographicsParents.query.xml @@ -0,0 +1,19 @@ + + + + + Parents + + + true + + study + animal + id + + + +
+ + + diff --git a/mcc/resources/queries/study/encounters/.qview.xml b/mcc/resources/queries/study/encounters/.qview.xml index f1da9f3a4..05e3ae27d 100644 --- a/mcc/resources/queries/study/encounters/.qview.xml +++ b/mcc/resources/queries/study/encounters/.qview.xml @@ -9,11 +9,8 @@ - - - - + diff --git a/mcc/resources/queries/study/flags/.qview.xml b/mcc/resources/queries/study/flags/.qview.xml index 05349f54b..35d818f7c 100644 --- a/mcc/resources/queries/study/flags/.qview.xml +++ b/mcc/resources/queries/study/flags/.qview.xml @@ -7,8 +7,8 @@ - + diff --git a/mcc/resources/queries/study/genomicDatasets.js b/mcc/resources/queries/study/genomicDatasets.js new file mode 100644 index 000000000..1121c8378 --- /dev/null +++ b/mcc/resources/queries/study/genomicDatasets.js @@ -0,0 +1,2 @@ +require("ehr/triggers").initScript(this); + diff --git a/mcc/resources/queries/study/genomicDatasets.query.xml b/mcc/resources/queries/study/genomicDatasets.query.xml new file mode 100644 index 000000000..c78b85463 --- /dev/null +++ b/mcc/resources/queries/study/genomicDatasets.query.xml @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + Data Type + + + SRA Accession + https://trace.ncbi.nlm.nih.gov/Traces/sra/?run=${sra_accession} + + +
+ + + diff --git a/mcc/resources/queries/study/genomicDatasets/.qview.xml b/mcc/resources/queries/study/genomicDatasets/.qview.xml new file mode 100644 index 000000000..e583f4f41 --- /dev/null +++ b/mcc/resources/queries/study/genomicDatasets/.qview.xml @@ -0,0 +1,12 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/queries/study/kinship/.qview.xml b/mcc/resources/queries/study/kinship/.qview.xml index 71e0ae624..b238340c8 100644 --- a/mcc/resources/queries/study/kinship/.qview.xml +++ b/mcc/resources/queries/study/kinship/.qview.xml @@ -4,12 +4,6 @@ - - - - - - \ No newline at end of file diff --git a/mcc/resources/queries/study/mostRecentObservations.sql b/mcc/resources/queries/study/mostRecentObservations.sql index ce43020e1..f44329c76 100644 --- a/mcc/resources/queries/study/mostRecentObservations.sql +++ b/mcc/resources/queries/study/mostRecentObservations.sql @@ -16,12 +16,12 @@ FROM study.clinical_observations co WHERE co.qcstate.publicdata = true and co.observation is not null AND - co.category IN ('Medical History', 'Fertility Status', 'Infant History', 'Current Housing Status', 'Availability') AND + co.category IN ('Medical History', 'Fertility Status', 'Infant History', 'Current Housing Status', 'Availability', 'Usage_Current', 'Usage_Future', 'Breeding Partner Id') AND co.date = (SELECT max(o.date) asMaxDate FROM study.clinical_observations o WHERE o.Id = co.Id AND o.qcstate.publicdata = true AND - o.category IN ('Medical History', 'Fertility Status', 'Infant History', 'Current Housing Status', 'Availability') AND + o.category IN ('Medical History', 'Fertility Status', 'Infant History', 'Current Housing Status', 'Availability', 'Usage_Current', 'Usage_Future', 'Breeding Partner Id') AND o.observation is not null ) GROUP BY co.id, co.category, co.date diff --git a/mcc/resources/queries/study/mostRecentObservationsPivoted.query.xml b/mcc/resources/queries/study/mostRecentObservationsPivoted.query.xml index dac4e59cb..c5e7098c6 100644 --- a/mcc/resources/queries/study/mostRecentObservationsPivoted.query.xml +++ b/mcc/resources/queries/study/mostRecentObservationsPivoted.query.xml @@ -4,7 +4,10 @@ Most Recent Observations + + true +
diff --git a/mcc/resources/queries/study/pedigree.query.xml b/mcc/resources/queries/study/pedigree.query.xml new file mode 100644 index 000000000..6e1620b87 --- /dev/null +++ b/mcc/resources/queries/study/pedigree.query.xml @@ -0,0 +1,12 @@ + + + + + Pedigree + + + +
+ + + diff --git a/mcc/resources/queries/study/pedigree.sql b/mcc/resources/queries/study/pedigree.sql new file mode 100644 index 000000000..c18ce19e6 --- /dev/null +++ b/mcc/resources/queries/study/pedigree.sql @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2013-2018 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ +SELECT + +d.id as Id, +d.dam as Dam, +d.sire as Sire, + +CASE (d.id.demographics.gender.origGender) + WHEN 'm' THEN 1 + WHEN 'f' THEN 2 + ELSE 3 +END AS gender, +d.id.demographics.gender as gender_code, +CASE (d.id.demographics.calculated_status) + WHEN 'Alive' THEN 0 + ELSE 1 +END +AS status, +d.id.demographics.calculated_status as status_code, +d.id.demographics.species, +'' as Display, +'Demographics' as source + +FROM study.demographicsParents d +WHERE d.numParents > 0 and d.id.demographics.species IS NOT NULL \ No newline at end of file diff --git a/mcc/resources/queries/study/weight/.qview.xml b/mcc/resources/queries/study/weight/.qview.xml index 3a0c5182c..5ea528af7 100644 --- a/mcc/resources/queries/study/weight/.qview.xml +++ b/mcc/resources/queries/study/weight/.qview.xml @@ -5,7 +5,7 @@ - + diff --git a/mcc/resources/referenceStudy/study/datasets/datasets_manifest.xml b/mcc/resources/referenceStudy/study/datasets/datasets_manifest.xml index bab0de73a..4040aaf0b 100644 --- a/mcc/resources/referenceStudy/study/datasets/datasets_manifest.xml +++ b/mcc/resources/referenceStudy/study/datasets/datasets_manifest.xml @@ -61,5 +61,8 @@ + + + diff --git a/mcc/resources/referenceStudy/study/datasets/datasets_metadata.xml b/mcc/resources/referenceStudy/study/datasets/datasets_metadata.xml index e5ae4f48e..912def432 100644 --- a/mcc/resources/referenceStudy/study/datasets/datasets_metadata.xml +++ b/mcc/resources/referenceStudy/study/datasets/datasets_metadata.xml @@ -661,6 +661,12 @@ varchar + + varchar + + + varchar + timestamp @@ -1035,4 +1041,32 @@ Kinship + + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + ptid + + + + timestamp + http://cpas.labkey.com/Study#VisitDate + http://cpas.labkey.com/Study#VisitDate + + + varchar + + + varchar + + + entityid + urn:ehr.labkey.org/#ObjectId + true + + + Genetic Datasets +
diff --git a/mcc/resources/schemas/dbscripts/postgresql/mcc-20.015-20.016.sql b/mcc/resources/schemas/dbscripts/postgresql/mcc-20.015-20.016.sql new file mode 100644 index 000000000..86128d4a0 --- /dev/null +++ b/mcc/resources/schemas/dbscripts/postgresql/mcc-20.015-20.016.sql @@ -0,0 +1,19 @@ +CREATE TABLE mcc.census ( + rowid serial, + yearNo int, + startdate timestamp, + enddate timestamp, + centerName varchar(1000), + totalBreedingPairs int, + totalLivingOffspring int, + survivalRates int, + marmosetsShipped int, + + container entityid, + created timestamp, + createdby userid, + modified timestamp, + modifiedby userid, + + CONSTRAINT PK_census PRIMARY KEY (rowid) +); \ No newline at end of file diff --git a/mcc/resources/schemas/dbscripts/sqlserver/mcc-20.015-20.016.sql b/mcc/resources/schemas/dbscripts/sqlserver/mcc-20.015-20.016.sql new file mode 100644 index 000000000..502ea2a5b --- /dev/null +++ b/mcc/resources/schemas/dbscripts/sqlserver/mcc-20.015-20.016.sql @@ -0,0 +1,19 @@ +CREATE TABLE mcc.census ( + rowid int identity(1,1), + yearNo int, + startdate datetime, + enddate datetime, + centerName varchar(1000), + totalBreedingPairs int, + totalLivingOffspring int, + survivalRates int, + marmosetsShipped int, + + container entityid, + created datetime, + createdby userid, + modified datetime, + modifiedby userid, + + CONSTRAINT PK_census PRIMARY KEY (rowid) +); \ No newline at end of file diff --git a/mcc/resources/schemas/mcc.xml b/mcc/resources/schemas/mcc.xml index d95f6542c..412deaf93 100644 --- a/mcc/resources/schemas/mcc.xml +++ b/mcc/resources/schemas/mcc.xml @@ -776,4 +776,83 @@ + + + + + + + MCC Census + DETAILED + rowid + + + Row Id + false + true + + + Year # + false + + + Period Start + false + + + Period End + false + + + Center Name + false + + + # Breeding Pairs + + + # Living Offspring + + + Survival Rates + + + # Animals Shipped + + + true + + + true + + + false + false + false + true + 29 + true + + + true + + + false + false + false + true + 29 + true + + + + ldk.context + + + +
diff --git a/mcc/resources/views/animalHistory.html b/mcc/resources/views/animalHistory.html new file mode 100644 index 000000000..512aea1da --- /dev/null +++ b/mcc/resources/views/animalHistory.html @@ -0,0 +1,32 @@ + \ No newline at end of file diff --git a/mcc/resources/views/animalHistory.view.xml b/mcc/resources/views/animalHistory.view.xml new file mode 100644 index 000000000..885cf5f88 --- /dev/null +++ b/mcc/resources/views/animalHistory.view.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/views/animalHistory.webpart.xml b/mcc/resources/views/animalHistory.webpart.xml new file mode 100644 index 000000000..c26a73a35 --- /dev/null +++ b/mcc/resources/views/animalHistory.webpart.xml @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/mcc/resources/views/mccCensus.html b/mcc/resources/views/mccCensus.html new file mode 100644 index 000000000..9e2ea9418 --- /dev/null +++ b/mcc/resources/views/mccCensus.html @@ -0,0 +1,13 @@ + \ No newline at end of file diff --git a/mcc/resources/views/mccCensus.view.xml b/mcc/resources/views/mccCensus.view.xml new file mode 100644 index 000000000..22104272d --- /dev/null +++ b/mcc/resources/views/mccCensus.view.xml @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/mcc/resources/views/mccCensus.webpart.xml b/mcc/resources/views/mccCensus.webpart.xml new file mode 100644 index 000000000..eed712c4e --- /dev/null +++ b/mcc/resources/views/mccCensus.webpart.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/mcc/resources/views/mccColonies.html b/mcc/resources/views/mccColonies.html new file mode 100644 index 000000000..5df06528c --- /dev/null +++ b/mcc/resources/views/mccColonies.html @@ -0,0 +1,52 @@ + \ No newline at end of file diff --git a/mcc/resources/views/mccColonies.view.xml b/mcc/resources/views/mccColonies.view.xml new file mode 100644 index 000000000..e55aaf566 --- /dev/null +++ b/mcc/resources/views/mccColonies.view.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/views/mccColonies.webpart.xml b/mcc/resources/views/mccColonies.webpart.xml new file mode 100644 index 000000000..df91e7d75 --- /dev/null +++ b/mcc/resources/views/mccColonies.webpart.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/mcc/resources/views/mccColony.html b/mcc/resources/views/mccColony.html index 8006ae21b..03617f758 100644 --- a/mcc/resources/views/mccColony.html +++ b/mcc/resources/views/mccColony.html @@ -71,6 +71,9 @@ },{ name: 'Parent IDs Assigned to Multiple MCC IDs', url: LABKEY.ActionURL.buildURL('query', 'executeQuery', null, {schemaName: 'mcc', queryName: 'duplicatedAggregatedDemographicsParents'}) + },{ + name: 'Aggregated Demographics (All Centers)', + url: LABKEY.ActionURL.buildURL('query', 'executeQuery', null, {schemaName: 'mcc', queryName: 'aggregatedDemographics'}) },{ name: 'Pedigree Export', url: LABKEY.ActionURL.buildURL('query', 'executeQuery', null, {schemaName: 'mcc', queryName: 'mccPedigree'}) diff --git a/mcc/resources/views/mccDemographics.html b/mcc/resources/views/mccDemographics.html new file mode 100644 index 000000000..f51dcf21b --- /dev/null +++ b/mcc/resources/views/mccDemographics.html @@ -0,0 +1,13 @@ + \ No newline at end of file diff --git a/mcc/resources/views/mccDemographics.view.xml b/mcc/resources/views/mccDemographics.view.xml new file mode 100644 index 000000000..c058132f0 --- /dev/null +++ b/mcc/resources/views/mccDemographics.view.xml @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/mcc/resources/views/mccDemographics.webpart.xml b/mcc/resources/views/mccDemographics.webpart.xml new file mode 100644 index 000000000..2e1aaa1d4 --- /dev/null +++ b/mcc/resources/views/mccDemographics.webpart.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/mcc/resources/views/populateData.html b/mcc/resources/views/populateData.html index edba7ff24..4cbae6282 100644 --- a/mcc/resources/views/populateData.html +++ b/mcc/resources/views/populateData.html @@ -145,12 +145,13 @@ schemaName: 'ehr_lookups', queryName: 'lookup_sets' },{ - // label: 'Reports', - // populateFn: 'populateFromFile', - // schemaName: 'ehr', - // queryName: 'reports', - // pk: 'rowid' - // },{ + label: 'Reports', + populateFn: 'populateFromFile', + moduleName: 'mcc', + schemaName: 'ehr', + queryName: 'reports', + pk: 'rowid' + },{ label: 'Species', populateFn: 'populateFromFile', moduleName: 'mcc', @@ -220,6 +221,13 @@ schemaName: 'ehr_lookups', queryName: 'geographic_origins', pk: 'rowid' + },{ + label: 'Parentage Relationships', + populateFn: 'populateFromFile', + moduleName: 'mcc', + schemaName: 'ehr_lookups', + queryName: 'parentageRelationship', + pk: 'rowid' },{ label: 'Gender Codes', populateFn: 'populateFromFile', diff --git a/mcc/resources/web/mcc/panel/MccImportPanel.js b/mcc/resources/web/mcc/panel/MccImportPanel.js index 5b4fa157c..43c0bc4d1 100644 --- a/mcc/resources/web/mcc/panel/MccImportPanel.js +++ b/mcc/resources/web/mcc/panel/MccImportPanel.js @@ -1,5 +1,6 @@ Ext4.define('MCC.panel.MccImportPanel', { extend: 'Ext.panel.Panel', + alias: 'widget.mcc-mccimportpanel', COLUMNS: [{ name: 'existingRecord', @@ -14,6 +15,18 @@ Ext4.define('MCC.panel.MccImportPanel', { alwaysShow: true, allowBlank: false, transform: 'center' + },{ + name: 'death', + labels: ['Death', 'dod', 'DOD (mm/dd/yyyy)'], + allowRowSpan: false, + alwaysShow: true, + allowBlank: true + },{ + name: 'causeOfDeath', + labels: ['Cause of Death'], + allowRowSpan: false, + alwaysShow: true, + allowBlank: true },{ name: 'Id', labels: ['Id', 'animal ID', 'AnimalId', 'MarmId', 'Marm Id'], @@ -34,6 +47,12 @@ Ext4.define('MCC.panel.MccImportPanel', { allowRowSpan: false, allowBlank: true, alwaysShow: true + },{ + name: 'shippingDestination', + labels: ['Shipping Destination'], + allowRowSpan: false, + allowBlank: true, + alwaysShow: true },{ name: 'species', labels: ['Species'], @@ -43,7 +62,7 @@ Ext4.define('MCC.panel.MccImportPanel', { transform: 'species' },{ name: 'birth', - labels: ['Birth', 'DOB'], + labels: ['Birth', 'DOB', 'DOB (mm/dd/yyyy)'], allowRowSpan: false, allowBlank: true, transform: 'genericDate' @@ -58,7 +77,8 @@ Ext4.define('MCC.panel.MccImportPanel', { labels: ['Status'], allowRowSpan: false, allowBlank: true, - alwaysShow: true + alwaysShow: true, + transform: 'status' },{ name: 'dam', labels: ['Dam', 'maternal ID'], @@ -79,7 +99,7 @@ Ext4.define('MCC.panel.MccImportPanel', { transform: 'weight' },{ name: 'weightDate', - labels: ['Date of Weight', 'date of weight'], + labels: ['Date of Weight', 'date of weight', 'date of weight (mm/dd/yyyy)'], alwaysShow: true, allowRowSpan: false, transform: 'genericDate', @@ -100,10 +120,15 @@ Ext4.define('MCC.panel.MccImportPanel', { transform: 'u24' },{ name: 'availability', - labels: ['Available to Transfer', 'availalble to transfer'], + labels: ['Available to Transfer', 'available to transfer'], allowRowSpan: false, allowBlank: true, transform: 'available' + },{ + name: 'breedingPartnerId', + labels: ['Breeding Partner Id'], + allowRowSpan: false, + allowBlank: true },{ name: 'housingStatus', labels: ['Current Housing Status'], @@ -128,6 +153,23 @@ Ext4.define('MCC.panel.MccImportPanel', { allowRowSpan: false, allowBlank: true, transform: 'medicalHistory' + },{ + name: 'currentUsage', + labels: ['Usage (Current)', 'usage (current)'], + allowRowSpan: false, + allowBlank: true, + transform: 'usage' + },{ + name: 'futureUsage', + labels: ['Usage (Future)', 'usage (future)'], + allowRowSpan: false, + allowBlank: true, + transform: 'usage' + },{ + name: 'studyNames', + labels: ['Study Names'], + allowRowSpan: false, + allowBlank: true },{ name: 'errors', labels: ['Warnings/Errors'], @@ -140,7 +182,7 @@ Ext4.define('MCC.panel.MccImportPanel', { stripLeadingNumbers: function(val) { if (val) { - val = val.replace(/^[0-9]( )+-( )+/, ''); + val = val.replace(/^[0-9]( )*[=-]( )*/, ''); } return val; @@ -163,6 +205,14 @@ Ext4.define('MCC.panel.MccImportPanel', { }, transforms: { + animalId: function(val) { + if (val) { + val = val.replace(/^\s+|\s+$/gm,''); + } + + return val; + }, + species: function(val, panel, row) { return val || 'CJ'; }, @@ -201,7 +251,14 @@ Ext4.define('MCC.panel.MccImportPanel', { sex: function(val, panel, row) { val = panel.stripLeadingNumbers(val); - val = panel.enforceAllowableValues(val, ['male', 'female'], row); + val = panel.enforceAllowableValues(val, ['male', 'female', 'unknown'], row); + + return(val); + }, + + status: function(val, panel, row) { + val = panel.stripLeadingNumbers(val); + val = panel.enforceAllowableValues(val, ['Alive', 'Dead', 'Shipped', 'Unknown'], row); return(val); }, @@ -215,7 +272,10 @@ Ext4.define('MCC.panel.MccImportPanel', { housingStatus: function(val, panel, row) { val = panel.stripLeadingNumbers(val); - val = panel.enforceAllowableValues(val, ['singly housed', 'natal family group', 'active breeding', 'social non breeding'], row); + if (val === 'social non breeding') { + val = 'social non-breeding'; + } + val = panel.enforceAllowableValues(val, ['singly housed', 'natal family unit', 'active breeding', 'social non-breeding'], row); return(val); }, @@ -229,7 +289,17 @@ Ext4.define('MCC.panel.MccImportPanel', { medicalHistory: function(val, panel, row) { val = panel.stripLeadingNumbers(val); - val = panel.enforceAllowableValues(val, ['naive animal', 'animal assigned to invasive study'], row); + if (val) { + val = val.replaceAll('naïve', 'naive', 'i'); + } + val = panel.enforceAllowableValues(val, ['naive animal', 'assigned to invasive study'], row); + + return(val); + }, + + usage: function(val, panel, row) { + val = panel.stripLeadingNumbers(val); + val = panel.enforceAllowableValues(val, ['breeding', 'experiment', 'TBD', 'other'], row); return(val); }, @@ -371,7 +441,7 @@ Ext4.define('MCC.panel.MccImportPanel', { LABKEY.Query.selectRows({ schemaName: 'study', queryName: 'demographics', - columns: 'Id,alternateIds,dam,sire,birth,death,colony,objectid,lsid,mccAlias/externalId', + columns: 'Id,alternateIds,dam,sire,birth,death,colony,objectid,lsid,mccAlias/externalId,Id/death/date,Id/MostRecentDeparture/MostRecentDeparture', scope: this, failure: LDK.Utils.getErrorCallback(), success: function(results) { @@ -411,16 +481,16 @@ Ext4.define('MCC.panel.MccImportPanel', { mergeWithDemographics: function(rows, demographicsRecords, errorMsgs) { Ext4.Array.forEach(rows, function(row){ - row.existingRecord = row.Id && demographicsRecords.allIds.indexOf(row.Id) > -1; + row.existingRecord = row.Id && demographicsRecords.allIds.indexOf(row.Id.toLowerCase()) > -1; if (row.existingRecord) { - var existingRecord = demographicsRecords.rowMap[row.Id]; + var existingRecord = demographicsRecords.rowMap[row.Id.toLowerCase()]; if (existingRecord.colony !== row.colony) { row.errors.push('Colony does not match existing row: ' + existingRecord.colony); } else { row.objectId = existingRecord.objectid; - var fields = ['birth', 'dam', 'sire', 'source']; + var fields = ['birth', 'dam', 'sire', 'source', 'alternateIds']; for (var idx in fields) { var fn = fields[idx]; @@ -434,6 +504,13 @@ Ext4.define('MCC.panel.MccImportPanel', { } } + if (existingRecord['Id/death/date']) { + row.existingDeathRecord = true; + } + + if (existingRecord['Id/MostRecentDeparture/MostRecentDeparture']) { + row.existingDepartureRecord = true; + } } //TODO: look for alternateIds? @@ -459,8 +536,8 @@ Ext4.define('MCC.panel.MccImportPanel', { ret.idsByCenter[center] = ret.idsByCenter[center] || []; ret.idsByCenter[center].push(row.Id); - ret.allIds.push(row.Id); - ret.rowMap[row.Id] = row; + ret.allIds.push(row.Id.toLowerCase()); + ret.rowMap[row.Id.toLowerCase()] = row; if (row.alternateIds) { var alterateIds = LDK.Utils.textToArray(row.alternateIds); @@ -661,7 +738,11 @@ Ext4.define('MCC.panel.MccImportPanel', { panel: this } }], - columns: columns + columns: columns, + initComplete: function(){ + var width = jQuery(id).width(); + Ext4.ComponentQuery.query('mcc-mccimportpanel')[0].setWidth(width + 100); + } }); previewArea.doLayout(); @@ -976,6 +1057,8 @@ Ext4.define('MCC.panel.MccImportPanel', { var observationInserts = []; var weightInserts = []; + var deathInserts = []; + var departureInserts = [] Ext4.Array.forEach(rawData, function(row){ if (row.existingRecord) { @@ -987,7 +1070,7 @@ Ext4.define('MCC.panel.MccImportPanel', { dam: row.dam, sire: row.sire, alternateIds: row.alternateIds, - colony: row.colony, + colony: row.shippingDestination || row.colony, source: row.source, gender: row.gender, u24_status: row.u24_status, @@ -1005,7 +1088,7 @@ Ext4.define('MCC.panel.MccImportPanel', { dam: row.dam, sire: row.sire, alternateIds: row.alternateIds, - colony: row.colony, + colony: row.shippingDestination || row.colony, source: row.source, gender: row.gender, u24_status: row.u24_status, @@ -1026,12 +1109,46 @@ Ext4.define('MCC.panel.MccImportPanel', { }); } + if (row.death) { + if (row.existingDeathRecord) { + console.log('Death record already exists: ' + row.Id); + } + else { + deathInserts.push({ + Id: row.Id, + date: row.death, + cause: row.causeOfDeath, + objectId: LABKEY.Utils.generateUUID().toUpperCase(), + QCStateLabel: 'Completed' + }); + } + } + + if (row.shippingDestination) { + if (row.existingDepartureRecord) { + console.log('Departure record already exists: ' + row.Id); + } + else { + departureInserts.push({ + Id: row.Id, + date: row.date, + destination: row.shippingDestination, + objectId: LABKEY.Utils.generateUUID().toUpperCase(), + QCStateLabel: 'Completed' + }); + } + } + var obsFieldMap = { availability: 'Availability', housingStatus: 'Current Housing Status', fertilityStatus: 'Fertility Status', infantHistory: 'Infant History', - medicalHistory: 'Medical History' + medicalHistory: 'Medical History', + currentUsage: 'Usage_Current', + futureUsage: 'Usage_Future', + breedingPartnerId: 'Breeding Partner Id', + studyNames: 'Study Names' } Ext4.Array.forEach(Ext4.Object.getKeys(obsFieldMap), function(fn){ @@ -1085,6 +1202,24 @@ Ext4.define('MCC.panel.MccImportPanel', { }); } + if (deathInserts.length) { + commands.push({ + command: 'insert', + schemaName: 'study', + queryName: 'deaths', + rows: deathInserts + }); + } + + if (departureInserts.length) { + commands.push({ + command: 'insert', + schemaName: 'study', + queryName: 'departures', + rows: departureInserts + }); + } + LABKEY.Query.saveRows({ commands: commands, scope: this, @@ -1094,6 +1229,8 @@ Ext4.define('MCC.panel.MccImportPanel', { '
Demographic Records Created: ' + demographicsInserts.length + '
Demographic Records Updated: ' + demographicsUpdates.length + '
Weight Records Created: ' + weightInserts.length + + '
Death Records Created: ' + deathInserts.length + + '
Departure Records Created: ' + departureInserts.length + '
Observation Records Created: ' + observationInserts.length, function(){ window.location = LABKEY.ActionURL.buildURL('project', 'begin.view'); }, this); diff --git a/mcc/resources/web/mcc/window/RenameIdWindow.js b/mcc/resources/web/mcc/window/RenameIdWindow.js new file mode 100644 index 000000000..0288e6fa7 --- /dev/null +++ b/mcc/resources/web/mcc/window/RenameIdWindow.js @@ -0,0 +1,99 @@ +Ext4.define('MCC.window.RenameIdWindow', { + extend: 'Ext.window.Window', + + statics: { + buttonHandler: function (dataRegionName) { + Ext4.create('MCC.window.RenameIdWindow', { + dataRegionName: dataRegionName + }).show(); + } + }, + + initComponent: function() { + Ext4.apply(this, { + title: 'Rename ID(s)', + bodyStyle: 'padding: 5px;', + defaults: { + bodyStyle: 'padding: 5px;' + }, + maxWidth: 550, + items: [{ + html: 'This will globally change all records associated with one animal ID to reference a new ID. This should not be used often, and is primarily intended for the situation where an incorrect ID was entered, or the source center provided a different ID after the fact. If an animal was shipped, it is generally fine to allow existing records to reference the original colony\'s ID, since both old and new IDs should point to the same MCC ID.', + style: 'padding-bottom: 10px;', + width: 500, + border: false + }, { + xtype: 'textarea', + width: 500, + height: 250, + boxLabel: 'Enter Old/New IDs, one pair per line:' + }], + buttons: [{ + text: 'Submit', + scope: this, + handler: this.doSubmit + }, { + text: 'Cancel', + handler: function (btn) { + btn.up('window').close(); + } + }] + }); + + this.callParent(arguments); + }, + + doSubmit: function(btn){ + var text = btn.up('window').down('textarea').getValue(); + if (!text) { + Ext4.Msg.alert('Error', 'No rows entered!'); + return; + } + + var rows = text.split(/[\n\r]/) + if (!rows.length) { + Ext4.Msg.alert('Error', 'No rows entered!'); + return; + } + + var oldToNew = {}; + var hadError = false; + Ext4.Array.forEach(rows, function(r){ + r = r.split(/[\t ;,]/) + if (r.length !== 2) { + Ext4.Msg.alert('Error', 'Row was not two elements long: ' + r.join(',')); + hadError = true; + return false; + } + + oldToNew[r[0]] = r[1]; + }, this); + + if (hadError) { + return; + } + + Ext4.Msg.wait('Saving...'); + LABKEY.Ajax.request({ + method: 'POST', + url: LABKEY.ActionURL.buildURL('mcc', 'renameIds'), + scope: this, + params: { + originalIds: Ext4.Object.getKeys(oldToNew), + newIds: Ext4.Object.getValues(oldToNew) + }, + success: LABKEY.Utils.getCallbackWrapper(function (response) { + Ext4.Msg.hide(); + this.close(); + + if (response.success) { + Ext4.Msg.alert('Success', 'Total IDs Updated: ' + response.totalIdsUpdated + '
Total Records Updated: ' + response.totalRecordsUpdated + (response.messages ? '
' + response.messages : '')); + } + else { + Ext4.Msg.alert('Error', 'Error renaming IDs: ' + response.error); + } + }), + failure: LDK.Utils.getErrorCallback() + }); + } +}); diff --git a/mcc/src/client/AnimalRequest/animal-request.tsx b/mcc/src/client/AnimalRequest/animal-request.tsx index ccacef6f6..f95c74c8d 100644 --- a/mcc/src/client/AnimalRequest/animal-request.tsx +++ b/mcc/src/client/AnimalRequest/animal-request.tsx @@ -704,7 +704,7 @@ export function AnimalRequest() { - + <Title text={"10. " + commentsPlaceholder}/> <div className="tw-w-full tw-px-3 tw-mb-6"> <ErrorMessageHandler isSubmitting={isSubmitting}> <div className="tw-w-full tw-px-3 tw-mb-6"> diff --git a/mcc/src/org/labkey/mcc/MccController.java b/mcc/src/org/labkey/mcc/MccController.java index 2fc49a80f..ef8bbe26b 100644 --- a/mcc/src/org/labkey/mcc/MccController.java +++ b/mcc/src/org/labkey/mcc/MccController.java @@ -24,6 +24,8 @@ import org.labkey.api.action.ConfirmAction; import org.labkey.api.action.MutatingApiAction; import org.labkey.api.action.SpringActionController; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.collections.CaseInsensitiveHashSet; import org.labkey.api.data.CompareType; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerManager; @@ -38,8 +40,12 @@ import org.labkey.api.module.ModuleLoader; import org.labkey.api.module.ModuleProperty; import org.labkey.api.pipeline.PipelineUrls; +import org.labkey.api.query.BatchValidationException; import org.labkey.api.query.DetailsURL; import org.labkey.api.query.FieldKey; +import org.labkey.api.query.InvalidKeyException; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.QueryUpdateServiceException; import org.labkey.api.security.AuthenticationManager; import org.labkey.api.security.Group; import org.labkey.api.security.GroupManager; @@ -57,8 +63,12 @@ import org.labkey.api.settings.AppProps; import org.labkey.api.settings.LookAndFeelProperties; import org.labkey.api.studies.StudiesService; +import org.labkey.api.study.Dataset; +import org.labkey.api.study.Study; +import org.labkey.api.study.StudyService; import org.labkey.api.util.ConfigurationException; import org.labkey.api.util.ExceptionUtil; +import org.labkey.api.util.GUID; import org.labkey.api.util.HtmlString; import org.labkey.api.util.MailHelper; import org.labkey.api.util.PageFlowUtil; @@ -66,6 +76,7 @@ import org.labkey.api.util.URLHelper; import org.labkey.api.view.HtmlView; import org.labkey.mcc.etl.ZimsImportTask; +import org.labkey.mcc.security.MccDataAdminPermission; import org.labkey.mcc.security.MccDataAdminRole; import org.labkey.mcc.security.MccFinalReviewerRole; import org.labkey.mcc.security.MccRabReviewerRole; @@ -78,6 +89,7 @@ import jakarta.mail.Address; import jakarta.mail.Message; +import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -85,7 +97,9 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; +import java.util.stream.IntStream; public class MccController extends SpringActionController { @@ -690,6 +704,9 @@ public boolean handlePost(Object o, BindException errors) throws Exception ModuleProperty mp = ehr.getModuleProperties().get("EHRStudyContainer"); mp.saveValue(getUser(), getContainer(), getContainer().getPath()); + ModuleProperty mp2 = ehr.getModuleProperties().get("DefaultAnimalHistoryReport"); + mp2.saveValue(getUser(), getContainer(), "demographics"); + StudiesService.get().importFolderDefinition(getContainer(), getUser(), ModuleLoader.getInstance().getModule(MccModule.NAME), new Path("referenceStudy")); return true; @@ -781,4 +798,169 @@ public void setRowIds(Integer[] rowIds) this.rowIds = rowIds; } } + + @RequiresPermission(MccDataAdminPermission.class) + public class RenameIdsAction extends MutatingApiAction<RenameIdsForm> + { + @Override + public Object execute(RenameIdsForm form, BindException errors) throws Exception + { + if (form.getOriginalIds() == null || form.getOriginalIds().length == 0) + { + errors.reject(ERROR_MSG, "Must provide a list of IDs to change"); + return null; + } + + if (form.getNewIds() == null || form.getOriginalIds().length != form.getNewIds().length) + { + errors.reject(ERROR_MSG, "Differing number of Original/New IDs were provided"); + return null; + } + + Study s = StudyService.get().getStudy(getContainer()); + if (s == null) + { + errors.reject(ERROR_MSG, "There is no study in this container"); + return null; + } + + final Map<String, String> oldToNew = new CaseInsensitiveHashMap<>(); + IntStream.range(0, form.getOriginalIds().length).forEach(i -> oldToNew.put(form.getOriginalIds()[i], form.getNewIds()[i])); + + final Set<String> idsUpdated = new CaseInsensitiveHashSet(); + final AtomicInteger totalRecordsUpdated = new AtomicInteger(); + final TableInfo mccAliases = QueryService.get().getUserSchema(getUser(), getContainer(), MccSchema.NAME).getTable(MccSchema.TABLE_ANIMAL_MAPPING); + + try (DbScope.Transaction transaction = DbScope.getLabKeyScope().ensureTransaction()) + { + // NOTE: include this value so it will get added to the audit trail. This is a loose way to connect changes made in this transaction + final String batchId = "MCC.Rename." + new GUID(); + Set<String> messages = new HashSet<>(); + for (Dataset ds : s.getDatasets()) + { + TableInfo ti = ds.getTableInfo(getUser()); + TableSelector ts = new TableSelector(ti, PageFlowUtil.set("Id", "lsid"), new SimpleFilter(FieldKey.fromString("Id"), oldToNew.keySet(), CompareType.IN), null); + if (!ts.exists()) + { + continue; + } + + List<Map<String, Object>> toUpdate = new ArrayList<>(); + List<Map<String, Object>> oldKeys = new ArrayList<>(); + ts.forEachResults(rs -> { + if (ds.isDemographicData()) + { + // test if a record exists with the new ID + if (new TableSelector(ti, new SimpleFilter(FieldKey.fromString("Id"), oldToNew.get(rs.getString(FieldKey.fromString("Id")))), null).exists()) + { + messages.add("Existing record for ID: " + oldToNew.get(rs.getString(FieldKey.fromString("Id"))) + " for dataset: " + ds.getLabel() + " in container: " + getContainer().getPath() + ", skipping rename"); + return; + } + } + + toUpdate.add(Map.of("lsid", rs.getString(FieldKey.fromString("lsid")), "Id", oldToNew.get(rs.getString(FieldKey.fromString("Id"))), "_batchId_", batchId)); + oldKeys.add(Map.of("lsid", rs.getString(FieldKey.fromString("lsid")))); + idsUpdated.add(rs.getString(FieldKey.fromString("Id"))); + totalRecordsUpdated.getAndIncrement(); + }); + + if (!toUpdate.isEmpty()) + { + try + { + ti.getUpdateService().updateRows(getUser(), getContainer(), toUpdate, oldKeys, null, null); + } + catch (InvalidKeyException | BatchValidationException | QueryUpdateServiceException | SQLException e) + { + _log.error("Error updating MCC dataset rows", e); + errors.reject(ERROR_MSG, "Error updating MCC dataset rows: " + e.getMessage()); + return null; + } + } + } + + for (String oldId : oldToNew.keySet()) + { + // Find the MCC ID of the new ID: + String mccIdForOldId = new TableSelector(mccAliases, PageFlowUtil.set("externalAlias"), new SimpleFilter(FieldKey.fromString("subjectname"), oldId), null).getObject(String.class); + String mccIdForNewId = new TableSelector(mccAliases, PageFlowUtil.set("externalAlias"), new SimpleFilter(FieldKey.fromString("subjectname"), oldToNew.get(oldId)), null).getObject(String.class); + + if (mccIdForOldId == null) + { + // This should not really happen... + _log.error("An MCC rename was performed where the original ID lacked an MCC alias: " + oldId); + messages.add("Missing MCC alias: " + oldId); + } + + if (mccIdForNewId == null) + { + if (mccIdForOldId != null) + { + // Create record for the new ID pointing to the MCC ID of the original + List<Map<String, Object>> toInsert = Arrays.asList(Map.of( + "subjectname", mccIdForOldId, + "_batchId_", batchId + )); + BatchValidationException bve = new BatchValidationException(); + mccAliases.getUpdateService().insertRows(getUser(), getContainer(), toInsert, bve, null, null); + if (bve.hasErrors()) + { + throw bve; + } + } + } + else if (mccIdForOldId != null) + { + messages.add("Both IDs have existing MCC aliases, no changes were made: " + oldId + " / " + oldToNew.get(oldId)); + } + } + + transaction.commit(); + + return new ApiSimpleResponse(Map.of( + "success", true, + "totalIdsUpdated", idsUpdated.size(), + "totalRecordsUpdated", totalRecordsUpdated.get(), + "messages", StringUtils.join(messages, "<br>") + )); + } + catch (Exception e) + { + _log.error("Error renaming MCC IDs", e); + + return new ApiSimpleResponse(Map.of( + "success", false, + "error", e.getMessage() + )); + } + + + } + } + + public static class RenameIdsForm + { + private String[] _originalIds; + private String[] _newIds; + + public String[] getOriginalIds() + { + return _originalIds; + } + + public void setOriginalIds(String[] originalIds) + { + _originalIds = originalIds; + } + + public String[] getNewIds() + { + return _newIds; + } + + public void setNewIds(String[] newIds) + { + _newIds = newIds; + } + } } diff --git a/mcc/src/org/labkey/mcc/MccMaintenanceTask.java b/mcc/src/org/labkey/mcc/MccMaintenanceTask.java index 1d000685c..c9800058a 100644 --- a/mcc/src/org/labkey/mcc/MccMaintenanceTask.java +++ b/mcc/src/org/labkey/mcc/MccMaintenanceTask.java @@ -1,14 +1,18 @@ package org.labkey.mcc; import org.apache.logging.log4j.Logger; +import org.labkey.api.data.CompareType; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.SimpleFilter; import org.labkey.api.data.TableInfo; import org.labkey.api.data.TableSelector; import org.labkey.api.ldk.LDKService; import org.labkey.api.module.ModuleLoader; +import org.labkey.api.query.FieldKey; import org.labkey.api.query.QueryService; import org.labkey.api.security.User; +import org.labkey.api.study.Dataset; import org.labkey.api.study.Study; import org.labkey.api.study.StudyService; import org.labkey.api.util.SystemMaintenance; @@ -81,6 +85,16 @@ private void checkForDuplicateAliases(Logger log) { log.error("Duplicate MCC aliases found in the folder: " + s.getContainer().getPath() + ". Please load the query mcc/duplicateAliases for more detail"); } + + for (Dataset ds : s.getDatasets()) + { + TableInfo dsTableInfo = ds.getTableInfo(u); + long missingRecords = new TableSelector(dsTableInfo, new SimpleFilter(FieldKey.fromString("Id/Demographics/Id"), null, CompareType.ISBLANK), null).getRowCount(); + if (missingRecords > 0) + { + log.warn("Found " + missingRecords + " dataset records with an ID not found in demographics for: " + ds.getLabel() + " / " + ds.getContainer().getPath()); + } + } } } } diff --git a/mcc/src/org/labkey/mcc/MccModule.java b/mcc/src/org/labkey/mcc/MccModule.java index b2b969cdb..c7c4fe169 100644 --- a/mcc/src/org/labkey/mcc/MccModule.java +++ b/mcc/src/org/labkey/mcc/MccModule.java @@ -23,6 +23,7 @@ import org.labkey.api.ehr.EHRService; import org.labkey.api.ldk.ExtendedSimpleModule; import org.labkey.api.ldk.LDKService; +import org.labkey.api.ldk.buttons.ShowEditUIButton; import org.labkey.api.module.Module; import org.labkey.api.module.ModuleContext; import org.labkey.api.query.DefaultSchema; @@ -33,6 +34,7 @@ import org.labkey.api.writer.ContainerUser; import org.labkey.mcc.query.MarkShippedButton; import org.labkey.mcc.query.MccEhrCustomizer; +import org.labkey.mcc.query.RenameIdButton; import org.labkey.mcc.query.ReviewerNotifyButton; import org.labkey.mcc.security.MccDataAdminRole; import org.labkey.mcc.security.MccFinalReviewPermission; @@ -59,7 +61,7 @@ public String getName() @Override public @Nullable Double getSchemaVersion() { - return 20.015; + return 20.016; } @Override @@ -121,6 +123,8 @@ private void registerEHRResources() EHRService.get().registerModule(this); EHRService.get().registerTableCustomizer(this, MccEhrCustomizer.class); EHRService.get().registerMoreActionsButton(new MarkShippedButton(), "study", "demographics"); + EHRService.get().registerMoreActionsButton(new RenameIdButton(), "study", "demographics"); + LDKService.get().registerQueryButton(new ShowEditUIButton(this, MccSchema.NAME, MccSchema.TABLE_CENSUS), MccSchema.NAME, MccSchema.TABLE_CENSUS); } @Override diff --git a/mcc/src/org/labkey/mcc/MccSchema.java b/mcc/src/org/labkey/mcc/MccSchema.java index 1745b67bb..03e4cd3ed 100644 --- a/mcc/src/org/labkey/mcc/MccSchema.java +++ b/mcc/src/org/labkey/mcc/MccSchema.java @@ -31,6 +31,7 @@ public class MccSchema public static final String TABLE_ANIMAL_REQUESTS = "animalRequests"; public static final String TABLE_REQUEST_REVIEWS = "requestReviews"; public static final String TABLE_REQUEST_SCORE = "requestScores"; + public static final String TABLE_CENSUS = "census"; public static MccSchema getInstance() { diff --git a/mcc/src/org/labkey/mcc/MccUserSchema.java b/mcc/src/org/labkey/mcc/MccUserSchema.java index 6b0a6d35a..8a503f291 100644 --- a/mcc/src/org/labkey/mcc/MccUserSchema.java +++ b/mcc/src/org/labkey/mcc/MccUserSchema.java @@ -1,7 +1,9 @@ package org.labkey.mcc; +import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.labkey.api.collections.CaseInsensitiveTreeSet; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerFilter; import org.labkey.api.data.DataColumn; @@ -13,26 +15,70 @@ import org.labkey.api.ldk.table.CustomPermissionsTable; import org.labkey.api.query.ExprColumn; import org.labkey.api.query.FieldKey; +import org.labkey.api.query.QueryDefinition; +import org.labkey.api.query.QueryException; +import org.labkey.api.query.QueryService; import org.labkey.api.query.SimpleUserSchema; import org.labkey.api.security.User; import org.labkey.api.security.permissions.DeletePermission; import org.labkey.api.security.permissions.InsertPermission; import org.labkey.api.security.permissions.ReadPermission; import org.labkey.api.security.permissions.UpdatePermission; +import org.labkey.api.util.logging.LogHelper; +import org.labkey.mcc.security.MccDataAdminPermission; import org.labkey.mcc.security.MccRabReviewPermission; import org.labkey.mcc.security.MccRequestAdminPermission; import org.labkey.mcc.security.MccRequestorPermission; import org.labkey.mcc.security.MccViewRequestsPermission; +import java.util.ArrayList; import java.util.Collections; +import java.util.List; +import java.util.Set; public class MccUserSchema extends SimpleUserSchema { + private static final Logger _log = LogHelper.getLogger(MccUserSchema.class, "MCC UserSchema messages"); + public MccUserSchema(User user, Container container, DbSchema dbschema) { super(MccSchema.NAME, "MCC-specific tables, such as requests", user, container, dbschema); } + @Override + public TableInfo createTable(String name, ContainerFilter cf) + { + if (supportsAggregatedTables()) + { + if (TABLE_AGGREGATED_WEIGHT.equalsIgnoreCase(name)) + { + return getWeightQuery(); + } + else if (TABLE_AGGREGATED_KINSHIP.equalsIgnoreCase(name)) + { + return getKinshipQuery(); + } + else if (TABLE_AGGREGATED_DEPARTURES.equalsIgnoreCase(name)) + { + return getDepartureQuery(); + } + else if (TABLE_AGGREGATED_GENOMICS.equalsIgnoreCase(name)) + { + return getGenomicsQuery(); + } + else if (TABLE_AGGREGATED_OBS.equalsIgnoreCase(name)) + { + return getObsQuery(); + } + else if (TABLE_AGGREGATED_CENSUS.equalsIgnoreCase(name)) + { + return getCensusQuery(); + } + } + + return super.createTable(name, cf); + } + @Override @Nullable protected TableInfo createWrappedTable(String name, @NotNull TableInfo schemaTable, ContainerFilter cf) @@ -74,11 +120,201 @@ else if (MccSchema.TABLE_REQUEST_SCORE.equalsIgnoreCase(name)) return addScoreColumns(ret); } + else if (MccSchema.TABLE_CENSUS.equalsIgnoreCase(name)) + { + CustomPermissionsTable<?> ret = new CustomPermissionsTable<>(this, schemaTable, cf); + ret.addPermissionMapping(InsertPermission.class, MccDataAdminPermission.class); + ret.addPermissionMapping(UpdatePermission.class, MccDataAdminPermission.class); + ret.addPermissionMapping(DeletePermission.class, MccDataAdminPermission.class); + + return ret.init(); + } return super.createWrappedTable(name, schemaTable, cf); } - // TODO: make automatic aggregation queries for ETLs + @Override + public Set<String> getTableNames() + { + Set<String> available = new CaseInsensitiveTreeSet(); + available.addAll(super.getTableNames()); + addAggregatedTableNames(available); + + return available; + } + + private static final String TABLE_AGGREGATED_WEIGHT = "aggregatedWeight"; + private static final String TABLE_AGGREGATED_KINSHIP = "aggregatedKinship"; + private static final String TABLE_AGGREGATED_DEPARTURES = "aggregatedDepartures"; + private static final String TABLE_AGGREGATED_OBS = "aggregatedObservations"; + private static final String TABLE_AGGREGATED_GENOMICS = "aggregatedGenomicDatasets"; + private static final String TABLE_AGGREGATED_CENSUS = "aggregatedCensus"; + + @Override + public Set<String> getVisibleTableNames() + { + Set<String> available = new CaseInsensitiveTreeSet(); + available.addAll(super.getVisibleTableNames()); + addAggregatedTableNames(available); + + return Collections.unmodifiableSet(available); + } + + private boolean supportsAggregatedTables() + { + return getContainer().hasPermission(getUser(), MccDataAdminPermission.class) && MccManager.get().getMCCInternalDataContainer(getContainer()) != null; + } + + private void addAggregatedTableNames(Set<String> available) + { + if (supportsAggregatedTables()) + { + available.add(TABLE_AGGREGATED_KINSHIP); + available.add(TABLE_AGGREGATED_WEIGHT); + available.add(TABLE_AGGREGATED_DEPARTURES); + available.add(TABLE_AGGREGATED_OBS); + available.add(TABLE_AGGREGATED_GENOMICS); + available.add(TABLE_AGGREGATED_CENSUS); + } + } + + private TableInfo getKinshipQuery() + { + String template = "SELECT\n" + + " d.Id.mccAlias.externalAlias as Id,\n" + + " d.Id as originalId,\n" + + " d.date,\n" + + " d.Id2MccAlias.externalAlias as Id2,\n" + + " d.Id2 as originalId2,\n" + + " d.kinship,\n" + + " d.relationship,\n" + + " d.objectid,\n" + + " d.container\n" + + "\n" + + "FROM \"<CONTAINER_PATH>\".study.kinship d WHERE d.qcstate.publicdata = true\n"; + + return makeAggregatedQuery(TABLE_AGGREGATED_KINSHIP, template); + } + + private TableInfo getWeightQuery() + { + String template = "SELECT\n" + + " d.Id.mccAlias.externalAlias as Id,\n" + + " d.Id as originalId,\n" + + " d.date,\n" + + " d.weight,\n" + + " d.objectid,\n" + + " d.container\n" + + "\n" + + "FROM \"<CONTAINER_PATH>\".study.weight d WHERE d.qcstate.publicdata = true\n"; + + return makeAggregatedQuery(TABLE_AGGREGATED_WEIGHT, template); + } + + private TableInfo getCensusQuery() + { + String template = "SELECT\n" + + " d.yearNo,\n" + + " d.startdate,\n" + + " d.enddate,\n" + + " d.centerName,\n" + + " d.totalBreedingPairs,\n" + + " d.totalLivingOffspring,\n" + + " d.survivalRates,\n" + + " d.marmosetsShipped,\n" + + " d.container\n" + + "\n" + + "FROM \"<CONTAINER_PATH>\".mcc.census d\n"; + + return makeAggregatedQuery(TABLE_AGGREGATED_CENSUS, template); + } + + private TableInfo getDepartureQuery() + { + String template = "SELECT\n" + + " d.Id.mccAlias.externalAlias as Id,\n" + + " d.Id as originalId,\n" + + " d.date,\n" + + " d.destination,\n" + + " d.objectid,\n" + + " d.container\n" + + "\n" + + "FROM \"<CONTAINER_PATH>\".study.departure d WHERE d.qcstate.publicdata = true\n"; + + return makeAggregatedQuery(TABLE_AGGREGATED_DEPARTURES, template); + } + + private TableInfo getObsQuery() + { + String template = "SELECT\n" + + " d.Id.mccAlias.externalAlias as Id,\n" + + " d.Id as originalId,\n" + + " d.date,\n" + + " d.category,\n" + + " d.observation,\n" + + " d.objectid,\n" + + " d.container\n" + + "\n" + + "FROM \"<CONTAINER_PATH>\".study.clinical_observations d WHERE d.qcstate.publicdata = true\n"; + + return makeAggregatedQuery(TABLE_AGGREGATED_OBS, template); + } + + private TableInfo getGenomicsQuery() + { + String template = "SELECT\n" + + " d.Id.mccAlias.externalAlias as Id,\n" + + " d.Id as originalId,\n" + + " d.date,\n" + + " d.datatype,\n" + + " d.sra_accession,\n" + + " d.objectid,\n" + + " d.container\n" + + "\n" + + "FROM \"<CONTAINER_PATH>\".study.genomicDatasets d WHERE d.qcstate.publicdata = true\n"; + + return makeAggregatedQuery(TABLE_AGGREGATED_GENOMICS, template); + } + + private TableInfo makeAggregatedQuery(String queryName, String sqlTemplate) + { + if (!getContainer().hasPermission(getUser(), MccDataAdminPermission.class)) + { + return null; + } + + QueryDefinition qd = QueryService.get().createQueryDef(getUser(), getContainer(), this, queryName); + StringBuilder sql = new StringBuilder(); + + Container parent = MccManager.get().getMCCInternalDataContainer(getContainer()); + if (parent == null) + { + return null; + } + + String unionClause = ""; + for (Container c : parent.getChildren()) + { + sql.append(unionClause); + unionClause = " UNION ALL "; + sql.append(sqlTemplate.replaceAll("<CONTAINER_PATH>", c.getPath())); + } + + qd.setSql(sql.toString()); + + List<QueryException> errors = new ArrayList<QueryException>(); + TableInfo ti = qd.getTable(errors, true); + if (!errors.isEmpty()) + { + _log.error("Problem with aggregated query: " + queryName); + for (QueryException e : errors) + { + _log.error(e.getMessage()); + } + } + + return ti; + } private CustomPermissionsTable<?> addScoreColumns(CustomPermissionsTable<?> ti) { diff --git a/mcc/src/org/labkey/mcc/etl/PopulateGeneticDataStep.java b/mcc/src/org/labkey/mcc/etl/PopulateGeneticDataStep.java new file mode 100644 index 000000000..a71c552d0 --- /dev/null +++ b/mcc/src/org/labkey/mcc/etl/PopulateGeneticDataStep.java @@ -0,0 +1,162 @@ +package org.labkey.mcc.etl; + +import org.apache.xmlbeans.XmlException; +import org.jetbrains.annotations.NotNull; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.di.DataIntegrationService; +import org.labkey.api.di.TaskRefTask; +import org.labkey.api.pipeline.PipelineJob; +import org.labkey.api.pipeline.PipelineJobException; +import org.labkey.api.pipeline.RecordedActionSet; +import org.labkey.api.query.BatchValidationException; +import org.labkey.api.query.DuplicateKeyException; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.QueryUpdateServiceException; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.writer.ContainerUser; +import org.labkey.mcc.MccManager; +import org.labkey.mcc.MccSchema; +import org.labkey.remoteapi.CommandException; +import org.labkey.remoteapi.query.SelectRowsCommand; +import org.labkey.remoteapi.query.SelectRowsResponse; + +import java.io.IOException; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class PopulateGeneticDataStep implements TaskRefTask +{ + protected final Map<String, String> _settings = new CaseInsensitiveHashMap<>(); + + protected ContainerUser _containerUser; + + private enum Settings + { + remoteSource() + } + + @Override + public RecordedActionSet run(@NotNull PipelineJob job) throws PipelineJobException + { + populateGeneticData(job); + + return new RecordedActionSet(); + } + + private void populateGeneticData(PipelineJob job) throws PipelineJobException + { + DataIntegrationService.RemoteConnection rc = DataIntegrationService.get().getRemoteConnection(_settings.get(Settings.remoteSource.name()), _containerUser.getContainer(), job.getLogger()); + if (rc == null) + { + throw new PipelineJobException("Unable to find remote connection: " + _settings.get(Settings.remoteSource.name())); + } + + try + { + //first select all rows from remote table + SelectRowsCommand sr = new SelectRowsCommand(MccSchema.NAME, "genomicDatasetsSource"); + sr.setColumns(Arrays.asList("Id", "date", "datatype", "sra_accession")); + + TableInfo aggregatedDemographics = QueryService.get().getUserSchema(job.getUser(), job.getContainer(), MccSchema.NAME).getTable("aggregatedDemographics"); + + //and select all rows from our source table + SelectRowsResponse srr = sr.execute(rc.connection, rc.remoteContainer); + Map<Container, List<Map<String, Object>>> toInsert = new HashMap<>(); + srr.getRows().forEach(x -> { + String mccId = (String)x.get("Id"); + + Collection<Map<String, Object>> rows = new TableSelector(aggregatedDemographics, PageFlowUtil.set("originalId", "container"), new SimpleFilter(FieldKey.fromString("Id"), mccId), null).getMapCollection(); + if (rows.isEmpty()) + { + job.getLogger().error("Unable to find ID: " + mccId); + return; + } + + if (rows.size() > 1) + { + job.getLogger().error("MCC ID linked to multiple containers: " + mccId); + return; + } + + Map<String, Object> row = rows.iterator().next(); + + Container target = ContainerManager.getForId(String.valueOf(row.get("container"))); + if (!toInsert.containsKey(target)) + { + toInsert.put(target, new ArrayList<>()); + } + + Map<String, Object> newRow = new CaseInsensitiveHashMap<>(); + newRow.put("Id", row.get("originalId")); + newRow.put("date", x.get("date")); + newRow.put("datatype", x.get("datatype")); + newRow.put("sra_accession", x.get("sra_accession")); + + toInsert.get(target).add(newRow); + }); + + Container c = MccManager.get().getMCCInternalDataContainer(job.getContainer()); + if (c == null) + { + throw new IllegalStateException("MCCInternalDataContainer not set"); + } + + for (Container child : c.getChildren()) + { + TableInfo gd = QueryService.get().getUserSchema(job.getUser(), child, "study").getTable("genomicDatasets"); + try + { + gd.getUpdateService().truncateRows(job.getUser(), child, null, null); + + if (toInsert.containsKey(child)) + { + BatchValidationException bve = new BatchValidationException(); + gd.getUpdateService().insertRows(job.getUser(), child, toInsert.get(child), bve, null, null); + if (bve.hasErrors()) + { + throw bve; + } + } + } + catch (BatchValidationException | SQLException | DuplicateKeyException | QueryUpdateServiceException e) + { + throw new PipelineJobException(e); + } + } + } + catch (CommandException | IOException e) + { + throw new PipelineJobException(e); + } + } + + @Override + public List<String> getRequiredSettings() + { + return Collections.unmodifiableList(Arrays.asList(Settings.remoteSource.name())); + } + + @Override + public void setSettings(Map<String, String> settings) throws XmlException + { + _settings.putAll(settings); + } + + @Override + public void setContainerUser(ContainerUser containerUser) + { + _containerUser = containerUser; + } +} \ No newline at end of file diff --git a/mcc/src/org/labkey/mcc/query/MccEhrCustomizer.java b/mcc/src/org/labkey/mcc/query/MccEhrCustomizer.java index 817d86a39..ff6a0d31c 100644 --- a/mcc/src/org/labkey/mcc/query/MccEhrCustomizer.java +++ b/mcc/src/org/labkey/mcc/query/MccEhrCustomizer.java @@ -3,8 +3,10 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.labkey.api.data.AbstractTableInfo; +import org.labkey.api.data.BaseColumnInfo; import org.labkey.api.data.ColumnInfo; import org.labkey.api.data.JdbcType; +import org.labkey.api.data.MutableColumnInfo; import org.labkey.api.data.SQLFragment; import org.labkey.api.data.TableInfo; import org.labkey.api.data.WrappedColumn; @@ -12,6 +14,7 @@ import org.labkey.api.query.ExprColumn; import org.labkey.api.query.FieldKey; import org.labkey.api.query.LookupForeignKey; +import org.labkey.api.query.QueryForeignKey; import org.labkey.api.query.UserSchema; import org.labkey.mcc.MccSchema; @@ -74,7 +77,7 @@ private void customizeWeight(AbstractTableInfo ti) private void customizeAnimalTable(AbstractTableInfo ti) { - for (String colName : new String[]{"MostRecentArrival", "numRoommates", "curLocation", "lastHousing", "weightChange", "CageClass", "MhcStatus"}) + for (String colName : new String[]{"MostRecentArrival", "numRoommates", "curLocation", "lastHousing", "weightChange", "CageClass", "MhcStatus", "history"}) { ColumnInfo ci = ti.getColumn(colName); if (ci != null) @@ -84,6 +87,30 @@ private void customizeAnimalTable(AbstractTableInfo ti) } addMccAlias(ti, "Id", "mccAlias", "MCC Alias"); + + if (ti.getColumn("mostRecentObservations") == null) + { + MutableColumnInfo col = getWrappedIdCol(ti.getUserSchema(), ti, "mostRecentObservations", "mostRecentObservationsPivoted"); + col.setLabel("Most Recent Observations"); + col.setDescription("Displays the most recent observation of each category"); + ti.addColumn(col); + } + + } + + private BaseColumnInfo getWrappedIdCol(UserSchema us, AbstractTableInfo ds, String name, String queryName) + { + + String colName = "Id"; + String targetCol = "Id"; + + WrappedColumn col = new WrappedColumn(ds.getColumn(colName), name); + col.setReadOnly(true); + col.setIsUnselectable(true); + col.setUserEditable(false); + col.setFk(new QueryForeignKey(us, null, queryName, targetCol, targetCol)); + + return col; } private void addMccAlias(AbstractTableInfo ti, String sourceCol, String name, String label) diff --git a/mcc/src/org/labkey/mcc/query/RenameIdButton.java b/mcc/src/org/labkey/mcc/query/RenameIdButton.java new file mode 100644 index 000000000..a5fc2ea9a --- /dev/null +++ b/mcc/src/org/labkey/mcc/query/RenameIdButton.java @@ -0,0 +1,18 @@ +package org.labkey.mcc.query; + +import org.labkey.api.ldk.table.SimpleButtonConfigFactory; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.view.template.ClientDependency; +import org.labkey.mcc.MccModule; +import org.labkey.mcc.security.MccDataAdminPermission; + +import java.util.List; + +public class RenameIdButton extends SimpleButtonConfigFactory +{ + public RenameIdButton() + { + super(ModuleLoader.getInstance().getModule(MccModule.class), "Rename ID", "MCC.window.RenameIdWindow.buttonHandler(dataRegionName);", List.of(ClientDependency.supplierFromPath("mcc/window/RenameIdWindow.js"))); + setPermission(MccDataAdminPermission.class); + } +} diff --git a/mcc/src/org/labkey/mcc/query/RequestScoreActionsDisplayColumnFactory.java b/mcc/src/org/labkey/mcc/query/RequestScoreActionsDisplayColumnFactory.java index f6100a7d3..3ca12b80e 100644 --- a/mcc/src/org/labkey/mcc/query/RequestScoreActionsDisplayColumnFactory.java +++ b/mcc/src/org/labkey/mcc/query/RequestScoreActionsDisplayColumnFactory.java @@ -13,6 +13,7 @@ import org.labkey.api.security.User; import org.labkey.api.security.UserManager; import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.view.HttpView; import org.labkey.api.view.template.ClientDependency; import org.labkey.mcc.MccManager; import org.labkey.mcc.security.MccFinalReviewPermission; @@ -32,6 +33,8 @@ public DisplayColumn createRenderer(ColumnInfo colInfo) { return new AbstractMccDisplayColumn(colInfo) { + private boolean _hasRegisteredApprovedHandler = false; + @Override public void renderGridCellContents(RenderContext ctx, Writer out) throws IOException { @@ -94,7 +97,14 @@ else if (st == MccManager.RequestStatus.PendingDecision) } else if (st == MccManager.RequestStatus.Approved) { - out.write("<br><a class=\"labkey-text-link\" onclick=\"MCC.window.ChangeStatusWindow.buttonHandler(" + PageFlowUtil.jsString(ctx.getCurrentRegion().getName()) + "," + requestRowId + ", 'Fulfilled')\">Mark Fulfilled</a>"); + out.write("<br><a class=\"labkey-text-link rsadc-approved\" data-requestrowid=" + PageFlowUtil.jsString(String.valueOf(requestRowId)) + ">Mark Fulfilled</a>"); + + if (!_hasRegisteredApprovedHandler) + { + HttpView.currentPageConfig().addHandlerForQuerySelector("a.rsadc-approved", "click", "MCC.window.ChangeStatusWindow.buttonHandler(" + PageFlowUtil.jsString(ctx.getCurrentRegion().getName()) + ", this.attributes.getNamedItem('data-requestrowid').value, 'Fulfilled'); return false;"); + + _hasRegisteredApprovedHandler = true; + } } } catch (IllegalArgumentException e) @@ -128,9 +138,4 @@ public void addQueryFieldKeys(Set<FieldKey> keys) } }; } - - private String getWithdrawnLine(RenderContext ctx, int requestRowId) - { - return "<br><a class=\"labkey-text-link\" onclick=\"MCC.window.ChangeStatusWindow.buttonHandler(" + PageFlowUtil.jsString(ctx.getCurrentRegion().getName()) + "," + requestRowId + ", 'Withdrawn')\">Withdraw Request</a>"; - } } diff --git a/mcc/src/org/labkey/mcc/query/TriggerHelper.java b/mcc/src/org/labkey/mcc/query/TriggerHelper.java index 957339003..02aed50cb 100644 --- a/mcc/src/org/labkey/mcc/query/TriggerHelper.java +++ b/mcc/src/org/labkey/mcc/query/TriggerHelper.java @@ -279,7 +279,7 @@ public int ensureMccAliasExists(Collection<String> rawIds, Map<Object, Object> e if (ciExistingAliases.containsKey(rs.getString(FieldKey.fromString("subjectname")))) { if (!ciExistingAliases.get(rs.getString(FieldKey.fromString("subjectname"))).equalsIgnoreCase(rs.getString(FieldKey.fromString("externalAlias")))) { - _log.error("Incoming MCC alias for: " + rs.getString(FieldKey.fromString("subjectname")) + " does not match existing: " + rs.getString(FieldKey.fromString("externalAlias"))); + _log.error("Incoming MCC alias for: " + rs.getString(FieldKey.fromString("subjectname")) + "(" + ciExistingAliases.get(rs.getString(FieldKey.fromString("subjectname"))) + ") does not match existing: " + rs.getString(FieldKey.fromString("externalAlias"))); } } }); diff --git a/mcc/test/src/org/labkey/test/tests/mcc/MccTest.java b/mcc/test/src/org/labkey/test/tests/mcc/MccTest.java index fdf94d2aa..b1da3960d 100644 --- a/mcc/test/src/org/labkey/test/tests/mcc/MccTest.java +++ b/mcc/test/src/org/labkey/test/tests/mcc/MccTest.java @@ -68,9 +68,9 @@ public void testMccModule() throws Exception testAnimalImportAndTransfer(); } - private static final String ANIMAL_DATA_HEADER = "animal ID\tprevious IDs\tsource\t\"DOB\n(MM/DD/YYYY)\"\tsex\tmaternal ID\tpaternal ID\t\"weight(grams)\"\t\"date of weight\n(MM/DD/YY)\"\tU24 status\tavailalble to transfer\tcurrent housing status\tinfant history\tfertility status\tmedical history\n"; + private static final String ANIMAL_DATA_HEADER = "animal ID\tprevious IDs\tsource\t\"DOB\n(MM/DD/YYYY)\"\tsex\tmaternal ID\tpaternal ID\t\"weight(grams)\"\t\"date of weight\n(MM/DD/YY)\"\tU24 status\tavailable to transfer\tcurrent housing status\tinfant history\tfertility status\tmedical history\n"; - private static final String ANIMAL_DATA1 = "12345\t\t\t7/10/2011\t0 - male\t23456\t23453\t382.8\t5/19/2021\t0 - not assigned to U24 breeding colony\t0 - not available for transfer\t1 - natal family group\t3 - successful rearing of offspring\t2 - successful offspring produced\t0 - naive animal\n"; + private static final String ANIMAL_DATA1 = "12345\t\t\t7/10/2011\t0 - male\t23456\t23453\t382.8\t5/19/2021\t0 - not assigned to U24 breeding colony\t0 - not available for transfer\t1 - natal family unit\t3 - successful rearing of offspring\t2 - successful offspring produced\t0 - naive animal\n"; private static final String ANIMAL_DATA2 = "Animal2\t\t\t6/3/2015\t1 - female\tDam2\tSire2\t361.2\t1/28/2021\t0 - not assigned to U24 breeding colony\t0 - not available for transfer\t2 - active breeding\t3 - successful rearing of offspring\t2 - successful offspring produced\t0 - naive animal\n"; diff --git a/primeseq/src/org/labkey/primeseq/PrimeseqController.java b/primeseq/src/org/labkey/primeseq/PrimeseqController.java index 855cbf6bb..d93d71258 100644 --- a/primeseq/src/org/labkey/primeseq/PrimeseqController.java +++ b/primeseq/src/org/labkey/primeseq/PrimeseqController.java @@ -311,7 +311,7 @@ private SQLFragment getSql(UpdateFilePathsForm form) sql.append("UPDATE Exp.Data SET DataFileUrl = replace(DataFileUrl, ").appendValue("file://" + sourcePrefix).append(", ").appendValue("file://" + replacementPrefix).append(") "); sql.append("WHERE DataFileUrl like ").appendValue("file://" + sourcePrefix + "%").append("\n"); - sql.append("UPDATE pipeline.StatusFiles SET FilePath = replace(FilePath, ").appendValue(sourcePrefix).append(", ").appendValue(replacementPrefix).append(" "); + sql.append("UPDATE pipeline.StatusFiles SET FilePath = replace(FilePath, ").appendValue(sourcePrefix).append(", ").appendValue(replacementPrefix).append(") "); sql.append("WHERE FilePath like ").appendValue(sourcePrefix + "%"); return sql; diff --git a/primeseq/src/org/labkey/primeseq/PrimeseqModule.java b/primeseq/src/org/labkey/primeseq/PrimeseqModule.java index a924e18fd..c8c3539ae 100644 --- a/primeseq/src/org/labkey/primeseq/PrimeseqModule.java +++ b/primeseq/src/org/labkey/primeseq/PrimeseqModule.java @@ -39,7 +39,6 @@ import org.labkey.primeseq.pipeline.ClusterMaintenanceTask; import org.labkey.primeseq.pipeline.DeleteJobCheckpointButton; import org.labkey.primeseq.pipeline.ExacloudResourceSettings; -import org.labkey.primeseq.pipeline.GeographicOriginStep; import org.labkey.primeseq.pipeline.MhcCleanupPipelineJob; import org.labkey.primeseq.pipeline.SequenceJobResourceAllocator; import org.labkey.primeseq.query.PerformMhcCleanupButton; @@ -121,8 +120,6 @@ public PipelineStartup() SequenceAnalysisService.get().registerFileHandler(new MethylationRateComparisonHandler()); SequenceAnalysisService.get().registerFileHandler(new CombineMethylationRatesHandler()); - SequencePipelineService.get().registerPipelineStep(new GeographicOriginStep.Provider()); - _hasRegistered = true; } } diff --git a/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java b/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java index fb92cabdc..9dba77ced 100644 --- a/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java +++ b/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java @@ -43,7 +43,8 @@ public Integer getPriority(TaskId taskId) { return (taskId.getNamespaceClass() != null && ( (taskId.getNamespaceClass().getName().startsWith("org.labkey.sequenceanalysis.pipeline") || - taskId.getNamespaceClass().getName().startsWith("org.labkey.jbrowse.pipeline")) + taskId.getNamespaceClass().getName().startsWith("org.labkey.jbrowse.pipeline") || + taskId.getNamespaceClass().getName().endsWith("GeneticCalculationsRTask")) )) ? 50 : null; } } @@ -53,6 +54,11 @@ private boolean isSequenceNormalizationTask(PipelineJob job) return (job.getActiveTaskId() != null && job.getActiveTaskId().getNamespaceClass().getName().endsWith("SequenceNormalizationTask")); } + private boolean isGeneticsTask(PipelineJob job) + { + return (job.getActiveTaskId() != null && job.getActiveTaskId().getNamespaceClass().getName().endsWith("GeneticCalculationsRTask")); + } + private boolean isLuceneIndexJob(PipelineJob job) { return (job.getActiveTaskId() != null && job.getActiveTaskId().getNamespaceClass().getName().endsWith("JBrowseLuceneTask")); @@ -95,6 +101,7 @@ public Integer getMaxRequestCpus(PipelineJob job) job.getLogger().debug("setting max CPUs to 8"); return 8; } + if (isLuceneIndexJob(job)) { job.getLogger().debug("setting max CPUs to 24"); @@ -152,6 +159,12 @@ public Integer getMaxRequestMemory(PipelineJob job) return 48; } + if (isGeneticsTask(job)) + { + job.getLogger().debug("setting memory to 72"); + return 72; + } + if (isCacheAlignerIndexesTask(job)) { job.getLogger().debug("setting memory to 48"); diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java index a9c200f6f..acf5d44e8 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java @@ -53,6 +53,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeMap; public class CellRangerVDJUtils { @@ -296,6 +297,8 @@ else if ("Low Counts".equals(hto)) int fullLengthNoClonotype = 0; int multiChainConverted = 0; int discordantLoci = 0; + Map<String, Integer> chainMap = new TreeMap<>(); + Set<String> knownBarcodes = new HashSet<>(); Map<HEADER_FIELD, Integer> headerToIdx = null; @@ -398,6 +401,8 @@ else if (discordantBarcodes.contains(barcode)) discordantLoci++; } + chainMap.put(locus, chainMap.getOrDefault(locus, 0) + 1); + // Aggregate by: cDNA_ID, cdr3, chain, raw_clonotype_id, coalescedContigName, vHit, dHit, jHit, cHit, cdr3_nt String key = StringUtils.join(new String[]{cDNA.toString(), line[headerToIdx.get(HEADER_FIELD.CDR3)], locus, rawClonotypeId, coalescedContigName, removeNone(line[headerToIdx.get(HEADER_FIELD.V_GENE)]), removeNone(line[headerToIdx.get(HEADER_FIELD.D_GENE)]), removeNone(line[headerToIdx.get(HEADER_FIELD.J_GENE)]), cGene, removeNone(line[headerToIdx.get(HEADER_FIELD.CDR3_NT)])}, "<>"); AssayModel am; @@ -447,6 +452,10 @@ else if (discordantBarcodes.contains(barcode)) _log.info("total rows lacking clonotype, but marked full-length: " + fullLengthNoClonotype); _log.info("total rows converted from Multi to TRA: " + multiChainConverted); _log.info("total rows with discordant chain/c-gene calls: " + discordantLoci); + for (String chain : chainMap.keySet()) + { + _log.info("total rows for " + chain + ":" + chainMap.get(chain)); + } } catch (IOException e)