Skip to content

Commit

Permalink
Merge discvr-24.11 to develop
Browse files Browse the repository at this point in the history
  • Loading branch information
@bbimber
bbimber committed Feb 4, 2025
2 parents f66999d + 3d7d48c commit eb7411e
Show file tree
Hide file tree
Showing 7 changed files with 60 additions and 51 deletions.
22 changes: 16 additions & 6 deletions mGAP/resources/views/mgapDataDashboard.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,7 @@
header: 'Upcoming Release',
items: [{
name: 'Release Instructions',
url: LABKEY.ActionURL.buildURL('query', 'executeQuery.view', null, {
schemaName: 'wiki',
queryName: 'page',
url: LABKEY.ActionURL.buildURL('wiki', 'page.view', null, {
name: 'mgapRelease'
})
},{
Expand Down Expand Up @@ -108,14 +106,26 @@
},{
header: 'Data and Processing',
items: [{
name: 'Readsets Without Alignments',
name: 'WGS/WXS Readsets Without BAM/CRAM',
url: LABKEY.ActionURL.buildURL('query', 'executeQuery.view', null, {
schemaName: 'sequenceanalysis',
queryName: 'sequence_readsets',
'query.outputFileTypes~doesnotcontain': 'Alignment',
'query.totalForwardReads~isnonblank': null,
'query.application~in' :'Whole Exome;Whole Genome: Deep Coverage',
'query.status~isblank': null,
'query.sort': 'name',
'query.maxRows': 250
})
},{
name: 'WGS/WXS Readsets Without A gVCF File',
url: LABKEY.ActionURL.buildURL('query', 'executeQuery.view', null, {
schemaName: 'sequenceanalysis',
queryName: 'sequence_readsets',
'query.totalAlignments~eq': 0,
'query.outputFileTypes~doesnotcontain': 'gVCF File',
'query.totalForwardReads~isnonblank': null,
'query.application~in' :'Whole Exome;Whole Genome: Deep Coverage',
'query.status~isblank': null,
'query.isArchived~eq': 0,
'query.sort': 'name',
'query.maxRows': 250
})
Expand Down
14 changes: 12 additions & 2 deletions mGAP/resources/web/mGAP/window/ReleaseWindow.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ Ext4.define('mGAP.window.ReleaseWindow', {
Ext4.Msg.hide();
var outputFiles = [];
var distinctGenomesBySpecies = {};
var distinctGenomes = [];
Ext4.Array.forEach(results.rows, function(r){
if (!r.vcfId) {
Ext4.Msg.alert('Error', 'Track lacks VCF ID: ' + r.trackName);
Expand All @@ -34,20 +35,29 @@ Ext4.define('mGAP.window.ReleaseWindow', {
distinctGenomesBySpecies[r.species] = distinctGenomesBySpecies[r.species] || [];
if (r['vcfId/library_id']) {
distinctGenomesBySpecies[r.species].push(r['vcfId/library_id']);
distinctGenomes.push(r['vcfId/library_id']);
}
}, this);

distinctGenomes = Ext4.unique(distinctGenomes)

if (!outputFiles.length){
Ext4.Msg.alert('Error', 'None of the tracks have VCF files');
return;
}

for (sn in Ext4.Object.getKeys(distinctGenomesBySpecies)) {
var hadError = false;
Ext4.Array.forEach(Ext4.Object.getKeys(distinctGenomesBySpecies), function(sn) {
var genomes = Ext4.Array.unique(distinctGenomesBySpecies[sn]);
if (genomes.length !== 1){
Ext4.Msg.alert('Error', 'All files must use the same genome. Genomes found for species ' + sn + ': ' + genomes.length);
return;
hadError = true;
return false;
}
}, this);

if (hadError) {
return;
}

LABKEY.Ajax.request({
Expand Down
5 changes: 5 additions & 0 deletions mGAP/src/org/labkey/mgap/pipeline/GenerateMgapTracksStep.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -456,6 +456,11 @@ private Map<String, File> processTracks(VariantProcessingStepOutputImpl output,
List<String> options = new ArrayList<>();
options.add("--remove-unused-alternates");

options.add("--recalculate-ac");
options.add("--keep-original-ac");
options.add("--original-ac-suffix");
options.add(".mGAP");

options.add("--sample-mapping-file");
options.add(sampleFile.getPath());

Expand Down
4 changes: 2 additions & 2 deletions mGAP/src/org/labkey/mgap/pipeline/mGapReleaseGenerator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ public class mGapReleaseGenerator extends AbstractParameterizedOutputHandler<Seq
public mGapReleaseGenerator()
{
super(ModuleLoader.getInstance().getModule(mGAPModule.class), "Create mGAP Release", "This will prepare an input VCF for use as an mGAP public release. This will optionally include: removing excess annotations and program records, limiting to SNVs (optional) and removing genotype data (optional). If genotypes are retained, the subject names will be checked for mGAP aliases and replaced as needed.", new LinkedHashSet<>(PageFlowUtil.set("sequenceanalysis/field/GenomeFileSelectorField.js")), Arrays.asList(
ToolParameterDescriptor.create("species", "Version", "The species, which is used to filter tracks", "ldk-simplelabkeycombo", new JSONObject(){{
ToolParameterDescriptor.create("species", "Species", "The species, which is used to filter tracks", "ldk-simplelabkeycombo", new JSONObject(){{
put("allowBlank", false);
put("doNotIncludeInTemplates", true);
put("width", 400);
Expand Down Expand Up @@ -1048,7 +1048,7 @@ private void checkVcfAnnotationsAndSamples(File vcfInput, boolean skipAnnotation

if (!nonCompliant.isEmpty())
{
throw new PipelineJobException("Names do not conform to format: " + StringUtils.join(nonCompliant, ","));
throw new PipelineJobException("Names do not conform to format: " + StringUtils.join(nonCompliant, ",") + ", in file: " + vcfInput.getName());
}
}

Expand Down
25 changes: 25 additions & 0 deletions mcc/src/org/labkey/mcc/etl/PopulateIdsStep.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,9 +43,34 @@ public RecordedActionSet run(@NotNull PipelineJob job) throws PipelineJobExcepti
populateForDemographics(job);
populateForKinship(job);

// Check for duplicate IDs and proactively error:
performDuplicateCheck(job);

return new RecordedActionSet();
}

private void performDuplicateCheck(PipelineJob job) throws PipelineJobException
{
// Query aggregated demographics:
UserSchema sourceSchema = QueryService.get().getUserSchema(_containerUser.getUser(), _containerUser.getContainer(), MccSchema.NAME);
if (sourceSchema == null)
{
throw new PipelineJobException("Unable to find source schema: " + MccSchema.NAME);
}

TableInfo sourceTi = sourceSchema.getTable("duplicatedAggregatedDemographics");
if (sourceTi == null)
{
throw new PipelineJobException("Unable to find table: duplicatedAggregatedDemographics");
}

TableSelector ts = new TableSelector(sourceTi);
if (ts.exists())
{
throw new PipelineJobException("There were duplicated IDs in aggregatedDemographics");
}
}

private void populateForDemographics(PipelineJob job) throws PipelineJobException
{
// Query aggregated demographics:
Expand Down
2 changes: 0 additions & 2 deletions primeseq/src/org/labkey/primeseq/pipeline/ExacloudResourceSettings.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,6 @@ public List<ToolParameterDescriptor> getParams()
put("minValue", 512);
}}, 1028),
ToolParameterDescriptor.create("localSSD", "Request Nodes With SSD Scratch", "If selected, -C ssdscratch will be added to the submit script, which limits to node with faster SSD scratch space. This might be important for I/O intense jobs.", "checkbox", null, null),
ToolParameterDescriptor.create("useLustre", "Use Old Lustre Working Space", "If selected, this job will use the older Lustre working space.", "checkbox", null, null),
ToolParameterDescriptor.create("requireInfiniband", "Require InfiniBand", "If selected, this job will be flagged as -C IB to force the node to have InfiniBand.", "checkbox", null, null),
ToolParameterDescriptor.create("gpus", "GPUs", "The number of GPUs requested for this job. If non-zero, the gpu partition will be used.", "ldk-integerfield", null, null)
);
}
Expand Down
39 changes: 0 additions & 39 deletions primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -310,7 +310,6 @@ public void addExtraSubmitScriptLines(PipelineJob job, RemoteExecutionEngine eng
possiblyAddSSD(job, engine, lines);
possiblyAddGpus(job, engine, lines);
possiblyAddExclusive(job, engine, lines);
possiblyAddInfiniband(job, engine, lines);
}
}

Expand All @@ -319,11 +318,6 @@ public void addExtraSubmitScriptLines(PipelineJob job, RemoteExecutionEngine eng
{
Map<String, Object> ret = new HashMap<>();

if (job instanceof HasJobParams && getUseLustreValue((HasJobParams)job))
{
job.getLogger().info("Requiring using original lustre as working space");
ret.put("USE_LUSTRE", "1");
}

return ret;
}
Expand Down Expand Up @@ -482,39 +476,6 @@ private void possiblyAddSSD(PipelineJob job, RemoteExecutionEngine engine, List<
}
}

private void possiblyAddInfiniband(PipelineJob job, RemoteExecutionEngine engine, List<String> lines)
{
Map<String, String> params = ((HasJobParams)job).getJobParams();
String val = StringUtils.trimToNull(params.get("resourceSettings.resourceSettings.requireInfiniband"));
if (val == null)
{
return;
}

boolean parsed = Boolean.parseBoolean(val);
if (parsed)
{
job.getLogger().info("Requiring node with infiniband");
String line = "#SBATCH -C IB";
if (!lines.contains(line))
{
lines.add(line);
}
}
}

private boolean getUseLustreValue(HasJobParams job)
{
Map<String, String> params = (job).getJobParams();
String val = StringUtils.trimToNull(params.get("resourceSettings.resourceSettings.useLustre"));
if (val == null)
{
return false;
}

return Boolean.parseBoolean(val);
}

private void possiblyAddQOS(PipelineJob job, RemoteExecutionEngine engine, List<String> lines)
{
//first remove existing
Expand Down

0 comments on commit eb7411e

Please sign in to comment.