MANTA-285 add option to keep all temporary files

ChangeLog.txt

@@ -1,3 +1,4 @@
+- MANTA-285 add option to keep all temp files to support workflow debug
 - MANTA-276 mv configure to top-level, mv guides to docs dir, add methods docs
 - MANTA-284 improve windows shell support by shortening very long cmdlines
 v0.29.3

docs/developerGuide/README.md

@@ -69,6 +69,15 @@ extending from the configuration example in the above build instructions, use:
 
     ../manta-A.B.C.release_src/src/configure --jobs=4 --prefix=/path/to/install --build-type=ASan
 
+### General Debugging: Inspecting temporary files 
+
+Manta's configuration step includes an extended option to keep all temporary
+files which would normally be deleted by the workflow as it runs. Keeping these
+files may be helpful in various debugging scenarios. To turn on this option, add
+`--retainTempFiles` as a configuration argument:
+
+    configManta.py [other_options...] --retainTempFiles
+
 ### Windows development support
 
 Manta does not link or run on windows. However, the build system does

src/python/bin/configManta.py

@@ -75,6 +75,9 @@ def addExtendedGroupOptions(self,group) :
         group.add_option("--candidateBins",type="int",dest="nonlocalWorkBins",metavar="candidateBins",
                          help="Provide the total number of tasks which candidate generation "
                             " will be sub-divided into. (default: %default)")
+        group.add_option("--retainTempFiles",
+                         dest="isRetainTempFiles", action="store_true",
+                         help="Keep all temporary files (for workflow debugging)")
 
         MantaWorkflowOptionsBase.addExtendedGroupOptions(self,group)
 
@@ -90,6 +93,7 @@ def getOptionDefaults(self) :
             'isUnstrandedRNA' : False,
             'useExistingAlignStats' : False,
             'useExistingChromDepths' : False,
+            'isRetainTempFiles' : False,
             'nonlocalWorkBins' : 256
                           })
         return defaults

src/python/lib/mantaWorkflow.py

@@ -81,8 +81,9 @@ def runStats(self,taskPrefix="",dependencies=None) :
     nextStepWait = set()
     nextStepWait.add(mergeTask)
 
-    rmStatsTmpCmd = getRmdirCmd() + [tmpStatsDir]
-    rmTask=self.addTask(preJoin(taskPrefix,"rmTmpDir"),rmStatsTmpCmd,dependencies=mergeTask, isForceLocal=True)
+    if not self.params.isRetainTempFiles :
+        rmStatsTmpCmd = getRmdirCmd() + [tmpStatsDir]
+        rmTask=self.addTask(preJoin(taskPrefix,"rmTmpDir"),rmStatsTmpCmd,dependencies=mergeTask, isForceLocal=True)
 
     # summarize stats in format that's easier for human review
     cmd = [self.params.mantaStatsSummaryBin]
@@ -194,8 +195,9 @@ def getGenomeSegmentGroups(params) :
     checkCmd.extend(["--graph-file", graphPath])
     checkTask = self.addTask(preJoin(taskPrefix,"checkLocusGraph"),checkCmd,dependencies=mergeTask,memMb=self.params.mergeMemMb)
 
-    rmGraphTmpCmd = getRmdirCmd() + [tmpGraphDir]
-    rmTask=self.addTask(preJoin(taskPrefix,"rmTmpDir"),rmGraphTmpCmd,dependencies=mergeTask)
+    if not self.params.isRetainTempFiles :
+        rmGraphTmpCmd = getRmdirCmd() + [tmpGraphDir]
+        rmTask=self.addTask(preJoin(taskPrefix,"rmTmpDir"),rmGraphTmpCmd,dependencies=mergeTask)
 
     graphStatsCmd  = [self.params.mantaGraphStatsBin,"--global"]
     graphStatsCmd.extend(["--graph-file",graphPath])
@@ -432,6 +434,12 @@ def getEdgeLogSortCmd(logListFile, outPath) :
     edgeStatsMergeCmd.extend(["--report-file",self.paths.getFinalEdgeStatsReportPath()])
     self.addTask(edgeStatsMergeTask, edgeStatsMergeCmd, dependencies=statsListTask, isForceLocal=True)
 
+    if not self.params.isRetainTempFiles :
+        # we could delete the temp hygenDir directory here, but it is used for debug so frequently it doesn't seem worth it at present.
+        # rmDirCmd = getRmdirCmd() + [hygenDir]
+        # rmDirTask=self.addTask(preJoin(taskPrefix,"rmTmpDir"),rmDirCmd,dependencies=TBD_XXX_MANY)
+        pass
+
     return nextStepWait
 
 

src/python/lib/sharedWorkflow.py

@@ -122,8 +122,9 @@ def _runDepthShared(self,taskPrefix, dependencies, bamList, outputPath, depthFun
     nextStepWait = set()
     nextStepWait.add(mergeTask)
 
-    rmTmpCmd = getRmdirCmd() + [tmpDir]
-    rmTask=self.addTask(preJoin(taskPrefix,"rmTmpDir"),rmTmpCmd,dependencies=mergeTask, isForceLocal=True)
+    if not self.params.isRetainTempFiles :
+        rmTmpCmd = getRmdirCmd() + [tmpDir]
+        rmTask=self.addTask(preJoin(taskPrefix,"rmTmpDir"),rmTmpCmd,dependencies=mergeTask, isForceLocal=True)
 
     return nextStepWait