Stability Update

MetaMorpheus/EngineLayer/Calibration/DataPointAcquisitionEngine.cs

@@ -40,6 +40,9 @@ public DataPointAcquisitionEngine(
             MinMS1isotopicPeaksNeededForConfirmedIdentification = minMS1isotopicPeaksNeededForConfirmedIdentification;
         }
 
+        private readonly object _ms1Lock = new();
+        private readonly object _ms2Lock = new();
+
         protected override MetaMorpheusEngineResults RunSpecific()
         {
             Status("Extracting data points:");
@@ -52,9 +55,6 @@ protected override MetaMorpheusEngineResults RunSpecific()
             List<LabeledDataPoint> Ms1List = new List<LabeledDataPoint>();
             List<LabeledDataPoint> Ms2List = new List<LabeledDataPoint>();
 
-            object lockObj = new object();
-            object lockObj2 = new object();
-
             int maxThreadsPerFile = CommonParameters.MaxThreadsToUsePerFile;
             int[] threads = Enumerable.Range(0, maxThreadsPerFile).ToArray();
             Parallel.ForEach(threads, (matchIndex) =>
@@ -85,7 +85,7 @@ protected override MetaMorpheusEngineResults RunSpecific()
 
                     List<LabeledDataPoint> ms2tuple = SearchMS2Spectrum(GoodScans[matchIndex], identification, ProductMassTolerance);
 
-                    lock (lockObj2)
+                    lock (_ms2Lock)
                     {
                         Ms2List.AddRange(ms2tuple);
                     }
@@ -102,7 +102,7 @@ protected override MetaMorpheusEngineResults RunSpecific()
 
                     var ms1tupleForward = SearchMS1Spectra(theoreticalMasses, theoreticalIntensities, ms2scanNumber, 1, peptideCharge, identification);
 
-                    lock (lockObj)
+                    lock (_ms1Lock)
                     {
                         Ms1List.AddRange(ms1tupleBack.Item1);
                         numMs1MassChargeCombinationsConsidered += ms1tupleBack.Item2;

MetaMorpheus/EngineLayer/FdrAnalysis/SpectralMatchGroup.cs

@@ -44,7 +44,9 @@ public static List<SpectralMatchGroup> GroupByBaseSequence(List<SpectralMatch> s
         /// </summary>
         public IEnumerable<SpectralMatch> GetBestMatches()
         { 
-            return SpectralMatches.GroupBy(p => p.FullSequence).Select(g => g.MaxBy(p => p));
+            return SpectralMatches
+                .GroupBy(p => p.FullSequence)
+                .Select(g => g.MaxBy(p => p));
         }
 
         /// <summary>

MetaMorpheus/TaskLayer/MetaMorpheusTask.cs

@@ -580,7 +580,7 @@
             {
                 MetaMorpheusEngine.FinishedSingleEngineHandler -= SingleEngineHandlerInTask;
                 var resultsFileName = Path.Combine(output_folder, "results.txt");
-        e.Data.Add("folder", output_folder);
+                e.Data.Add("folder", output_folder);
                 using (StreamWriter file = new StreamWriter(resultsFileName))
                 {
                     file.WriteLine(GlobalVariables.MetaMorpheusVersion.Equals("1.0.0.0") ? "MetaMorpheus: Not a release version" : "MetaMorpheus: version " + GlobalVariables.MetaMorpheusVersion);

MetaMorpheus/TaskLayer/MyFileManager.cs

@@ -1,18 +1,19 @@
 using EngineLayer;
 using MassSpectrometry;
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.IO;
 using System.IO.Compression;
 using Readers;
+using System.Reflection.Metadata.Ecma335;
 
 namespace TaskLayer
 {
     public class MyFileManager
     {
         private readonly bool DisposeOfFileWhenDone;
-        private readonly Dictionary<string, MsDataFile> MyMsDataFiles = new Dictionary<string, MsDataFile>();
-        private readonly object FileLoadingLock = new object();
+        private readonly ConcurrentDictionary<string, MsDataFile> MyMsDataFiles = new ConcurrentDictionary<string, MsDataFile>();
 
         public MyFileManager(bool disposeOfFileWhenDone)
         {
@@ -23,7 +24,7 @@ public MyFileManager(bool disposeOfFileWhenDone)
 
         public bool SeeIfOpen(string path)
         {
-            return (MyMsDataFiles.ContainsKey(path) && MyMsDataFiles[path] != null);
+            return MyMsDataFiles.TryGetValue(path, out var file);
         }
 
         public MsDataFile LoadFile(string origDataFile, CommonParameters commonParameters)
@@ -42,25 +43,19 @@ public MsDataFile LoadFile(string origDataFile, CommonParameters commonParameter
                 filter = null;
             }
 
-            if (MyMsDataFiles.TryGetValue(origDataFile, out MsDataFile value) && value != null)
-            {
+            if (MyMsDataFiles.TryGetValue(origDataFile, out MsDataFile value))
                 return value;
-            }
 
-            // By now know that need to load this file!!!
-            lock (FileLoadingLock) // Lock because reading is sequential
-            {
-                MyMsDataFiles[origDataFile] = MsDataFileReader.GetDataFile(origDataFile)
-                    .LoadAllStaticData(filter, commonParameters.MaxThreadsToUsePerFile);
-                return MyMsDataFiles[origDataFile];
-            }
+            return MyMsDataFiles.GetOrAdd(origDataFile,
+                MsDataFileReader.GetDataFile(origDataFile).LoadAllStaticData(filter, commonParameters.MaxThreadsToUsePerFile));
         }
 
         internal void DoneWithFile(string origDataFile)
         {
             if (DisposeOfFileWhenDone)
             {
-                MyMsDataFiles[origDataFile] = null;
+                // This would only return false if the file was not in the dictionary
+                MyMsDataFiles.TryRemove(origDataFile, out var file);
             }
         }
 

MetaMorpheus/TaskLayer/SearchTask/SearchTask.cs

@@ -84,15 +84,16 @@ public static MassDiffAcceptor GetMassDiffAcceptor(Tolerance precursorMassTolera
             }
         }
 
-        protected override MyTaskResults RunSpecific(string OutputFolder, List<DbForTask> dbFilenameList, List<string> currentRawFileList, string taskId, FileSpecificParameters[] fileSettingsList)
+        protected override MyTaskResults RunSpecific(string OutputFolder, List<DbForTask> dbFilenameList, List<string> currentRawFileList, string taskId, 
+            FileSpecificParameters[] fileSettingsList)
         {
             MyFileManager myFileManager = new MyFileManager(SearchParameters.DisposeOfFileWhenDone);
             var fileSpecificCommonParams = fileSettingsList.Select(b => SetAllFileSpecificCommonParams(CommonParameters, b));
 
             // start loading first spectra file in the background
-            Task<MsDataFile> nextFileLoadingTask = new(() => myFileManager.LoadFile(currentRawFileList[0], SetAllFileSpecificCommonParams(CommonParameters, fileSettingsList[0])));
+            string fileToLoad = currentRawFileList[0];
+            Task<MsDataFile> nextFileLoadingTask = new(() => myFileManager.LoadFile(fileToLoad, SetAllFileSpecificCommonParams(CommonParameters, fileSettingsList[0])));
             nextFileLoadingTask.Start();
-
 
             if (SearchParameters.DoLabelFreeQuantification)
             {

MetaMorpheus/Test/EverythingRunnerEngineTestCase.cs

@@ -140,17 +140,22 @@ static EverythingRunnerEngineTestCase()
             string myDatabase = Path.Combine(TestContext.CurrentContext.TestDirectory,
                 @"TestData\TaGe_SA_A549_3_snip.fasta");
             searchTaskLoaded.CommonParameters.QValueCutoffForPepCalculation = 0.01;
+
+            // Bottom-up Q-value Test Case
             _cases.Add(EverythingRunnerEngineTestCases.BottomUpQValue,
                 new EverythingRunnerEngineTestCase(EverythingRunnerEngineTestCases.BottomUpQValue,
                     new List<(string, MetaMorpheusTask)> { ("postSearchAnalysisTaskTestOutput", searchTaskLoaded) },
                     new List<string> { myFile1, myFile2 }, new List<DbForTask> { new DbForTask(myDatabase, false) },
                     false));
+
+            // Bottom-up Q-value Single File Test Case
             _cases.Add(EverythingRunnerEngineTestCases.BottomUpQValueSingle,
                 new EverythingRunnerEngineTestCase(EverythingRunnerEngineTestCases.BottomUpQValueSingle,
                     new List<(string, MetaMorpheusTask)> { ("postSearchAnalysisTaskTestOutput", searchTaskLoaded) },
                     new List<string> { myFile2 },
                     new List<DbForTask> { new DbForTask(myDatabase, false) }, false));
 
+            // Bottom-up Q-value No Individual Files Write MzId Test Case
             searchTaskLoaded = Toml.ReadFile<SearchTask>(myTomlPath, MetaMorpheusTask.tomlConfig);
             searchTaskLoaded.SearchParameters.WriteIndividualFiles = false;
             searchTaskLoaded.SearchParameters.WriteMzId = true;
@@ -161,6 +166,7 @@ static EverythingRunnerEngineTestCase()
                     new List<string> { myFile1, myFile2 }, new List<DbForTask> { new DbForTask(myDatabase, false) },
                     false));
 
+            // Bottom-up Q-value No Individual Files Write PepXml Test Case
             searchTaskLoaded = Toml.ReadFile<SearchTask>(myTomlPath, MetaMorpheusTask.tomlConfig);
             searchTaskLoaded.SearchParameters.WriteIndividualFiles = false;
             searchTaskLoaded.SearchParameters.WriteMzId = false;
@@ -171,6 +177,7 @@ static EverythingRunnerEngineTestCase()
                     new List<string> { myFile1, myFile2 }, new List<DbForTask> { new DbForTask(myDatabase, false) },
                     false));
 
+            // Bottom-up Pep Q-value Test Case
             myTomlPath = Path.Combine(TestContext.CurrentContext.TestDirectory,
                 @"TestData\Task2-SearchTaskconfig.toml");
             searchTaskLoaded = Toml.ReadFile<SearchTask>(myTomlPath, MetaMorpheusTask.tomlConfig);
@@ -181,6 +188,7 @@ static EverythingRunnerEngineTestCase()
                     new List<string> { myFile1, myFile2 }, new List<DbForTask> { new DbForTask(myDatabase, false) },
                     false));
 
+            // Top-down Q-value Test Case
             myTomlPath = Path.Combine(TestContext.CurrentContext.TestDirectory,
                 @"TopDownTestData\TopDownSearchToml.toml");
             searchTaskLoaded = Toml.ReadFile<SearchTask>(myTomlPath, MetaMorpheusTask.tomlConfig);
@@ -192,6 +200,8 @@ static EverythingRunnerEngineTestCase()
                     new List<(string, MetaMorpheusTask)> { ("postSearchAnalysisTaskTestOutput", searchTaskLoaded) },
                     new List<string> { myFile1, myFile2 }, new List<DbForTask> { new DbForTask(myDatabase, false) },
                     true));
+
+            // Top-down Q-value Single File Test Case
             _cases.Add(EverythingRunnerEngineTestCases.TopDownQValueSingle,
                 new EverythingRunnerEngineTestCase(EverythingRunnerEngineTestCases.TopDownQValueSingle,
                     new List<(string, MetaMorpheusTask)> { ("postSearchAnalysisTaskTestOutput", searchTaskLoaded) },

MetaMorpheus/Test/XLSearchOutputTest.cs

@@ -52,6 +52,7 @@ public static void WriteXlSpectralLibraryTest()
             string myFile = Path.Combine(TestContext.CurrentContext.TestDirectory, @"XlTestData\2017-11-21_XL_DSSO_Ribosome_RT60min_28800-28898.mzML");
             string myDatabase = Path.Combine(TestContext.CurrentContext.TestDirectory, @"XlTestData\RibosomeGO.fasta");
 
+            if (Directory.Exists(outputFolder)) Directory.Delete(outputFolder, true);
             Directory.CreateDirectory(outputFolder);
 
             XLSearchTask xLSearch = new XLSearchTask

MetaMorpheus/Test/XLTest.cs

@@ -363,6 +363,15 @@ public static void TestCsmSort()
             Assert.That(csmListList.Select(c => c.First().XLTotalScore).ToList(), Is.EquivalentTo(new List<double> { 43d, 33d }));
             Assert.That(csmListList.Select(c => c.First().BaseSequence).ToList(), Is.EquivalentTo(new List<string> { "LEDITPEP", "VPEPTIDE" }));
             Assert.That(csmListList.Select(c => c.First().BetaPeptide.BaseSequence).ToList(), Is.EquivalentTo(new List<string> { "VEDI", "APEP" }));
+
+
+            Protein protForwardAsDecoy = new Protein(sequence: "VPEPTIDELPEPTIDEAPEPTIDE", accession: "DECOY", isDecoy: true);
+            PeptideWithSetModifications pwsmV_D = new PeptideWithSetModifications(protForwardAsDecoy, new DigestionParams(), 1, 8, CleavageSpecificity.Full, "VPEPTIDE", 0, mod, 0, null);
+
+            csmOne.AddOrReplace(pwsmV_D, 3, 0, true, new List<MatchedFragmentIon>(), 0);
+            csmOne.ResolveAllAmbiguities();
+            Assert.That(csmOne.BestMatchingBioPolymersWithSetMods.Count() == 1);
+            Assert.That(!csmOne.BestMatchingBioPolymersWithSetMods.First().Peptide.Parent.IsDecoy);
         }
 
         [Test]