From e25660a1a8b02326a43699b751b4d0ca5ef42894 Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 17 Jun 2024 19:59:10 -0500 Subject: [PATCH 1/7] Only quantify peptides that clear the q-value threshold --- .../CrosslinkSearch/CrosslinkSpectralMatch.cs | 8 +- .../FdrAnalysis/FdrAnalysisEngine.cs | 51 +++++---- .../FdrAnalysis/PEPValueAnalysisGeneric.cs | 34 ++++-- .../GlycoSearch/GlycoSpectralMatch.cs | 2 +- MetaMorpheus/EngineLayer/Gptmd/GptmdEngine.cs | 2 +- .../ModificationAnalysisEngine.cs | 2 +- .../NonSpecificEnzymeSearchEngine.cs | 16 +-- .../ProteinParsimonyEngine.cs | 4 +- .../ProteinScoringAndFdrEngine.cs | 4 +- .../EngineLayer/PsmTsv/PsmTsvWriter.cs | 20 ++-- MetaMorpheus/EngineLayer/SpectralMatch.cs | 57 ++++++--- .../CalibrationTask/CalibrationTask.cs | 2 +- .../GlycoSearchTask/GlycoProteinParsimony.cs | 4 +- .../PostGlycoSearchAnalysisTask.cs | 8 +- .../MbrAnalysis/SpectralRecoveryPSM.cs | 6 +- .../MbrAnalysis/SpectralRecoveryRunner.cs | 22 ++-- MetaMorpheus/TaskLayer/MetaMorpheusTask.cs | 4 +- MetaMorpheus/TaskLayer/PepXMLWriter.cs | 2 +- .../TaskLayer/SearchTask/MzIdentMLWriter.cs | 4 +- .../SearchTask/PostSearchAnalysisTask.cs | 108 +++++++++++++----- .../XLSearchTask/PostXLSearchAnalysisTask.cs | 38 +++--- .../TaskLayer/XLSearchTask/WriteXlFile.cs | 8 +- MetaMorpheus/Test/FdrTest.cs | 30 ++--- .../Test/MultiProteaseParsimonyTest.cs | 20 ++-- .../Test/PostSearchAnalysisTaskTests.cs | 9 ++ MetaMorpheus/Test/SearchEngineTests.cs | 8 +- MetaMorpheus/Test/SpectralRecoveryTest.cs | 2 +- .../Test/TestData/Task1-SearchTaskconfig.toml | 2 +- MetaMorpheus/Test/TestPsm.cs | 2 +- MetaMorpheus/Test/XLTest.cs | 4 +- 30 files changed, 299 insertions(+), 184 deletions(-) diff --git a/MetaMorpheus/EngineLayer/CrosslinkSearch/CrosslinkSpectralMatch.cs b/MetaMorpheus/EngineLayer/CrosslinkSearch/CrosslinkSpectralMatch.cs index 4bdf2522c..e8d92e202 100644 --- a/MetaMorpheus/EngineLayer/CrosslinkSearch/CrosslinkSpectralMatch.cs +++ b/MetaMorpheus/EngineLayer/CrosslinkSearch/CrosslinkSpectralMatch.cs @@ -492,18 +492,18 @@ public override string ToString() sb.Append((IsDecoy || BetaPeptide.IsDecoy) ? "D" : (IsContaminant || BetaPeptide.IsContaminant) ? "C" : "T"); sb.Append("\t"); } - if (FdrInfo == null) + if (PsmFdrInfo == null) { sb.Append("0\t0\t0\t"); } else { - sb.Append(FdrInfo.QValue.ToString()); + sb.Append(PsmFdrInfo.QValue.ToString()); sb.Append("\t"); - sb.Append(FdrInfo.PEP.ToString()); sb.Append("\t"); + sb.Append(PsmFdrInfo.PEP.ToString()); sb.Append("\t"); - sb.Append(FdrInfo.PEP_QValue.ToString()); sb.Append("\t"); + sb.Append(PsmFdrInfo.PEP_QValue.ToString()); sb.Append("\t"); } return sb.ToString(); } diff --git a/MetaMorpheus/EngineLayer/FdrAnalysis/FdrAnalysisEngine.cs b/MetaMorpheus/EngineLayer/FdrAnalysis/FdrAnalysisEngine.cs index bd7048b00..b214f2e77 100644 --- a/MetaMorpheus/EngineLayer/FdrAnalysis/FdrAnalysisEngine.cs +++ b/MetaMorpheus/EngineLayer/FdrAnalysis/FdrAnalysisEngine.cs @@ -14,16 +14,21 @@ public class FdrAnalysisEngine : MetaMorpheusEngine private readonly string AnalysisType; private readonly string OutputFolder; // used for storing PEP training models private readonly bool DoPEP; + private readonly bool PeptideLevelFdr; public FdrAnalysisEngine(List psms, int massDiffAcceptorNumNotches, CommonParameters commonParameters, - List<(string fileName, CommonParameters fileSpecificParameters)> fileSpecificParameters, List nestedIds, string analysisType = "PSM", bool doPEP = true, string outputFolder = null) : base(commonParameters, fileSpecificParameters, nestedIds) + List<(string fileName, CommonParameters fileSpecificParameters)> fileSpecificParameters, + List nestedIds, string analysisType = "PSM", + bool doPEP = true, string outputFolder = null, + bool peptideLevelFdr = false) : base(commonParameters, fileSpecificParameters, nestedIds) { AllPsms = psms.OrderByDescending(p => p).ToList(); MassDiffAcceptorNumNotches = massDiffAcceptorNumNotches; ScoreCutoff = commonParameters.ScoreCutoff; AnalysisType = analysisType; - this.OutputFolder = outputFolder; - this.DoPEP = doPEP; + OutputFolder = outputFolder; + DoPEP = doPEP; + PeptideLevelFdr = peptideLevelFdr; if (fileSpecificParameters == null) throw new ArgumentNullException("file specific parameters cannot be null"); } @@ -34,7 +39,7 @@ protected override MetaMorpheusEngineResults RunSpecific() Status("Running FDR analysis..."); DoFalseDiscoveryRateAnalysis(myAnalysisResults); Status("Done."); - myAnalysisResults.PsmsWithin1PercentFdr = AllPsms.Count(b => b.FdrInfo.QValue <= 0.01 && !b.IsDecoy); + myAnalysisResults.PsmsWithin1PercentFdr = AllPsms.Count(b => b.GetFdrInfo(PeptideLevelFdr).QValue <= 0.01 && !b.IsDecoy); return myAnalysisResults; } @@ -64,13 +69,13 @@ private void DoFalseDiscoveryRateAnalysis(FdrAnalysisResults myAnalysisResults) } } - private static void QValueInverted(List psms) + private void QValueInverted(List psms) { psms.Reverse(); //this calculation is performed from bottom up. So, we begin the loop by computing qValue //and qValueNotch for the last/lowest scoring psm in the bunch - double qValue = (psms[0].FdrInfo.CumulativeDecoy + 1) / psms[0].FdrInfo.CumulativeTarget; - double qValueNotch = (psms[0].FdrInfo.CumulativeDecoyNotch + 1) / psms[0].FdrInfo.CumulativeTargetNotch; + double qValue = (psms[0].GetFdrInfo(PeptideLevelFdr).CumulativeDecoy + 1) / psms[0].GetFdrInfo(PeptideLevelFdr).CumulativeTarget; + double qValueNotch = (psms[0].GetFdrInfo(PeptideLevelFdr).CumulativeDecoyNotch + 1) / psms[0].GetFdrInfo(PeptideLevelFdr).CumulativeTargetNotch; //Assign FDR values to PSMs for (int i = 0; i < psms.Count; i++) @@ -78,13 +83,13 @@ private static void QValueInverted(List psms) // Stop if canceled if (GlobalVariables.StopLoops) { break; } - qValue = Math.Min(qValue, (psms[i].FdrInfo.CumulativeDecoy + 1) / psms[i].FdrInfo.CumulativeTarget); - qValueNotch = Math.Min(qValueNotch, (psms[i].FdrInfo.CumulativeDecoyNotch + 1) / psms[i].FdrInfo.CumulativeTargetNotch); + qValue = Math.Min(qValue, (psms[i].GetFdrInfo(PeptideLevelFdr).CumulativeDecoy + 1) / psms[i].GetFdrInfo(PeptideLevelFdr).CumulativeTarget); + qValueNotch = Math.Min(qValueNotch, (psms[i].GetFdrInfo(PeptideLevelFdr).CumulativeDecoyNotch + 1) / psms[i].GetFdrInfo(PeptideLevelFdr).CumulativeTargetNotch); - double pep = psms[i].FdrInfo == null ? double.NaN : psms[i].FdrInfo.PEP; - double pepQValue = psms[i].FdrInfo == null ? double.NaN : psms[i].FdrInfo.PEP_QValue; + double pep = psms[i].GetFdrInfo(PeptideLevelFdr) == null ? double.NaN : psms[i].GetFdrInfo(PeptideLevelFdr).PEP; + double pepQValue = psms[i].GetFdrInfo(PeptideLevelFdr) == null ? double.NaN : psms[i].GetFdrInfo(PeptideLevelFdr).PEP_QValue; - psms[i].SetQandPEPvalues(qValue, qValueNotch, pep, pepQValue); + psms[i].SetQandPEPvalues(qValue, qValueNotch, pep, pepQValue, PeptideLevelFdr); } psms.Reverse(); //we inverted the psms for this calculation. now we need to put them back into the original order @@ -136,10 +141,10 @@ private void QValueTraditional(List psms) double qValue = Math.Min(1, cumulativeDecoy / cumulativeTarget); double qValueNotch = Math.Min(1, cumulativeDecoyPerNotch[notch] / cumulativeTargetPerNotch[notch]); - double pep = psm.FdrInfo == null ? double.NaN : psm.FdrInfo.PEP; - double pepQValue = psm.FdrInfo == null ? double.NaN : psm.FdrInfo.PEP_QValue; + double pep = psm.GetFdrInfo(PeptideLevelFdr) == null ? double.NaN : psm.GetFdrInfo(PeptideLevelFdr).PEP; + double pepQValue = psm.GetFdrInfo(PeptideLevelFdr) == null ? double.NaN : psm.GetFdrInfo(PeptideLevelFdr).PEP_QValue; - psm.SetFdrValues(cumulativeTarget, cumulativeDecoy, qValue, cumulativeTargetPerNotch[notch], cumulativeDecoyPerNotch[notch], qValueNotch, pep, pepQValue); + psm.SetFdrValues(cumulativeTarget, cumulativeDecoy, qValue, cumulativeTargetPerNotch[notch], cumulativeDecoyPerNotch[notch], qValueNotch, pep, pepQValue, PeptideLevelFdr); } } @@ -156,7 +161,7 @@ public void Compute_PEPValue(FdrAnalysisResults myAnalysisResults) searchType = "top-down"; } - myAnalysisResults.BinarySearchTreeMetrics = PEP_Analysis_Cross_Validation.ComputePEPValuesForAllPSMsGeneric(AllPsms, searchType, this.FileSpecificParameters, this.OutputFolder); + myAnalysisResults.BinarySearchTreeMetrics = PEP_Analysis_Cross_Validation.ComputePEPValuesForAllPSMsGeneric(AllPsms, searchType, this.FileSpecificParameters, this.OutputFolder, PeptideLevelFdr); Compute_PEPValue_Based_QValue(AllPsms); } @@ -169,14 +174,14 @@ public void Compute_PEPValue(FdrAnalysisResults myAnalysisResults) if (AnalysisType == "crosslink" && AllPsms.Count > 100) { - myAnalysisResults.BinarySearchTreeMetrics = PEP_Analysis_Cross_Validation.ComputePEPValuesForAllPSMsGeneric(AllPsms, "crosslink", this.FileSpecificParameters, this.OutputFolder); + myAnalysisResults.BinarySearchTreeMetrics = PEP_Analysis_Cross_Validation.ComputePEPValuesForAllPSMsGeneric(AllPsms, "crosslink", this.FileSpecificParameters, this.OutputFolder, PeptideLevelFdr); Compute_PEPValue_Based_QValue(AllPsms); } } - public static void Compute_PEPValue_Based_QValue(List psms) + public void Compute_PEPValue_Based_QValue(List psms) { - double[] allPEPValues = psms.Select(p => p.FdrInfo.PEP).ToArray(); + double[] allPEPValues = psms.Select(p => p.PsmFdrInfo.PEP).ToArray(); // Currently, we only calculate PEP at the PSM level, so this is hardcoded to access the int[] psmsArrayIndicies = Enumerable.Range(0, psms.Count).ToArray(); Array.Sort(allPEPValues, psmsArrayIndicies);//sort the second thing by the first @@ -185,7 +190,9 @@ public static void Compute_PEPValue_Based_QValue(List psms) { runningSum += allPEPValues[i]; double qValue = runningSum / (i + 1); - psms[psmsArrayIndicies[i]].FdrInfo.PEP_QValue = Math.Round(qValue, 6); + psms[psmsArrayIndicies[i]].GetFdrInfo(PeptideLevelFdr).PEP_QValue = Math.Round(qValue, 6); + // Copy the PSM level PEP value to the peptide level + if (PeptideLevelFdr) psms[psmsArrayIndicies[i]].GetFdrInfo(PeptideLevelFdr).PEP = allPEPValues[i]; } } /// @@ -197,8 +204,8 @@ public void CountPsm(List proteasePsms) var allUnambiguousPsms = proteasePsms.Where(psm => psm.FullSequence != null).ToList(); var unambiguousPsmsLessThanOnePercentFdr = allUnambiguousPsms.Where(psm => - psm.FdrInfo.QValue <= 0.01 - && psm.FdrInfo.QValueNotch <= 0.01) + psm.GetFdrInfo(PeptideLevelFdr).QValue <= 0.01 + && psm.GetFdrInfo(PeptideLevelFdr).QValueNotch <= 0.01) .GroupBy(p => p.FullSequence); Dictionary sequenceToPsmCount = new Dictionary(); diff --git a/MetaMorpheus/EngineLayer/FdrAnalysis/PEPValueAnalysisGeneric.cs b/MetaMorpheus/EngineLayer/FdrAnalysis/PEPValueAnalysisGeneric.cs index 15282674e..3af421bbf 100644 --- a/MetaMorpheus/EngineLayer/FdrAnalysis/PEPValueAnalysisGeneric.cs +++ b/MetaMorpheus/EngineLayer/FdrAnalysis/PEPValueAnalysisGeneric.cs @@ -25,9 +25,16 @@ public static class PEP_Analysis_Cross_Validation private static Dictionary>> fileSpecificTimeDependantHydrophobicityAverageAndDeviation_unmodified = new Dictionary>>(); private static Dictionary>> fileSpecificTimeDependantHydrophobicityAverageAndDeviation_modified = new Dictionary>>(); private static Dictionary>> fileSpecificTimeDependantHydrophobicityAverageAndDeviation_CZE = new Dictionary>>(); - - public static string ComputePEPValuesForAllPSMsGeneric(List psms, string searchType, List<(string fileName, CommonParameters fileSpecificParameters)> fileSpecificParameters, string outputFolder) + private static bool PeptideLevelFdr; + + public static string ComputePEPValuesForAllPSMsGeneric( + List psms, + string searchType, + List<(string fileName, CommonParameters fileSpecificParameters)> fileSpecificParameters, + string outputFolder, + bool peptideLevelFdr = false) { + PeptideLevelFdr = peptideLevelFdr; string[] trainingVariables = PsmData.trainingInfos[searchType]; //ensure that the order is always stable. @@ -301,14 +308,17 @@ public static List[] Get_PSM_Group_Indices(List psms, int nu return groupsOfIndicies; } - public static void RemoveBestMatchingPeptidesWithLowPEP(SpectralMatch psm, List indiciesOfPeptidesToRemove, List notches, List pwsmList, List pepValuePredictions, ref int ambiguousPeptidesRemovedCount) + public static void RemoveBestMatchingPeptidesWithLowPEP(SpectralMatch psm, + List indiciesOfPeptidesToRemove, List notches, List pwsmList, + List pepValuePredictions, + ref int ambiguousPeptidesRemovedCount) { foreach (int i in indiciesOfPeptidesToRemove) { psm.RemoveThisAmbiguousPeptide(notches[i], pwsmList[i]); ambiguousPeptidesRemovedCount++; } - psm.FdrInfo.PEP = 1 - pepValuePredictions.Max(); + psm.GetFdrInfo(PeptideLevelFdr).PEP = 1 - pepValuePredictions.Max(); } /// @@ -337,10 +347,14 @@ public static void GetIndiciesOfPeptidesToRemove(List indiciesOfPeptidesToR public static int GetChargeStateMode(List psms) { - return psms.Where(p => p.IsDecoy != true && p.FdrInfo.QValue <= 0.01).Select(p => p.ScanPrecursorCharge).GroupBy(n => n).OrderByDescending(g => g.Count()).Select(g => g.Key).FirstOrDefault(); + return psms.Where(p => p.IsDecoy != true && p.GetFdrInfo(PeptideLevelFdr).QValue <= 0.01).Select(p => p.ScanPrecursorCharge).GroupBy(n => n).OrderByDescending(g => g.Count()).Select(g => g.Key).FirstOrDefault(); } - public static Dictionary>> ComputeHydrophobicityValues(List psms, List<(string fileName, CommonParameters fileSpecificParameters)> fileSpecificParameters, bool computeHydrophobicitiesforModifiedPeptides) + public static Dictionary>> ComputeHydrophobicityValues( + List psms, + List<(string fileName, CommonParameters fileSpecificParameters)> fileSpecificParameters, + bool computeHydrophobicitiesforModifiedPeptides, + bool peptideLevelFdr = false) { SSRCalc3 calc = new SSRCalc3("SSRCalc 3.0 (300A)", SSRCalc3.Column.A300); @@ -356,7 +370,7 @@ public static Dictionary>> Compute Dictionary> hydrophobicities = new Dictionary>(); Dictionary> averagesCommaStandardDeviations = new Dictionary>(); - foreach (SpectralMatch psm in psms.Where(f => (f.FullFilePath == null || Path.GetFileName(f.FullFilePath) == filename) && f.FdrInfo.QValue <= 0.01 && !f.IsDecoy)) + foreach (SpectralMatch psm in psms.Where(f => (f.FullFilePath == null || Path.GetFileName(f.FullFilePath) == filename) && f.GetFdrInfo(PeptideLevelFdr).QValue <= 0.01 && !f.IsDecoy)) { List fullSequences = new List(); foreach ((int notch, IBioPolymerWithSetMods pwsm) in psm.BestMatchingBioPolymersWithSetMods) @@ -458,7 +472,7 @@ public static Dictionary>> Compute Dictionary> mobilities = new Dictionary>(); Dictionary> averagesCommaStandardDeviations = new Dictionary>(); - foreach (SpectralMatch psm in psms.Where(f => (f.FullFilePath == null || Path.GetFileName(f.FullFilePath) == filename) && f.FdrInfo.QValue <= 0.01 && !f.IsDecoy)) + foreach (SpectralMatch psm in psms.Where(f => (f.FullFilePath == null || Path.GetFileName(f.FullFilePath) == filename) && f.GetFdrInfo(PeptideLevelFdr).QValue <= 0.01 && !f.IsDecoy)) { List fullSequences = new List(); foreach ((int notch, IBioPolymerWithSetMods pwsm) in psm.BestMatchingBioPolymersWithSetMods) @@ -642,7 +656,7 @@ public static IEnumerable CreatePsmData(string searchType, List<(string label = false; newPsmData = CreateOnePsmDataEntry(searchType, fileSpecificParameters, psm, timeDependantHydrophobicityAverageAndDeviation_unmodified, timeDependantHydrophobicityAverageAndDeviation_modified, fileSpecificMedianFragmentMassErrors, chargeStateMode, csm.BestMatchingBioPolymersWithSetMods.First().Peptide, 0, label); } - else if (!csm.IsDecoy && !csm.BetaPeptide.IsDecoy && psm.FdrInfo.QValue <= 0.0005) + else if (!csm.IsDecoy && !csm.BetaPeptide.IsDecoy && psm.GetFdrInfo(PeptideLevelFdr).QValue <= 0.0005) { label = true; newPsmData = CreateOnePsmDataEntry(searchType, fileSpecificParameters, psm, timeDependantHydrophobicityAverageAndDeviation_unmodified, timeDependantHydrophobicityAverageAndDeviation_modified, fileSpecificMedianFragmentMassErrors, chargeStateMode, csm.BestMatchingBioPolymersWithSetMods.First().Peptide, 0, label); @@ -662,7 +676,7 @@ public static IEnumerable CreatePsmData(string searchType, List<(string label = false; newPsmData = CreateOnePsmDataEntry(searchType, fileSpecificParameters, psm, timeDependantHydrophobicityAverageAndDeviation_unmodified, timeDependantHydrophobicityAverageAndDeviation_modified, fileSpecificMedianFragmentMassErrors, chargeStateMode, peptideWithSetMods, notch, label); } - else if (!peptideWithSetMods.Parent.IsDecoy && psm.FdrInfo.QValue <= 0.005) + else if (!peptideWithSetMods.Parent.IsDecoy && psm.GetFdrInfo(PeptideLevelFdr).QValue <= 0.005) { label = true; newPsmData = CreateOnePsmDataEntry(searchType, fileSpecificParameters, psm, timeDependantHydrophobicityAverageAndDeviation_unmodified, timeDependantHydrophobicityAverageAndDeviation_modified, fileSpecificMedianFragmentMassErrors, chargeStateMode, peptideWithSetMods, notch, label); diff --git a/MetaMorpheus/EngineLayer/GlycoSearch/GlycoSpectralMatch.cs b/MetaMorpheus/EngineLayer/GlycoSearch/GlycoSpectralMatch.cs index acc6db3be..ae2c82b55 100644 --- a/MetaMorpheus/EngineLayer/GlycoSearch/GlycoSpectralMatch.cs +++ b/MetaMorpheus/EngineLayer/GlycoSearch/GlycoSpectralMatch.cs @@ -248,7 +248,7 @@ public string SingleToString() sb.Append((IsDecoy) ? "D" : (IsContaminant) ? "C" : "T"); sb.Append("\t"); - sb.Append(FdrInfo!=null? FdrInfo.QValue.ToString() : "-1" ); sb.Append("\t"); + sb.Append(PsmFdrInfo!=null? PsmFdrInfo.QValue.ToString() : "-1" ); sb.Append("\t"); sb.Append("0" + "\t"); //This is space for PEP diff --git a/MetaMorpheus/EngineLayer/Gptmd/GptmdEngine.cs b/MetaMorpheus/EngineLayer/Gptmd/GptmdEngine.cs index 8c62576a6..c34782d99 100644 --- a/MetaMorpheus/EngineLayer/Gptmd/GptmdEngine.cs +++ b/MetaMorpheus/EngineLayer/Gptmd/GptmdEngine.cs @@ -70,7 +70,7 @@ protected override MetaMorpheusEngineResults RunSpecific() int modsAdded = 0; //foreach peptide in each psm and for each modification that matches the notch, //add that modification to every allowed residue - foreach (var psm in AllIdentifications.Where(b => b.FdrInfo.QValueNotch <= 0.05 && !b.IsDecoy)) + foreach (var psm in AllIdentifications.Where(b => b.PsmFdrInfo.QValueNotch <= 0.05 && !b.IsDecoy)) { // get file-specific precursor tolerance Tolerance precursorMassTolerance = FilePathToPrecursorMassTolerance[psm.FullFilePath]; diff --git a/MetaMorpheus/EngineLayer/ModificationAnalysis/ModificationAnalysisEngine.cs b/MetaMorpheus/EngineLayer/ModificationAnalysis/ModificationAnalysisEngine.cs index 44350e1cb..037104188 100644 --- a/MetaMorpheus/EngineLayer/ModificationAnalysis/ModificationAnalysisEngine.cs +++ b/MetaMorpheus/EngineLayer/ModificationAnalysis/ModificationAnalysisEngine.cs @@ -19,7 +19,7 @@ protected override MetaMorpheusEngineResults RunSpecific() ModificationAnalysisResults myAnalysisResults = new ModificationAnalysisResults(this); - var confidentTargetPsms = NewPsms.Where(b => b.FdrInfo.QValue <= 0.01 && !b.IsDecoy).ToList(); + var confidentTargetPsms = NewPsms.Where(b => b.PsmFdrInfo.QValue <= 0.01 && !b.IsDecoy).ToList(); // For the database ones, only need un-ambiguous protein and location in protein var forObserved = confidentTargetPsms diff --git a/MetaMorpheus/EngineLayer/NonSpecificEnzymeSearch/NonSpecificEnzymeSearchEngine.cs b/MetaMorpheus/EngineLayer/NonSpecificEnzymeSearch/NonSpecificEnzymeSearchEngine.cs index 1426bd4aa..acd6caa28 100644 --- a/MetaMorpheus/EngineLayer/NonSpecificEnzymeSearch/NonSpecificEnzymeSearchEngine.cs +++ b/MetaMorpheus/EngineLayer/NonSpecificEnzymeSearch/NonSpecificEnzymeSearchEngine.cs @@ -460,7 +460,7 @@ public static List ResolveFdrCategorySpecificPsms(List ResolveFdrCategorySpecificPsms(List x != null).Count(x => x.FdrInfo.QValue <= 0.01); //set ranking as number of psms above 1% FDR + ranking[i] = AllPsms[i].Where(x => x != null).Count(x => x.PsmFdrInfo.QValue <= 0.01); //set ranking as number of psms above 1% FDR indexesOfInterest.Add(i); } } @@ -515,9 +515,9 @@ public static List ResolveFdrCategorySpecificPsms(List minorPsm.FdrInfo.QValue) + if (majorPsm.PsmFdrInfo.QValue > minorPsm.PsmFdrInfo.QValue) { - minorPsm.FdrInfo.QValue = majorPsm.FdrInfo.QValue; + minorPsm.PsmFdrInfo.QValue = majorPsm.PsmFdrInfo.QValue; } minorPsmIndex++; } @@ -527,9 +527,9 @@ public static List ResolveFdrCategorySpecificPsms(List minorPsm.FdrInfo.QValue) + if (majorPsm.PsmFdrInfo.QValue > minorPsm.PsmFdrInfo.QValue) { - minorPsm.FdrInfo.QValue = majorPsm.FdrInfo.QValue; + minorPsm.PsmFdrInfo.QValue = majorPsm.PsmFdrInfo.QValue; } minorPsmIndex++; } @@ -548,7 +548,7 @@ public static List ResolveFdrCategorySpecificPsms(List bestPsm.Score)) { @@ -587,7 +587,7 @@ public static List ResolveFdrCategorySpecificPsms(List b.FdrInfo.QValue).ThenByDescending(b => b.Score).ToList(); + return bestPsmsList.OrderBy(b => b.PsmFdrInfo.QValue).ThenByDescending(b => b.Score).ToList(); } public static List GetVariableTerminalMods(FragmentationTerminus fragmentationTerminus, List variableModifications) diff --git a/MetaMorpheus/EngineLayer/ProteinParsimony/ProteinParsimonyEngine.cs b/MetaMorpheus/EngineLayer/ProteinParsimony/ProteinParsimonyEngine.cs index d57cc666b..081aab800 100644 --- a/MetaMorpheus/EngineLayer/ProteinParsimony/ProteinParsimonyEngine.cs +++ b/MetaMorpheus/EngineLayer/ProteinParsimony/ProteinParsimonyEngine.cs @@ -41,11 +41,11 @@ public ProteinParsimonyEngine(List allPsms, bool modPeptidesAreDi // KEEP contaminants for use in parsimony! if (modPeptidesAreDifferent) { - _fdrFilteredPsms = allPsms.Where(p => p.FullSequence != null && p.FdrInfo.QValue <= FdrCutoffForParsimony && p.FdrInfo.QValueNotch <= FdrCutoffForParsimony).ToList(); + _fdrFilteredPsms = allPsms.Where(p => p.FullSequence != null && p.PsmFdrInfo.QValue <= FdrCutoffForParsimony && p.PsmFdrInfo.QValueNotch <= FdrCutoffForParsimony).ToList(); } else { - _fdrFilteredPsms = allPsms.Where(p => p.BaseSequence != null && p.FdrInfo.QValue <= FdrCutoffForParsimony && p.FdrInfo.QValueNotch <= FdrCutoffForParsimony).ToList(); + _fdrFilteredPsms = allPsms.Where(p => p.BaseSequence != null && p.PsmFdrInfo.QValue <= FdrCutoffForParsimony && p.PsmFdrInfo.QValueNotch <= FdrCutoffForParsimony).ToList(); } // peptides to use in parsimony = peptides observed in high-confidence PSMs (including decoys) diff --git a/MetaMorpheus/EngineLayer/ProteinScoringAndFdr/ProteinScoringAndFdrEngine.cs b/MetaMorpheus/EngineLayer/ProteinScoringAndFdr/ProteinScoringAndFdrEngine.cs index d1459cc87..16ac6cadb 100644 --- a/MetaMorpheus/EngineLayer/ProteinScoringAndFdr/ProteinScoringAndFdrEngine.cs +++ b/MetaMorpheus/EngineLayer/ProteinScoringAndFdr/ProteinScoringAndFdrEngine.cs @@ -42,7 +42,7 @@ private void ScoreProteinGroups(List proteinGroups, IEnumerable>(); foreach (var psm in psmList) { - if (psm.FdrInfo.QValueNotch <= 0.01 && psm.FdrInfo.QValue <= 0.01) + if (psm.PsmFdrInfo.QValueNotch <= 0.01 && psm.PsmFdrInfo.QValue <= 0.01) { if ((TreatModPeptidesAsDifferentPeptides && psm.FullSequence != null) || (!TreatModPeptidesAsDifferentPeptides && psm.BaseSequence != null)) { @@ -155,7 +155,7 @@ private List DoProteinFdr(List proteinGroups) } pg.BestPeptideScore = pg.AllPsmsBelowOnePercentFDR.Max(psm => psm.Score); - pg.BestPeptideQValue = pg.AllPsmsBelowOnePercentFDR.Min(psm => psm.FdrInfo.QValueNotch); + pg.BestPeptideQValue = pg.AllPsmsBelowOnePercentFDR.Min(psm => psm.PsmFdrInfo.QValueNotch); } // pick the best notch-QValue for each paired accession diff --git a/MetaMorpheus/EngineLayer/PsmTsv/PsmTsvWriter.cs b/MetaMorpheus/EngineLayer/PsmTsv/PsmTsvWriter.cs index 89404391e..bc1377d68 100644 --- a/MetaMorpheus/EngineLayer/PsmTsv/PsmTsvWriter.cs +++ b/MetaMorpheus/EngineLayer/PsmTsv/PsmTsvWriter.cs @@ -316,7 +316,7 @@ internal static void AddMatchedIonsData(Dictionary s, List s, SpectralMatch peptide) + internal static void AddMatchScoreData(Dictionary s, SpectralMatch peptide, bool asPeptide) { string spectralAngle = peptide == null ? " " : peptide.SpectralAngle.ToString("F4"); string localizedScores = " "; @@ -339,16 +339,16 @@ internal static void AddMatchScoreData(Dictionary s, SpectralMat string PEP = " "; string PEP_Qvalue = " "; - if (peptide != null && peptide.FdrInfo != null) + if (peptide != null && peptide.GetFdrInfo(asPeptide) != null) { - cumulativeTarget = peptide.FdrInfo.CumulativeTarget.ToString(CultureInfo.InvariantCulture); - cumulativeDecoy = peptide.FdrInfo.CumulativeDecoy.ToString(CultureInfo.InvariantCulture); - qValue = peptide.FdrInfo.QValue.ToString("F6", CultureInfo.InvariantCulture); - cumulativeTargetNotch = peptide.FdrInfo.CumulativeTargetNotch.ToString(CultureInfo.InvariantCulture); - cumulativeDecoyNotch = peptide.FdrInfo.CumulativeDecoyNotch.ToString(CultureInfo.InvariantCulture); - qValueNotch = peptide.FdrInfo.QValueNotch.ToString("F6", CultureInfo.InvariantCulture); - PEP = peptide.FdrInfo.PEP.ToString(); - PEP_Qvalue = peptide.FdrInfo.PEP_QValue.ToString(); + cumulativeTarget = peptide.GetFdrInfo(asPeptide).CumulativeTarget.ToString(CultureInfo.InvariantCulture); + cumulativeDecoy = peptide.GetFdrInfo(asPeptide).CumulativeDecoy.ToString(CultureInfo.InvariantCulture); + qValue = peptide.GetFdrInfo(asPeptide).QValue.ToString("F6", CultureInfo.InvariantCulture); + cumulativeTargetNotch = peptide.GetFdrInfo(asPeptide).CumulativeTargetNotch.ToString(CultureInfo.InvariantCulture); + cumulativeDecoyNotch = peptide.GetFdrInfo(asPeptide).CumulativeDecoyNotch.ToString(CultureInfo.InvariantCulture); + qValueNotch = peptide.GetFdrInfo(asPeptide).QValueNotch.ToString("F6", CultureInfo.InvariantCulture); + PEP = peptide.GetFdrInfo(asPeptide).PEP.ToString(); + PEP_Qvalue = peptide.GetFdrInfo(asPeptide).PEP_QValue.ToString(); } s[PsmTsvHeader.CumulativeTarget] = cumulativeTarget; s[PsmTsvHeader.CumulativeDecoy] = cumulativeDecoy; diff --git a/MetaMorpheus/EngineLayer/SpectralMatch.cs b/MetaMorpheus/EngineLayer/SpectralMatch.cs index e04e82fa5..9fe2844ba 100644 --- a/MetaMorpheus/EngineLayer/SpectralMatch.cs +++ b/MetaMorpheus/EngineLayer/SpectralMatch.cs @@ -71,7 +71,8 @@ protected SpectralMatch(IBioPolymerWithSetMods peptide, int notch, double score, public string FullFilePath { get; private set; } public int ScanIndex { get; } public int NumDifferentMatchingPeptides { get { return _BestMatchingBioPolymersWithSetMods.Count; } } - public FdrInfo FdrInfo { get; private set; } + public FdrInfo PsmFdrInfo { get; private set; } + public FdrInfo PeptideFdrInfo { get; private set; } public PsmData PsmData_forPEPandPercolator { get; set; } public double Score { get; private set; } @@ -216,9 +217,13 @@ public void ResolveAllAmbiguities() MatchedFragmentIons = BioPolymersWithSetModsToMatchingFragments.First().Value; } - public void SetFdrValues(double cumulativeTarget, double cumulativeDecoy, double qValue, double cumulativeTargetNotch, double cumulativeDecoyNotch, double qValueNotch, double pep, double pepQValue) + public void SetFdrValues(double cumulativeTarget, double cumulativeDecoy, + double qValue, double cumulativeTargetNotch, + double cumulativeDecoyNotch, double qValueNotch, + double pep, double pepQValue, + bool peptideFdr = false) { - FdrInfo = new FdrInfo + var fdrInfo = new FdrInfo { CumulativeTarget = cumulativeTarget, CumulativeDecoy = cumulativeDecoy, @@ -229,7 +234,22 @@ public void SetFdrValues(double cumulativeTarget, double cumulativeDecoy, double PEP = pep, PEP_QValue = pepQValue }; + + if (peptideFdr) + { + PeptideFdrInfo = fdrInfo; + } + else + { + PsmFdrInfo = fdrInfo; + } } + + public FdrInfo GetFdrInfo(bool peptideFdr = false) + { + return peptideFdr ? PeptideFdrInfo : PsmFdrInfo; + } + /// /// This method is used to compute qValue etc for the inverted set of psms /// We neither compute nor calculated cumulativeTarget, cumulativeDecoy, etc for the inverted set. @@ -240,12 +260,23 @@ public void SetFdrValues(double cumulativeTarget, double cumulativeDecoy, double /// /// /// - public void SetQandPEPvalues(double qValue, double qValueNotch, double pep, double pepQValue) + public void SetQandPEPvalues(double qValue, double qValueNotch, double pep, double pepQValue, bool peptideLevelFdr = false) { - FdrInfo.QValue = qValue; - FdrInfo.QValueNotch = qValueNotch; - FdrInfo.PEP = pep; - FdrInfo.PEP_QValue = pepQValue; + if(peptideLevelFdr) + { + PeptideFdrInfo.QValue = qValue; + PeptideFdrInfo.QValueNotch = qValueNotch; + PeptideFdrInfo.PEP = pep; + PeptideFdrInfo.PEP_QValue = pepQValue; + } + else + { + PsmFdrInfo.QValue = qValue; + PsmFdrInfo.QValueNotch = qValueNotch; + PsmFdrInfo.PEP = pep; + PsmFdrInfo.PEP_QValue = pepQValue; + } + } @@ -264,18 +295,18 @@ public override string ToString() return ToString(new Dictionary()); } - public string ToString(IReadOnlyDictionary ModstoWritePruned) + public string ToString(IReadOnlyDictionary ModstoWritePruned, bool asPeptide = false) { - return string.Join("\t", DataDictionary(this, ModstoWritePruned).Values); + return string.Join("\t", DataDictionary(this, ModstoWritePruned, asPeptide).Values); } - public static Dictionary DataDictionary(SpectralMatch psm, IReadOnlyDictionary ModsToWritePruned) + public static Dictionary DataDictionary(SpectralMatch psm, IReadOnlyDictionary ModsToWritePruned, bool asPeptide = false) { Dictionary s = new Dictionary(); PsmTsvWriter.AddBasicMatchData(s, psm); PsmTsvWriter.AddPeptideSequenceData(s, psm, ModsToWritePruned); PsmTsvWriter.AddMatchedIonsData(s, psm?.MatchedFragmentIons); - PsmTsvWriter.AddMatchScoreData(s, psm); + PsmTsvWriter.AddMatchScoreData(s, psm, asPeptide); return s; } @@ -353,7 +384,7 @@ protected SpectralMatch(SpectralMatch psm, List<(int Notch, IBioPolymerWithSetMo ScanPrecursorMass = psm.ScanPrecursorMass; FullFilePath = psm.FullFilePath; ScanIndex = psm.ScanIndex; - FdrInfo = psm.FdrInfo; + PsmFdrInfo = psm.PsmFdrInfo; Score = psm.Score; Xcorr = psm.Xcorr; RunnerUpScore = psm.RunnerUpScore; diff --git a/MetaMorpheus/TaskLayer/CalibrationTask/CalibrationTask.cs b/MetaMorpheus/TaskLayer/CalibrationTask/CalibrationTask.cs index 82b2762a1..8a7320f84 100644 --- a/MetaMorpheus/TaskLayer/CalibrationTask/CalibrationTask.cs +++ b/MetaMorpheus/TaskLayer/CalibrationTask/CalibrationTask.cs @@ -278,7 +278,7 @@ private DataPointAquisitionResults GetDataAcquisitionResults(MsDataFile myMsData _ = new FdrAnalysisEngine(allPsms, searchMode.NumNotches, CommonParameters, FileSpecificParameters, new List { taskId, "Individual Spectra Files", fileNameWithoutExtension }, doPEP: false).Run(); - List goodIdentifications = allPsms.Where(b => b.FdrInfo.QValueNotch < 0.001 && !b.IsDecoy && b.FullSequence != null).ToList(); + List goodIdentifications = allPsms.Where(b => b.PsmFdrInfo.QValueNotch < 0.001 && !b.IsDecoy && b.FullSequence != null).ToList(); if (!goodIdentifications.Any()) { diff --git a/MetaMorpheus/TaskLayer/GlycoSearchTask/GlycoProteinParsimony.cs b/MetaMorpheus/TaskLayer/GlycoSearchTask/GlycoProteinParsimony.cs index 9d90d65d5..1cc247464 100644 --- a/MetaMorpheus/TaskLayer/GlycoSearchTask/GlycoProteinParsimony.cs +++ b/MetaMorpheus/TaskLayer/GlycoSearchTask/GlycoProteinParsimony.cs @@ -70,13 +70,13 @@ public GlycoProteinParsimony(string proteinAccess, int proteinPos, char aminoAci if (!localizedGlycan.ContainsKey(proPosId)) { - GlycoProteinParsimony gpp = new GlycoProteinParsimony(gsm.Accession, proteinPos, gsm.BaseSequence[local.Item1-2], local.Item3, gsm.FdrInfo.QValue, gsm.LocalizationLevel, prob); + GlycoProteinParsimony gpp = new GlycoProteinParsimony(gsm.Accession, proteinPos, gsm.BaseSequence[local.Item1-2], local.Item3, gsm.PsmFdrInfo.QValue, gsm.LocalizationLevel, prob); localizedGlycan.Add(proPosId, gpp); } else { bool islocalized = (local.Item3 || localizedGlycan[proPosId].IsLocalized); - double minQValue = localizedGlycan[proPosId].MinQValue > gsm.FdrInfo.QValue ? gsm.FdrInfo.QValue : localizedGlycan[proPosId].MinQValue; + double minQValue = localizedGlycan[proPosId].MinQValue > gsm.PsmFdrInfo.QValue ? gsm.PsmFdrInfo.QValue : localizedGlycan[proPosId].MinQValue; double maxProb = localizedGlycan[proPosId].MaxProbability > prob ? localizedGlycan[proPosId].MaxProbability : prob; var localLevel = localizedGlycan[proPosId].BestLocalizeLevel < gsm.LocalizationLevel ? localizedGlycan[proPosId].BestLocalizeLevel : gsm.LocalizationLevel; diff --git a/MetaMorpheus/TaskLayer/GlycoSearchTask/PostGlycoSearchAnalysisTask.cs b/MetaMorpheus/TaskLayer/GlycoSearchTask/PostGlycoSearchAnalysisTask.cs index 6578a1790..6d9bbbaf7 100644 --- a/MetaMorpheus/TaskLayer/GlycoSearchTask/PostGlycoSearchAnalysisTask.cs +++ b/MetaMorpheus/TaskLayer/GlycoSearchTask/PostGlycoSearchAnalysisTask.cs @@ -44,7 +44,7 @@ public MyTaskResults Run(string OutputFolder, List dbFilenameList, Li //This is all psms for all files including glyco- and non-glyco psms. SingleFDRAnalysis(allPSMs, commonParameters, new List { taskId }); - List filteredGsms = allPSMs.Where(p => p.FdrInfo.QValue < 0.01).ToList(); + List filteredGsms = allPSMs.Where(p => p.PsmFdrInfo.QValue < 0.01).ToList(); //write individual file results if (Parameters.GlycoSearchParameters.WriteIndividualFiles) @@ -263,7 +263,7 @@ private void GlycoAccessionAnalysis(List gsms, string indivi } //get non-ambigous psms above cutoffs that are normally used for parsimony - List _filteredPsms = psmsForProteinParsimony.Where(p => p.FullSequence != null && p.FdrInfo.QValue <= 0.01 && p.FdrInfo.QValueNotch <= 0.01).ToList(); + List _filteredPsms = psmsForProteinParsimony.Where(p => p.FullSequence != null && p.PsmFdrInfo.QValue <= 0.01 && p.PsmFdrInfo.QValueNotch <= 0.01).ToList(); // if there are no peptides observed, there are no proteins; return an empty list of protein groups if (_filteredPsms.Count != 0) @@ -366,8 +366,8 @@ private void QuantificationAnalysis() // get PSMs to pass to FlashLFQ var unambiguousPsmsBelowOnePercentFdr = Parameters.AllPsms.Where(p => - p.FdrInfo.QValue <= 0.01 - && p.FdrInfo.QValueNotch <= 0.01 + p.PsmFdrInfo.QValue <= 0.01 + && p.PsmFdrInfo.QValueNotch <= 0.01 && !p.IsDecoy && p.FullSequence != null).ToList(); diff --git a/MetaMorpheus/TaskLayer/MbrAnalysis/SpectralRecoveryPSM.cs b/MetaMorpheus/TaskLayer/MbrAnalysis/SpectralRecoveryPSM.cs index 1ee92d8b6..96379eba5 100644 --- a/MetaMorpheus/TaskLayer/MbrAnalysis/SpectralRecoveryPSM.cs +++ b/MetaMorpheus/TaskLayer/MbrAnalysis/SpectralRecoveryPSM.cs @@ -65,9 +65,9 @@ public static string TabSeparatedHeader public override string ToString() { - string originalPsmQ = originalSpectralMatch == null ? "Spectrum Not Found" : originalSpectralMatch.FdrInfo.QValue.ToString(); - string originalPsmPEP = originalSpectralMatch == null ? "" : originalSpectralMatch.FdrInfo.PEP.ToString(); - string originalPsmPEPQ = originalSpectralMatch == null ? "" : originalSpectralMatch.FdrInfo.PEP_QValue.ToString(); + string originalPsmQ = originalSpectralMatch == null ? "Spectrum Not Found" : originalSpectralMatch.PsmFdrInfo.QValue.ToString(); + string originalPsmPEP = originalSpectralMatch == null ? "" : originalSpectralMatch.PsmFdrInfo.PEP.ToString(); + string originalPsmPEPQ = originalSpectralMatch == null ? "" : originalSpectralMatch.PsmFdrInfo.PEP_QValue.ToString(); var sb = new StringBuilder(); sb.Append(spectralLibraryMatch); sb.Append('\t'); diff --git a/MetaMorpheus/TaskLayer/MbrAnalysis/SpectralRecoveryRunner.cs b/MetaMorpheus/TaskLayer/MbrAnalysis/SpectralRecoveryRunner.cs index 34e5a591d..d8a7307a5 100644 --- a/MetaMorpheus/TaskLayer/MbrAnalysis/SpectralRecoveryRunner.cs +++ b/MetaMorpheus/TaskLayer/MbrAnalysis/SpectralRecoveryRunner.cs @@ -118,7 +118,7 @@ public static SpectralRecoveryResults RunSpectralRecoveryAlgorithm( { List allPsms = parameters.AllPsms. OrderByDescending(p => p.Score). - ThenBy(p => p.FdrInfo.QValue). + ThenBy(p => p.PsmFdrInfo.QValue). ThenBy(p => p.FullFilePath). ThenBy(x => x.ScanNumber). ThenBy(p => p.FullSequence). @@ -158,11 +158,11 @@ private static List GetAllPeptides( double qValueCutoff = 0.01; if (parameters.AllPsms.Count > 100)//PEP is not computed when there are fewer than 100 psms { - peptides.RemoveAll(p => p.FdrInfo.PEP_QValue > qValueCutoff); + peptides.RemoveAll(p => p.PsmFdrInfo.PEP_QValue > qValueCutoff); } else { - peptides.RemoveAll(p => p.FdrInfo.QValue > qValueCutoff); + peptides.RemoveAll(p => p.PsmFdrInfo.QValue > qValueCutoff); } return peptides; @@ -191,7 +191,7 @@ private static SpectralMatch BestPsmForMbrPeak(IEnumerable peptid private static void AssignEstimatedPsmQvalue(ConcurrentDictionary bestMbrMatches, List allPsms) { double[] allScores = allPsms.Select(s => s.Score).OrderByDescending(s => s).ToArray(); - double[] allQValues = allPsms.OrderByDescending(s => s.Score).Select(q => q.FdrInfo.QValue).ToArray(); + double[] allQValues = allPsms.OrderByDescending(s => s.Score).Select(q => q.PsmFdrInfo.QValue).ToArray(); foreach (SpectralRecoveryPSM match in bestMbrMatches.Values) { @@ -204,12 +204,12 @@ private static void AssignEstimatedPsmQvalue(ConcurrentDictionary pepValues = bestMbrMatches. Select(p => p.Value.spectralLibraryMatch). Where(p => p != null). - OrderBy(p => p.FdrInfo.PEP). - Select(p => p.FdrInfo.PEP). + OrderBy(p => p.PsmFdrInfo.PEP). + Select(p => p.PsmFdrInfo.PEP). ToList(); foreach (SpectralRecoveryPSM match in bestMbrMatches.Values) @@ -271,18 +271,18 @@ private static void AssignEstimatedPsmPepQValue(ConcurrentDictionary variableM } } - protected static void WritePsmsToTsv(IEnumerable psms, string filePath, IReadOnlyDictionary modstoWritePruned) + protected static void WritePsmsToTsv(IEnumerable psms, string filePath, IReadOnlyDictionary modstoWritePruned, bool asPeptide = false) { using (StreamWriter output = new StreamWriter(filePath)) { output.WriteLine(SpectralMatch.GetTabSeparatedHeader()); foreach (var psm in psms) { - output.WriteLine(psm.ToString(modstoWritePruned)); + output.WriteLine(psm.ToString(modstoWritePruned), asPeptide); } } } diff --git a/MetaMorpheus/TaskLayer/PepXMLWriter.cs b/MetaMorpheus/TaskLayer/PepXMLWriter.cs index 3b30ab068..98a6fd69a 100644 --- a/MetaMorpheus/TaskLayer/PepXMLWriter.cs +++ b/MetaMorpheus/TaskLayer/PepXMLWriter.cs @@ -151,7 +151,7 @@ public static void WritePepXml(List psms, List databas search_score = new pepXML.Generated.nameValueType[] { new pepXML.Generated.nameValueType{ name = "Score", value = psm.Score.ToString()}, - new pepXML.Generated.nameValueType{ name = "Qvalue", value = psm.FdrInfo.QValue.ToString() } + new pepXML.Generated.nameValueType{ name = "Qvalue", value = psm.PsmFdrInfo.QValue.ToString() } }, }; searchHits.Add(searchHit); diff --git a/MetaMorpheus/TaskLayer/SearchTask/MzIdentMLWriter.cs b/MetaMorpheus/TaskLayer/SearchTask/MzIdentMLWriter.cs index 4b842483d..833e129ec 100644 --- a/MetaMorpheus/TaskLayer/SearchTask/MzIdentMLWriter.cs +++ b/MetaMorpheus/TaskLayer/SearchTask/MzIdentMLWriter.cs @@ -418,7 +418,7 @@ public static void WriteMzIdentMl(IEnumerable psms, List p.Peptide).Distinct().Count()], @@ -436,7 +436,7 @@ public static void WriteMzIdentMl(IEnumerable psms, List> PsmsGroupedByFile { get; set; } private SpectralRecoveryResults SpectralRecoveryResults { get; set; } private List _filteredPsms; + private List _filteredPeptides; private bool _pepFilteringNotPerformed; private string _filterType; private double _filterThreshold; @@ -75,6 +76,7 @@ public MyTaskResults Run() FilterAllPsms(); DoMassDifferenceLocalizationAnalysis(); + PeptideAnalysis(); ProteinAnalysis(); QuantificationAnalysis(); @@ -145,11 +147,11 @@ private void FilterAllPsms() _filteredPsms = _filterType.Equals("q-value") ? Parameters.AllPsms.Where(p => - p.FdrInfo.QValue <= _filterThreshold - && p.FdrInfo.QValueNotch <= _filterThreshold) + p.PsmFdrInfo.QValue <= _filterThreshold + && p.PsmFdrInfo.QValueNotch <= _filterThreshold) .ToList() : Parameters.AllPsms.Where(p => - p.FdrInfo.PEP_QValue <= _filterThreshold) + p.PsmFdrInfo.PEP_QValue <= _filterThreshold) .ToList(); // This property is used for calculating file specific results, which requires calculating @@ -157,6 +159,42 @@ private void FilterAllPsms() PsmsGroupedByFile = Parameters.AllPsms.GroupBy(p => p.FullFilePath); } + /// + /// Sets the private field _filteredPeptides by removing all peptides with Q and Q_Notch or PEP_QValues greater + /// than a user defined threshold. Q-Value and PEP Q-Value filtering are mutually exculsive. + /// In cases where PEP filtering was selected but PEP wasn't performed due to insufficient number of results, + /// filtering defaults to Q and Q_Notch. + /// _filteredPsms can be accessed through the GetFilteredPsms method. + /// This method is called by the WritePeptideResults method, and must be performed before QuantificationAnalysis + /// + private void FilterAllPeptides(List peptideMatches) + { + _filterType = "q-value"; + _filterThreshold = Math.Min(CommonParameters.QValueThreshold, CommonParameters.PepQValueThreshold); + + if (CommonParameters.PepQValueThreshold < CommonParameters.QValueThreshold) + { + if (Parameters.AllPsms.Count < 100) + { + _pepFilteringNotPerformed = true; + } + else + { + _filterType = "pep q-value"; + } + } + + _filteredPeptides = _filterType.Equals("q-value") + ? peptideMatches.Where(p => + p.PeptideFdrInfo.QValue <= _filterThreshold + && p.PeptideFdrInfo.QValueNotch <= _filterThreshold) + .ToList() + : peptideMatches.Where(p => + p.PeptideFdrInfo.PEP_QValue <= _filterThreshold) + .ToList(); + } + + public IEnumerable GetFilteredPsms(bool includeDecoys, bool includeContaminants, bool includeAmbiguous) { @@ -171,29 +209,29 @@ public IEnumerable GetFilteredPsms(bool includeDecoys, bool inclu /// /// A list of PSMs to be modified in place /// The number of target psms scoring below threshold - private void FilterSpecificPsms(List fileSpecificPsmsOrPeptides, out int psmOrPeptideCountForResults) + private void FilterSpecificPsms(List fileSpecificPsmsOrPeptides, out int psmOrPeptideCountForResults, bool peptideLevelFiltering = false) { psmOrPeptideCountForResults = _filterType.Equals("q-value") ? fileSpecificPsmsOrPeptides.Count(p => !p.IsDecoy - && p.FdrInfo.QValue <= _filterThreshold - && p.FdrInfo.QValueNotch <= _filterThreshold) + && p.GetFdrInfo(peptideLevelFiltering).QValue <= _filterThreshold + && p.GetFdrInfo(peptideLevelFiltering).QValueNotch <= _filterThreshold) : fileSpecificPsmsOrPeptides.Count(p => !p.IsDecoy - && p.FdrInfo.PEP_QValue <= _filterThreshold); + && p.GetFdrInfo(peptideLevelFiltering).PEP_QValue <= _filterThreshold); if (!Parameters.SearchParameters.WriteHighQValuePsms) { if (_filterType.Equals("q-value")) { fileSpecificPsmsOrPeptides.RemoveAll(p => - p.FdrInfo.QValue > _filterThreshold | - p.FdrInfo.QValueNotch > _filterThreshold); + p.GetFdrInfo(peptideLevelFiltering).QValue > _filterThreshold | + p.GetFdrInfo(peptideLevelFiltering).QValueNotch > _filterThreshold); } else { fileSpecificPsmsOrPeptides.RemoveAll(p => - p.FdrInfo.PEP_QValue > _filterThreshold); + p.GetFdrInfo(peptideLevelFiltering).PEP_QValue > _filterThreshold); } } if (!Parameters.SearchParameters.WriteDecoys) @@ -221,9 +259,9 @@ private void CalculatePsmFdr() // sort by q-value because of group FDR stuff // e.g. multiprotease FDR, non/semi-specific protease, etc Parameters.AllPsms = Parameters.AllPsms - .OrderBy(p => p.FdrInfo.QValue) + .OrderBy(p => p.PsmFdrInfo.QValue) .ThenByDescending(p => p.Score) - .ThenBy(p => p.FdrInfo.CumulativeTarget) + .ThenBy(p => p.PsmFdrInfo.CumulativeTarget) .ToList(); Status("Done estimating PSM FDR!", Parameters.SearchTaskId); @@ -590,7 +628,8 @@ private void QuantificationAnalysis() matchBetweenRuns: Parameters.SearchParameters.MatchBetweenRuns, useSharedPeptidesForProteinQuant: Parameters.SearchParameters.UseSharedPeptidesForLFQ, silent: true, - maxThreads: CommonParameters.MaxThreadsToUsePerFile); + maxThreads: CommonParameters.MaxThreadsToUsePerFile, + peptideSequencesToUse: Parameters.SearchParameters.SilacLabels == null ? _filteredPeptides.Select(peptide => peptide.FullSequence).ToList() : null); if (flashLFQIdentifications.Any()) { @@ -653,7 +692,7 @@ private void HistogramAnalysis() /// /// PSMs to be written /// Full file path, up to and including the filename and extensioh. - protected void WritePsmsToTsv(IEnumerable psms, string filePath) + protected void WritePsmsToTsv(IEnumerable psms, string filePath, bool asPeptide = false) { if (Parameters.SearchParameters.DoMultiplexQuantification && Parameters.MultiplexModification != null && @@ -661,11 +700,11 @@ protected void WritePsmsToTsv(IEnumerable psms, string filePath) .SelectMany(pwsm => pwsm.Peptide.AllModsOneIsNterminus.Values) .Any(mod => mod.OriginalId.Equals(Parameters.MultiplexModification.OriginalId)))) { - WritePsmPlusMultiplexIons(psms, filePath); + WritePsmPlusMultiplexIons(psms, filePath, asPeptide); } else { - WritePsmsToTsv(psms, filePath, Parameters.SearchParameters.ModsToWriteSelection); + WritePsmsToTsv(psms, filePath, Parameters.SearchParameters.ModsToWriteSelection, asPeptide); } } @@ -1091,7 +1130,7 @@ private void WritePrunedDatabase() includeContaminants: true, includeAmbiguous: false).ToList(); - var originalModPsms = Parameters.AllPsms.Where(b => b.FdrInfo.QValueNotch <= 0.01 && b.FdrInfo.QValue <= 0.01 && !b.IsDecoy && b.FullSequence != null).ToList(); + var originalModPsms = Parameters.AllPsms.Where(b => b.PsmFdrInfo.QValueNotch <= 0.01 && b.PsmFdrInfo.QValue <= 0.01 && !b.IsDecoy && b.FullSequence != null).ToList(); var proteinToConfidentModifiedSequences = new Dictionary>(); HashSet modPsmsFullSeq = modPsms.Select(p => p.FullSequence).ToHashSet(); @@ -1312,6 +1351,20 @@ private void WritePrunedDatabase() } } + private void PeptideAnalysis() + { + List peptides = Parameters.AllPsms + .GroupBy(b => b.FullSequence) + .Select(b => b.FirstOrDefault()).ToList(); + + new FdrAnalysisEngine(peptides, Parameters.NumNotches, CommonParameters, + FileSpecificParameters, new List { Parameters.SearchTaskId }, + "Peptide", peptideLevelFdr: true, doPEP: false).Run(); + + FilterSpecificPsms(peptides, out int psmOrPeptideCountForResults, peptideLevelFiltering: true); + FilterAllPeptides(peptides); // This sets the _allPeptides field + } + private void WritePeptideResults() { Status("Writing peptide results...", Parameters.SearchTaskId); @@ -1325,11 +1378,12 @@ private void WritePeptideResults() new FdrAnalysisEngine(peptides, Parameters.NumNotches, CommonParameters, FileSpecificParameters, new List { Parameters.SearchTaskId }, - "Peptide").Run(); + "Peptide", peptideLevelFdr: true).Run(); - FilterSpecificPsms(peptides, out int psmOrPeptideCountForResults); + FilterSpecificPsms(peptides, out int psmOrPeptideCountForResults, peptideLevelFiltering: true); + FilterAllPeptides(peptides); - WritePsmsToTsv(peptides, writtenFile); + WritePsmsToTsv(peptides, writtenFile, asPeptide: true); FinishedWritingFile(writtenFile, new List { Parameters.SearchTaskId }); Parameters.SearchTaskResults.AddPsmPeptideProteinSummaryText( @@ -1350,9 +1404,9 @@ private void WritePeptideResults() // FDR Analysis is performed again for each file. File specific results show the results that would be // generated by analyzing one file by itself. Therefore, the FDR info should change between AllResults and FileSpecific new FdrAnalysisEngine(peptidesForFile, Parameters.NumNotches, CommonParameters, FileSpecificParameters, - new List { Parameters.SearchTaskId }, "Peptide").Run(); + new List { Parameters.SearchTaskId }, "Peptide", peptideLevelFdr: true).Run(); - FilterSpecificPsms(peptidesForFile, out psmOrPeptideCountForResults); + FilterSpecificPsms(peptidesForFile, out psmOrPeptideCountForResults, peptideLevelFiltering: true); Parameters.SearchTaskResults.AddTaskSummaryText( strippedFileName + " Target " + GlobalVariables.AnalyteType.ToLower() + "s with " @@ -1368,13 +1422,13 @@ private void WritePeptideResults() // write best (highest-scoring) PSM per peptide filename = "_" + GlobalVariables.AnalyteType + "s.psmtsv"; writtenFile = Path.Combine(Parameters.IndividualResultsOutputFolder, strippedFileName + filename); - WritePsmsToTsv(peptidesForFile, writtenFile); + WritePsmsToTsv(peptidesForFile, writtenFile, asPeptide: true); FinishedWritingFile(writtenFile, new List { Parameters.SearchTaskId, "Individual Spectra Files", file.First().FullFilePath }); } } } - private void WritePsmPlusMultiplexIons(IEnumerable psms, string filePath) + private void WritePsmPlusMultiplexIons(IEnumerable psms, string filePath, bool asPeptide) { PpmTolerance ionTolerance = new PpmTolerance(10); double[] reporterIonMzs = Parameters.MultiplexModification.DiagnosticIons.First().Value @@ -1393,7 +1447,7 @@ private void WritePsmPlusMultiplexIons(IEnumerable psms, string f GetMultiplexIonIntensities(psm.MsDataScan.MassSpectrum, reporterIonMzs, ionTolerance) .Select(d => d.ToString(CultureInfo.CurrentCulture)); - output.Write(psm.ToString(Parameters.SearchParameters.ModsToWriteSelection).Trim()); + output.Write(psm.ToString(Parameters.SearchParameters.ModsToWriteSelection, asPeptide).Trim()); output.Write('\t'); output.WriteLine(String.Join('\t', labelIonIntensities)); } @@ -1502,9 +1556,9 @@ private void WriteVariantResults() new List { Parameters.SearchTaskId }, "variant_PSMs").Run(); possibleVariantPsms - .OrderBy(p => p.FdrInfo.QValue) + .OrderBy(p => p.PsmFdrInfo.QValue) .ThenByDescending(p => p.Score) - .ThenBy(p => p.FdrInfo.CumulativeTarget) + .ThenBy(p => p.PsmFdrInfo.CumulativeTarget) .ToList(); FilterSpecificPsms(possibleVariantPsms, out int countOfConfidentPsms); diff --git a/MetaMorpheus/TaskLayer/XLSearchTask/PostXLSearchAnalysisTask.cs b/MetaMorpheus/TaskLayer/XLSearchTask/PostXLSearchAnalysisTask.cs index fdf0e6f76..832963786 100644 --- a/MetaMorpheus/TaskLayer/XLSearchTask/PostXLSearchAnalysisTask.cs +++ b/MetaMorpheus/TaskLayer/XLSearchTask/PostXLSearchAnalysisTask.cs @@ -43,21 +43,21 @@ public MyTaskResults Run(string outputFolder, List dbFilenameList, Li ComputeXlinkQandPValues(allPsms, intraCsms, interCsms, commonParameters, taskId); WriteCsvFiles(outputFolder, interCsms, intraCsms, singlePsms, loopPsms, deadendPsms, taskId, xlSearchParameters); - MyTaskResults.AddTaskSummaryText("Target inter-crosslinks within 1% FDR: " + interCsms.Count(p => p.FdrInfo.QValue <= 0.01 && !p.IsDecoy && !p.BetaPeptide.IsDecoy)); - MyTaskResults.AddTaskSummaryText("Target intra-crosslinks within 1% FDR: " + intraCsms.Count(p => p.FdrInfo.QValue <= 0.01 && !p.IsDecoy && !p.BetaPeptide.IsDecoy)); - MyTaskResults.AddTaskSummaryText("Target single peptides within 1% FDR: " + singlePsms.Count(p => p.FdrInfo.QValue <= 0.01 && !p.IsDecoy)); - MyTaskResults.AddTaskSummaryText("Target loop-linked peptides within 1% FDR: " + loopPsms.Count(p => p.FdrInfo.QValue <= 0.01 && !p.IsDecoy)); - MyTaskResults.AddTaskSummaryText("Target deadend peptides within 1% FDR: " + deadendPsms.Count(p => p.FdrInfo.QValue <= 0.01 && !p.IsDecoy)); + MyTaskResults.AddTaskSummaryText("Target inter-crosslinks within 1% FDR: " + interCsms.Count(p => p.PsmFdrInfo.QValue <= 0.01 && !p.IsDecoy && !p.BetaPeptide.IsDecoy)); + MyTaskResults.AddTaskSummaryText("Target intra-crosslinks within 1% FDR: " + intraCsms.Count(p => p.PsmFdrInfo.QValue <= 0.01 && !p.IsDecoy && !p.BetaPeptide.IsDecoy)); + MyTaskResults.AddTaskSummaryText("Target single peptides within 1% FDR: " + singlePsms.Count(p => p.PsmFdrInfo.QValue <= 0.01 && !p.IsDecoy)); + MyTaskResults.AddTaskSummaryText("Target loop-linked peptides within 1% FDR: " + loopPsms.Count(p => p.PsmFdrInfo.QValue <= 0.01 && !p.IsDecoy)); + MyTaskResults.AddTaskSummaryText("Target deadend peptides within 1% FDR: " + deadendPsms.Count(p => p.PsmFdrInfo.QValue <= 0.01 && !p.IsDecoy)); // write pepXML if (xlSearchParameters.WritePepXml) { List writeToXml = new List(); - writeToXml.AddRange(intraCsms.Where(p => !p.IsDecoy && !p.BetaPeptide.IsDecoy && p.FdrInfo.QValue <= 0.05)); - writeToXml.AddRange(interCsms.Where(p => !p.IsDecoy && !p.BetaPeptide.IsDecoy && p.FdrInfo.QValue <= 0.05)); - writeToXml.AddRange(singlePsms.Where(p => !p.IsDecoy && p.FdrInfo.QValue <= 0.05)); - writeToXml.AddRange(loopPsms.Where(p => !p.IsDecoy && p.FdrInfo.QValue <= 0.05)); - writeToXml.AddRange(deadendPsms.Where(p => !p.IsDecoy && p.FdrInfo.QValue <= 0.05)); + writeToXml.AddRange(intraCsms.Where(p => !p.IsDecoy && !p.BetaPeptide.IsDecoy && p.PsmFdrInfo.QValue <= 0.05)); + writeToXml.AddRange(interCsms.Where(p => !p.IsDecoy && !p.BetaPeptide.IsDecoy && p.PsmFdrInfo.QValue <= 0.05)); + writeToXml.AddRange(singlePsms.Where(p => !p.IsDecoy && p.PsmFdrInfo.QValue <= 0.05)); + writeToXml.AddRange(loopPsms.Where(p => !p.IsDecoy && p.PsmFdrInfo.QValue <= 0.05)); + writeToXml.AddRange(deadendPsms.Where(p => !p.IsDecoy && p.PsmFdrInfo.QValue <= 0.05)); writeToXml = writeToXml.OrderBy(p => p.ScanNumber).ToList(); foreach (var fullFilePath in currentRawFileList) @@ -106,11 +106,11 @@ public List RunSingleFile(string outputFolder, string ta WriteCsvFiles(outputFolder, interCsms, intraCsms, singlePsms, loopPsms, deadendPsms, taskId, xlSearchParameters); - allFilterPsms.AddRange(interCsms.Where(p=>p.FdrInfo.QValue <= 0.2)); - allFilterPsms.AddRange(intraCsms.Where(p => p.FdrInfo.QValue <= 0.2)); - allFilterPsms.AddRange(singlePsms.Where(p => p.FdrInfo.QValue <= 0.2)); - allFilterPsms.AddRange(loopPsms.Where(p => p.FdrInfo.QValue <= 0.2)); - allFilterPsms.AddRange(deadendPsms.Where(p => p.FdrInfo.QValue <= 0.2)); + allFilterPsms.AddRange(interCsms.Where(p=>p.PsmFdrInfo.QValue <= 0.2)); + allFilterPsms.AddRange(intraCsms.Where(p => p.PsmFdrInfo.QValue <= 0.2)); + allFilterPsms.AddRange(singlePsms.Where(p => p.PsmFdrInfo.QValue <= 0.2)); + allFilterPsms.AddRange(loopPsms.Where(p => p.PsmFdrInfo.QValue <= 0.2)); + allFilterPsms.AddRange(deadendPsms.Where(p => p.PsmFdrInfo.QValue <= 0.2)); return allFilterPsms; } @@ -228,8 +228,8 @@ private void DoCrosslinkFdrAnalysis(List csms) double qValue = Math.Min(1, (double)cumulativeDecoy / (cumulativeTarget <= 0 ? 1 : cumulativeTarget)); double qValueNotch = 0; //maybe we should assign this some day? - double pep = csm.FdrInfo == null ? double.NaN : csm.FdrInfo.PEP; - double pepQValue = csm.FdrInfo == null ? double.NaN : csm.FdrInfo.PEP_QValue; + double pep = csm.PsmFdrInfo == null ? double.NaN : csm.PsmFdrInfo.PEP; + double pepQValue = csm.PsmFdrInfo == null ? double.NaN : csm.PsmFdrInfo.PEP_QValue; csm.SetFdrValues(cumulativeTarget, cumulativeDecoy, qValue, 0, 0, qValueNotch, pep, pepQValue); } @@ -247,7 +247,7 @@ private void WriteXlSpectralLibrary( foreach (var csmGroups in linkedCsms.Where(c => !c.IsDecoy - && c.FdrInfo.QValueNotch < 0.05) + && c.PsmFdrInfo.QValueNotch < 0.05) .GroupBy(c => (c.UniqueSequence, c.ScanPrecursorCharge))) { CrosslinkSpectralMatch bestCsm = csmGroups.MaxBy(c => c.XLTotalScore); @@ -263,7 +263,7 @@ private void WriteXlSpectralLibrary( foreach (var singlePsmGroup in singlePsms.Where(c => !c.IsDecoy - && c.FdrInfo.QValueNotch < 0.05) + && c.PsmFdrInfo.QValueNotch < 0.05) .GroupBy(c => (c.FullSequence, c.ScanPrecursorCharge))) { CrosslinkSpectralMatch bestPsm = singlePsmGroup.MaxBy(c => c.Score); diff --git a/MetaMorpheus/TaskLayer/XLSearchTask/WriteXlFile.cs b/MetaMorpheus/TaskLayer/XLSearchTask/WriteXlFile.cs index 65800667a..cbd5c1840 100644 --- a/MetaMorpheus/TaskLayer/XLSearchTask/WriteXlFile.cs +++ b/MetaMorpheus/TaskLayer/XLSearchTask/WriteXlFile.cs @@ -232,7 +232,7 @@ public static void WritePepXML_xl(List items, List items, List items, List items, List p != null)) { - var b = item.FdrInfo.PEP; + var b = item.PsmFdrInfo.PEP; if (b >= 0.5) { trueCount++; @@ -302,7 +302,7 @@ public static void TestComputePEPValue() foreach (var item in psmCopyForCZETest.Where(p => p != null)) { - var b = item.FdrInfo.PEP; + var b = item.PsmFdrInfo.PEP; if (b >= 0.5) { trueCount++; diff --git a/MetaMorpheus/Test/MultiProteaseParsimonyTest.cs b/MetaMorpheus/Test/MultiProteaseParsimonyTest.cs index 2cea5a6f0..a29d1660b 100644 --- a/MetaMorpheus/Test/MultiProteaseParsimonyTest.cs +++ b/MetaMorpheus/Test/MultiProteaseParsimonyTest.cs @@ -710,7 +710,7 @@ public static void TestPSMFdrFiltering_Simulated() var fdrFilteredPsms = new List(); foreach (SpectralMatch psm in psms) { - if (psm != null && psm.FdrInfo.QValue <= 0.0100 && psm.FdrInfo.QValueNotch <= 0.0100) + if (psm != null && psm.PsmFdrInfo.QValue <= 0.0100 && psm.PsmFdrInfo.QValueNotch <= 0.0100) { fdrFilteredPsms.Add(psm); } @@ -1032,15 +1032,15 @@ public static void MultiProteaseParsimony_TestingProteaseSpecificFDRCalculations new FdrAnalysisEngine(psms, 0, new CommonParameters(), fsp, new List()).Run(); psms = psms.OrderByDescending(p => p.Score).ToList(); - Assert.AreEqual(0.00, Math.Round(psms[1].FdrInfo.QValue, 2)); - Assert.AreEqual(0.00, Math.Round(psms[2].FdrInfo.QValue, 2)); - Assert.AreEqual(0.00, Math.Round(psms[3].FdrInfo.QValue, 2)); - Assert.AreEqual(0.5, Math.Round(psms[4].FdrInfo.QValue, 2)); - Assert.AreEqual(0.33, Math.Round(psms[5].FdrInfo.QValue, 2)); - Assert.AreEqual(0.00, Math.Round(psms[6].FdrInfo.QValue, 2)); - Assert.AreEqual(0.33, Math.Round(psms[7].FdrInfo.QValue, 2)); - Assert.AreEqual(0.67, Math.Round(psms[8].FdrInfo.QValue, 2)); - Assert.AreEqual(0.5, Math.Round(psms[9].FdrInfo.QValue, 2)); + Assert.AreEqual(0.00, Math.Round(psms[1].PsmFdrInfo.QValue, 2)); + Assert.AreEqual(0.00, Math.Round(psms[2].PsmFdrInfo.QValue, 2)); + Assert.AreEqual(0.00, Math.Round(psms[3].PsmFdrInfo.QValue, 2)); + Assert.AreEqual(0.5, Math.Round(psms[4].PsmFdrInfo.QValue, 2)); + Assert.AreEqual(0.33, Math.Round(psms[5].PsmFdrInfo.QValue, 2)); + Assert.AreEqual(0.00, Math.Round(psms[6].PsmFdrInfo.QValue, 2)); + Assert.AreEqual(0.33, Math.Round(psms[7].PsmFdrInfo.QValue, 2)); + Assert.AreEqual(0.67, Math.Round(psms[8].PsmFdrInfo.QValue, 2)); + Assert.AreEqual(0.5, Math.Round(psms[9].PsmFdrInfo.QValue, 2)); } } } \ No newline at end of file diff --git a/MetaMorpheus/Test/PostSearchAnalysisTaskTests.cs b/MetaMorpheus/Test/PostSearchAnalysisTaskTests.cs index 140903d0a..d1bb1bc7b 100644 --- a/MetaMorpheus/Test/PostSearchAnalysisTaskTests.cs +++ b/MetaMorpheus/Test/PostSearchAnalysisTaskTests.cs @@ -1,5 +1,6 @@ using System.Collections.Generic; using System.IO; +using EngineLayer; using Nett; using NUnit.Framework; using TaskLayer; @@ -19,6 +20,11 @@ public static void AllResultsAndResultsTxtTests() string myFile1 = Path.Combine(TestContext.CurrentContext.TestDirectory, @"TestData\TaGe_SA_A549_3_snip.mzML"); string myFile2 = Path.Combine(TestContext.CurrentContext.TestDirectory, @"TestData\TaGe_SA_A549_3_snip_2.mzML"); string myDatabase = Path.Combine(TestContext.CurrentContext.TestDirectory, @"TestData\TaGe_SA_A549_3_snip.fasta"); + ExperimentalDesign.WriteExperimentalDesignToFile( new List + { + new FlashLFQ.SpectraFileInfo(myFile1, "A", 0, 0, 0), + new FlashLFQ.SpectraFileInfo(myFile2, "A", 1, 0, 0) + }); EverythingRunnerEngine engineToml = new(new List<(string, MetaMorpheusTask)> { ("postSearchAnalysisTaskTestOutput", searchTaskLoaded) }, new List { myFile1, myFile2 }, new List { new DbForTask(myDatabase, false) }, outputFolder); engineToml.Run(); @@ -60,6 +66,9 @@ public static void AllResultsAndResultsTxtTests() Assert.AreEqual("TaGe_SA_A549_3_snip_2 target PSMs with q-value = 0.01: " + TaGe_SA_A549_3_snip_2ExpectedPsms, results[17]); Assert.AreEqual("TaGe_SA_A549_3_snip_2 Target peptides with q-value = 0.01 : " + TaGe_SA_A549_3_snip_2ExpectedPeptides, results[23]); + string peptideQuantFile = Path.Combine(outputFolder, "postSearchAnalysisTaskTestOutput", "AllQuantifiedPeptides.tsv"); + string[] peptideQuant = File.ReadAllLines(peptideQuantFile); + Assert.AreEqual(175, peptideQuantFile.Length); // 174 peptides + header. Make sure that we're quantifying only those peptides with q-value <= 0.01 Directory.Delete(outputFolder, true); diff --git a/MetaMorpheus/Test/SearchEngineTests.cs b/MetaMorpheus/Test/SearchEngineTests.cs index 3e439b4ec..45e16431f 100644 --- a/MetaMorpheus/Test/SearchEngineTests.cs +++ b/MetaMorpheus/Test/SearchEngineTests.cs @@ -552,13 +552,13 @@ public static void TestModernSearchEngineLowResOneRealSpectrum() // Single search mode Assert.AreEqual(12, allPsmsArray.Length); - var goodPsm = nonNullPsms.Where(p => p.FdrInfo.QValue <= 0.01).ToList(); + var goodPsm = nonNullPsms.Where(p => p.PsmFdrInfo.QValue <= 0.01).ToList(); var myMatchedIons = goodPsm[0].MatchedFragmentIons; Assert.AreEqual(47, myMatchedIons.Count()); - var goodScore = nonNullPsms.Where(p => p.FdrInfo.QValue <= 0.01).Select(s => s.Score).ToList(); + var goodScore = nonNullPsms.Where(p => p.PsmFdrInfo.QValue <= 0.01).Select(s => s.Score).ToList(); goodScore.Sort(); Assert.AreEqual(2, goodPsm.Count()); } @@ -655,7 +655,7 @@ public static void TestClassicSearchEngineLowResSimple() // Single search mode Assert.AreEqual(535, allPsmsArray.Length); - var goodScore = nonNullPsms.Where(p => p.FdrInfo.QValue <= 0.01).Select(s => s.Score).ToList(); + var goodScore = nonNullPsms.Where(p => p.PsmFdrInfo.QValue <= 0.01).Select(s => s.Score).ToList(); Assert.AreEqual(181, goodScore.Count()); Directory.Delete(outputFolder, true); @@ -747,7 +747,7 @@ public static void TestModernSearchEngineLowResSimple() // Single search mode Assert.AreEqual(535, allPsmsArray.Length); - var goodScore = nonNullPsms.Where(p => p.FdrInfo.QValue <= 0.01).Select(s => s.Score).ToList(); + var goodScore = nonNullPsms.Where(p => p.PsmFdrInfo.QValue <= 0.01).Select(s => s.Score).ToList(); Assert.AreEqual(181, goodScore.Count()); } diff --git a/MetaMorpheus/Test/SpectralRecoveryTest.cs b/MetaMorpheus/Test/SpectralRecoveryTest.cs index e46ca8994..c758ac880 100644 --- a/MetaMorpheus/Test/SpectralRecoveryTest.cs +++ b/MetaMorpheus/Test/SpectralRecoveryTest.cs @@ -146,7 +146,7 @@ public static void SpectralRecoveryPostSearchAnalysisTest() List expectedMatches = mbrPsms.Select(p => p.BaseSeq).Intersect(expectedMbrPsms.Select(p => p.BaseSeq).ToList()).ToList(); Assert.That(matches2ng.Count >= 2); - Assert.That(matches02ng.Count >= 8); + Assert.That(matches02ng.Count >= 7); // AS: 6/17/24 - Fewer quantification events (8 -> 7) because one of the peptides doesn't clear peptide level FDR threshold Assert.That(expectedMatches.Count >= 3); // FlashLFQ doesn't find all 6 expected peaks, only 3. MbrAnalysis finds these three peaks //TODO: Add test for recovering fdrInfo from original. Currently, PsmTsvReader doesn't support the new columns, so it's hard to test diff --git a/MetaMorpheus/Test/TestData/Task1-SearchTaskconfig.toml b/MetaMorpheus/Test/TestData/Task1-SearchTaskconfig.toml index 1b1b8022c..90806b529 100644 --- a/MetaMorpheus/Test/TestData/Task1-SearchTaskconfig.toml +++ b/MetaMorpheus/Test/TestData/Task1-SearchTaskconfig.toml @@ -15,7 +15,7 @@ MassDiffAcceptorType = "Exact" WritePrunedDatabase = false KeepAllUniprotMods = true DoLocalizationAnalysis = true -DoQuantification = false +DoQuantification = true SearchType = "Classic" LocalFdrCategories = ["FullySpecific"] MaxFragmentSize = 30000.0 diff --git a/MetaMorpheus/Test/TestPsm.cs b/MetaMorpheus/Test/TestPsm.cs index 7a0f63966..b2e28b393 100644 --- a/MetaMorpheus/Test/TestPsm.cs +++ b/MetaMorpheus/Test/TestPsm.cs @@ -305,7 +305,7 @@ public static void TestPsmMatchingToTargetAndDecoyWithDifferentSequences() List<(string fileName, CommonParameters fileSpecificParameters)> fsp = new List<(string fileName, CommonParameters fileSpecificParameters)> { ("filename", commonParameters) }; new FdrAnalysisEngine(new List { psm }, 1, new CommonParameters(), fsp, new List()).Run(); - Assert.AreEqual(0.5, psm.FdrInfo.CumulativeDecoy); + Assert.AreEqual(0.5, psm.PsmFdrInfo.CumulativeDecoy); } [Test] diff --git a/MetaMorpheus/Test/XLTest.cs b/MetaMorpheus/Test/XLTest.cs index b30e5f40e..0de41f4c2 100644 --- a/MetaMorpheus/Test/XLTest.cs +++ b/MetaMorpheus/Test/XLTest.cs @@ -543,7 +543,7 @@ public static void XlTest_MoreComprehensive() deadendNH2 = 0; deadendTris = 0; - foreach (CrosslinkSpectralMatch csm in firstCsmsFromListsOfCsms.Where(c => c.FdrInfo.QValue <= 0.01).ToList()) + foreach (CrosslinkSpectralMatch csm in firstCsmsFromListsOfCsms.Where(c => c.PsmFdrInfo.QValue <= 0.01).ToList()) { switch (csm.CrossType) { @@ -728,7 +728,7 @@ public static void XlTest_MoreComprehensive() deadendNH2 = 0; deadendTris = 0; - foreach (CrosslinkSpectralMatch csm in firstCsmsFromListsOfCsms.Where(c => c.FdrInfo.QValue <= 0.01).ToList()) + foreach (CrosslinkSpectralMatch csm in firstCsmsFromListsOfCsms.Where(c => c.PsmFdrInfo.QValue <= 0.01).ToList()) { switch (csm.CrossType) { From 5dd1ce936c11564974d4bfc75481c337a7b1b74c Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 18 Jun 2024 10:23:28 -0500 Subject: [PATCH 2/7] Fixed failing test --- MetaMorpheus/Test/PostSearchAnalysisTaskTests.cs | 2 +- MetaMorpheus/Test/Test.csproj | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/MetaMorpheus/Test/PostSearchAnalysisTaskTests.cs b/MetaMorpheus/Test/PostSearchAnalysisTaskTests.cs index d1bb1bc7b..a810125ac 100644 --- a/MetaMorpheus/Test/PostSearchAnalysisTaskTests.cs +++ b/MetaMorpheus/Test/PostSearchAnalysisTaskTests.cs @@ -68,7 +68,7 @@ public static void AllResultsAndResultsTxtTests() string peptideQuantFile = Path.Combine(outputFolder, "postSearchAnalysisTaskTestOutput", "AllQuantifiedPeptides.tsv"); string[] peptideQuant = File.ReadAllLines(peptideQuantFile); - Assert.AreEqual(175, peptideQuantFile.Length); // 174 peptides + header. Make sure that we're quantifying only those peptides with q-value <= 0.01 + Assert.AreEqual(175, peptideQuant.Length); // 174 peptides + header. Make sure that we're quantifying only those peptides with q-value <= 0.01 Directory.Delete(outputFolder, true); diff --git a/MetaMorpheus/Test/Test.csproj b/MetaMorpheus/Test/Test.csproj index 6c6f355cd..5bbe63cb6 100644 --- a/MetaMorpheus/Test/Test.csproj +++ b/MetaMorpheus/Test/Test.csproj @@ -303,6 +303,12 @@ Always + + Always + + + Always + Always From 716cd407658fc6a7cb60a668f1cdb121b56ae8c1 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 18 Jun 2024 10:35:08 -0500 Subject: [PATCH 3/7] Added OnlyQuantifyConfidentPeptides toggle --- .../GUI/TaskWindows/SearchTaskWindow.xaml | 18 ++++++++++++++++++ .../GUI/TaskWindows/SearchTaskWindow.xaml.cs | 2 ++ .../SearchTask/PostSearchAnalysisTask.cs | 4 +++- .../TaskLayer/SearchTask/SearchParameters.cs | 2 ++ 4 files changed, 25 insertions(+), 1 deletion(-) diff --git a/MetaMorpheus/GUI/TaskWindows/SearchTaskWindow.xaml b/MetaMorpheus/GUI/TaskWindows/SearchTaskWindow.xaml index 9f3205c57..cb1a18934 100644 --- a/MetaMorpheus/GUI/TaskWindows/SearchTaskWindow.xaml +++ b/MetaMorpheus/GUI/TaskWindows/SearchTaskWindow.xaml @@ -544,6 +544,24 @@ + + + + + + + + + + + + + + + + + + diff --git a/MetaMorpheus/GUI/TaskWindows/SearchTaskWindow.xaml.cs b/MetaMorpheus/GUI/TaskWindows/SearchTaskWindow.xaml.cs index 678e1392c..9bacbf2ad 100644 --- a/MetaMorpheus/GUI/TaskWindows/SearchTaskWindow.xaml.cs +++ b/MetaMorpheus/GUI/TaskWindows/SearchTaskWindow.xaml.cs @@ -190,6 +190,7 @@ private void UpdateFieldsFromTask(SearchTask task) CheckBoxNoQuant.IsChecked = !task.SearchParameters.DoLabelFreeQuantification; CheckBoxLFQ.IsChecked = task.SearchParameters.DoLabelFreeQuantification; CheckBoxUseSharedPeptidesForQuant.IsChecked = task.SearchParameters.UseSharedPeptidesForLFQ; + CheckBoxOnlyQuantifyConfidentPeptides.IsChecked = task.SearchParameters.OnlyQuantifyConfidentPeptides; CheckBoxMultiplex.IsChecked = task.SearchParameters.DoMultiplexQuantification; MultiplexComboBox.SelectedItem = task.SearchParameters.MultiplexModId ?? _defaultMultiplexType; // If Spectral Recovery is enabled @@ -628,6 +629,7 @@ private void SaveButton_Click(object sender, RoutedEventArgs e) TheTask.SearchParameters.NoOneHitWonders = CheckBoxNoOneHitWonders.IsChecked.Value; TheTask.SearchParameters.DoLabelFreeQuantification = !CheckBoxNoQuant.IsChecked.Value; TheTask.SearchParameters.UseSharedPeptidesForLFQ = CheckBoxUseSharedPeptidesForQuant.IsChecked.Value; + TheTask.SearchParameters.OnlyQuantifyConfidentPeptides = CheckBoxOnlyQuantifyConfidentPeptides.IsChecked.Value; TheTask.SearchParameters.DoSpectralRecovery = CheckBoxLFQwSpectralRecovery.IsChecked.Value; TheTask.SearchParameters.DoMultiplexQuantification = CheckBoxMultiplex.IsChecked.Value; TheTask.SearchParameters.MultiplexModId = (string)MultiplexComboBox.SelectedItem; diff --git a/MetaMorpheus/TaskLayer/SearchTask/PostSearchAnalysisTask.cs b/MetaMorpheus/TaskLayer/SearchTask/PostSearchAnalysisTask.cs index 5eb8197ad..ef90e7bc7 100644 --- a/MetaMorpheus/TaskLayer/SearchTask/PostSearchAnalysisTask.cs +++ b/MetaMorpheus/TaskLayer/SearchTask/PostSearchAnalysisTask.cs @@ -629,7 +629,9 @@ private void QuantificationAnalysis() useSharedPeptidesForProteinQuant: Parameters.SearchParameters.UseSharedPeptidesForLFQ, silent: true, maxThreads: CommonParameters.MaxThreadsToUsePerFile, - peptideSequencesToUse: Parameters.SearchParameters.SilacLabels == null ? _filteredPeptides.Select(peptide => peptide.FullSequence).ToList() : null); + peptideSequencesToUse: (Parameters.SearchParameters.SilacLabels == null && Parameters.SearchParameters.OnlyQuantifyConfidentPeptides) + ? _filteredPeptides.Select(peptide => peptide.FullSequence).ToList() + : null); if (flashLFQIdentifications.Any()) { diff --git a/MetaMorpheus/TaskLayer/SearchTask/SearchParameters.cs b/MetaMorpheus/TaskLayer/SearchTask/SearchParameters.cs index 080ee2c6d..34b113bcd 100644 --- a/MetaMorpheus/TaskLayer/SearchTask/SearchParameters.cs +++ b/MetaMorpheus/TaskLayer/SearchTask/SearchParameters.cs @@ -17,6 +17,7 @@ public SearchParameters() ModPeptidesAreDifferent = false; DoLabelFreeQuantification = true; UseSharedPeptidesForLFQ = false; + OnlyQuantifyConfidentPeptides = true; DoSpectralRecovery = false; QuantifyPpmTol = 5; SearchTarget = true; @@ -78,6 +79,7 @@ public SearchParameters() public bool DoLocalizationAnalysis { get; set; } public bool DoLabelFreeQuantification { get; set; } public bool UseSharedPeptidesForLFQ { get; set; } + public bool OnlyQuantifyConfidentPeptides { get; set; } public bool DoMultiplexQuantification { get; set; } public string MultiplexModId { get; set; } public bool DoSpectralRecovery { get; set; } From e2437c9f2f025843128896d94c346129b9464741 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 18 Jun 2024 11:21:43 -0500 Subject: [PATCH 4/7] Reimplemented FdrInfo field, which now points to PsmFdrInfo --- .../GlycoSearch/GlycoSpectralMatch.cs | 2 +- MetaMorpheus/EngineLayer/Gptmd/GptmdEngine.cs | 2 +- .../ModificationAnalysisEngine.cs | 2 +- .../NonSpecificEnzymeSearchEngine.cs | 16 ++++---- .../ProteinParsimonyEngine.cs | 4 +- .../ProteinScoringAndFdrEngine.cs | 4 +- MetaMorpheus/EngineLayer/SpectralMatch.cs | 2 + .../CalibrationTask/CalibrationTask.cs | 2 +- .../GlycoSearchTask/GlycoProteinParsimony.cs | 4 +- .../PostGlycoSearchAnalysisTask.cs | 8 ++-- .../MbrAnalysis/SpectralRecoveryRunner.cs | 22 +++++------ MetaMorpheus/TaskLayer/PepXMLWriter.cs | 2 +- .../TaskLayer/SearchTask/MzIdentMLWriter.cs | 4 +- .../XLSearchTask/PostXLSearchAnalysisTask.cs | 38 +++++++++---------- .../TaskLayer/XLSearchTask/WriteXlFile.cs | 8 ++-- MetaMorpheus/Test/XLTest.cs | 3 +- 16 files changed, 63 insertions(+), 60 deletions(-) diff --git a/MetaMorpheus/EngineLayer/GlycoSearch/GlycoSpectralMatch.cs b/MetaMorpheus/EngineLayer/GlycoSearch/GlycoSpectralMatch.cs index ae2c82b55..acc6db3be 100644 --- a/MetaMorpheus/EngineLayer/GlycoSearch/GlycoSpectralMatch.cs +++ b/MetaMorpheus/EngineLayer/GlycoSearch/GlycoSpectralMatch.cs @@ -248,7 +248,7 @@ public string SingleToString() sb.Append((IsDecoy) ? "D" : (IsContaminant) ? "C" : "T"); sb.Append("\t"); - sb.Append(PsmFdrInfo!=null? PsmFdrInfo.QValue.ToString() : "-1" ); sb.Append("\t"); + sb.Append(FdrInfo!=null? FdrInfo.QValue.ToString() : "-1" ); sb.Append("\t"); sb.Append("0" + "\t"); //This is space for PEP diff --git a/MetaMorpheus/EngineLayer/Gptmd/GptmdEngine.cs b/MetaMorpheus/EngineLayer/Gptmd/GptmdEngine.cs index c34782d99..8c62576a6 100644 --- a/MetaMorpheus/EngineLayer/Gptmd/GptmdEngine.cs +++ b/MetaMorpheus/EngineLayer/Gptmd/GptmdEngine.cs @@ -70,7 +70,7 @@ protected override MetaMorpheusEngineResults RunSpecific() int modsAdded = 0; //foreach peptide in each psm and for each modification that matches the notch, //add that modification to every allowed residue - foreach (var psm in AllIdentifications.Where(b => b.PsmFdrInfo.QValueNotch <= 0.05 && !b.IsDecoy)) + foreach (var psm in AllIdentifications.Where(b => b.FdrInfo.QValueNotch <= 0.05 && !b.IsDecoy)) { // get file-specific precursor tolerance Tolerance precursorMassTolerance = FilePathToPrecursorMassTolerance[psm.FullFilePath]; diff --git a/MetaMorpheus/EngineLayer/ModificationAnalysis/ModificationAnalysisEngine.cs b/MetaMorpheus/EngineLayer/ModificationAnalysis/ModificationAnalysisEngine.cs index 037104188..44350e1cb 100644 --- a/MetaMorpheus/EngineLayer/ModificationAnalysis/ModificationAnalysisEngine.cs +++ b/MetaMorpheus/EngineLayer/ModificationAnalysis/ModificationAnalysisEngine.cs @@ -19,7 +19,7 @@ protected override MetaMorpheusEngineResults RunSpecific() ModificationAnalysisResults myAnalysisResults = new ModificationAnalysisResults(this); - var confidentTargetPsms = NewPsms.Where(b => b.PsmFdrInfo.QValue <= 0.01 && !b.IsDecoy).ToList(); + var confidentTargetPsms = NewPsms.Where(b => b.FdrInfo.QValue <= 0.01 && !b.IsDecoy).ToList(); // For the database ones, only need un-ambiguous protein and location in protein var forObserved = confidentTargetPsms diff --git a/MetaMorpheus/EngineLayer/NonSpecificEnzymeSearch/NonSpecificEnzymeSearchEngine.cs b/MetaMorpheus/EngineLayer/NonSpecificEnzymeSearch/NonSpecificEnzymeSearchEngine.cs index acd6caa28..1426bd4aa 100644 --- a/MetaMorpheus/EngineLayer/NonSpecificEnzymeSearch/NonSpecificEnzymeSearchEngine.cs +++ b/MetaMorpheus/EngineLayer/NonSpecificEnzymeSearch/NonSpecificEnzymeSearchEngine.cs @@ -460,7 +460,7 @@ public static List ResolveFdrCategorySpecificPsms(List ResolveFdrCategorySpecificPsms(List x != null).Count(x => x.PsmFdrInfo.QValue <= 0.01); //set ranking as number of psms above 1% FDR + ranking[i] = AllPsms[i].Where(x => x != null).Count(x => x.FdrInfo.QValue <= 0.01); //set ranking as number of psms above 1% FDR indexesOfInterest.Add(i); } } @@ -515,9 +515,9 @@ public static List ResolveFdrCategorySpecificPsms(List minorPsm.PsmFdrInfo.QValue) + if (majorPsm.FdrInfo.QValue > minorPsm.FdrInfo.QValue) { - minorPsm.PsmFdrInfo.QValue = majorPsm.PsmFdrInfo.QValue; + minorPsm.FdrInfo.QValue = majorPsm.FdrInfo.QValue; } minorPsmIndex++; } @@ -527,9 +527,9 @@ public static List ResolveFdrCategorySpecificPsms(List minorPsm.PsmFdrInfo.QValue) + if (majorPsm.FdrInfo.QValue > minorPsm.FdrInfo.QValue) { - minorPsm.PsmFdrInfo.QValue = majorPsm.PsmFdrInfo.QValue; + minorPsm.FdrInfo.QValue = majorPsm.FdrInfo.QValue; } minorPsmIndex++; } @@ -548,7 +548,7 @@ public static List ResolveFdrCategorySpecificPsms(List bestPsm.Score)) { @@ -587,7 +587,7 @@ public static List ResolveFdrCategorySpecificPsms(List b.PsmFdrInfo.QValue).ThenByDescending(b => b.Score).ToList(); + return bestPsmsList.OrderBy(b => b.FdrInfo.QValue).ThenByDescending(b => b.Score).ToList(); } public static List GetVariableTerminalMods(FragmentationTerminus fragmentationTerminus, List variableModifications) diff --git a/MetaMorpheus/EngineLayer/ProteinParsimony/ProteinParsimonyEngine.cs b/MetaMorpheus/EngineLayer/ProteinParsimony/ProteinParsimonyEngine.cs index 081aab800..d57cc666b 100644 --- a/MetaMorpheus/EngineLayer/ProteinParsimony/ProteinParsimonyEngine.cs +++ b/MetaMorpheus/EngineLayer/ProteinParsimony/ProteinParsimonyEngine.cs @@ -41,11 +41,11 @@ public ProteinParsimonyEngine(List allPsms, bool modPeptidesAreDi // KEEP contaminants for use in parsimony! if (modPeptidesAreDifferent) { - _fdrFilteredPsms = allPsms.Where(p => p.FullSequence != null && p.PsmFdrInfo.QValue <= FdrCutoffForParsimony && p.PsmFdrInfo.QValueNotch <= FdrCutoffForParsimony).ToList(); + _fdrFilteredPsms = allPsms.Where(p => p.FullSequence != null && p.FdrInfo.QValue <= FdrCutoffForParsimony && p.FdrInfo.QValueNotch <= FdrCutoffForParsimony).ToList(); } else { - _fdrFilteredPsms = allPsms.Where(p => p.BaseSequence != null && p.PsmFdrInfo.QValue <= FdrCutoffForParsimony && p.PsmFdrInfo.QValueNotch <= FdrCutoffForParsimony).ToList(); + _fdrFilteredPsms = allPsms.Where(p => p.BaseSequence != null && p.FdrInfo.QValue <= FdrCutoffForParsimony && p.FdrInfo.QValueNotch <= FdrCutoffForParsimony).ToList(); } // peptides to use in parsimony = peptides observed in high-confidence PSMs (including decoys) diff --git a/MetaMorpheus/EngineLayer/ProteinScoringAndFdr/ProteinScoringAndFdrEngine.cs b/MetaMorpheus/EngineLayer/ProteinScoringAndFdr/ProteinScoringAndFdrEngine.cs index 16ac6cadb..d1459cc87 100644 --- a/MetaMorpheus/EngineLayer/ProteinScoringAndFdr/ProteinScoringAndFdrEngine.cs +++ b/MetaMorpheus/EngineLayer/ProteinScoringAndFdr/ProteinScoringAndFdrEngine.cs @@ -42,7 +42,7 @@ private void ScoreProteinGroups(List proteinGroups, IEnumerable>(); foreach (var psm in psmList) { - if (psm.PsmFdrInfo.QValueNotch <= 0.01 && psm.PsmFdrInfo.QValue <= 0.01) + if (psm.FdrInfo.QValueNotch <= 0.01 && psm.FdrInfo.QValue <= 0.01) { if ((TreatModPeptidesAsDifferentPeptides && psm.FullSequence != null) || (!TreatModPeptidesAsDifferentPeptides && psm.BaseSequence != null)) { @@ -155,7 +155,7 @@ private List DoProteinFdr(List proteinGroups) } pg.BestPeptideScore = pg.AllPsmsBelowOnePercentFDR.Max(psm => psm.Score); - pg.BestPeptideQValue = pg.AllPsmsBelowOnePercentFDR.Min(psm => psm.PsmFdrInfo.QValueNotch); + pg.BestPeptideQValue = pg.AllPsmsBelowOnePercentFDR.Min(psm => psm.FdrInfo.QValueNotch); } // pick the best notch-QValue for each paired accession diff --git a/MetaMorpheus/EngineLayer/SpectralMatch.cs b/MetaMorpheus/EngineLayer/SpectralMatch.cs index 9fe2844ba..41b2545bd 100644 --- a/MetaMorpheus/EngineLayer/SpectralMatch.cs +++ b/MetaMorpheus/EngineLayer/SpectralMatch.cs @@ -71,6 +71,8 @@ protected SpectralMatch(IBioPolymerWithSetMods peptide, int notch, double score, public string FullFilePath { get; private set; } public int ScanIndex { get; } public int NumDifferentMatchingPeptides { get { return _BestMatchingBioPolymersWithSetMods.Count; } } + + public FdrInfo FdrInfo => PsmFdrInfo; public FdrInfo PsmFdrInfo { get; private set; } public FdrInfo PeptideFdrInfo { get; private set; } public PsmData PsmData_forPEPandPercolator { get; set; } diff --git a/MetaMorpheus/TaskLayer/CalibrationTask/CalibrationTask.cs b/MetaMorpheus/TaskLayer/CalibrationTask/CalibrationTask.cs index 8a7320f84..82b2762a1 100644 --- a/MetaMorpheus/TaskLayer/CalibrationTask/CalibrationTask.cs +++ b/MetaMorpheus/TaskLayer/CalibrationTask/CalibrationTask.cs @@ -278,7 +278,7 @@ private DataPointAquisitionResults GetDataAcquisitionResults(MsDataFile myMsData _ = new FdrAnalysisEngine(allPsms, searchMode.NumNotches, CommonParameters, FileSpecificParameters, new List { taskId, "Individual Spectra Files", fileNameWithoutExtension }, doPEP: false).Run(); - List goodIdentifications = allPsms.Where(b => b.PsmFdrInfo.QValueNotch < 0.001 && !b.IsDecoy && b.FullSequence != null).ToList(); + List goodIdentifications = allPsms.Where(b => b.FdrInfo.QValueNotch < 0.001 && !b.IsDecoy && b.FullSequence != null).ToList(); if (!goodIdentifications.Any()) { diff --git a/MetaMorpheus/TaskLayer/GlycoSearchTask/GlycoProteinParsimony.cs b/MetaMorpheus/TaskLayer/GlycoSearchTask/GlycoProteinParsimony.cs index 1cc247464..9d90d65d5 100644 --- a/MetaMorpheus/TaskLayer/GlycoSearchTask/GlycoProteinParsimony.cs +++ b/MetaMorpheus/TaskLayer/GlycoSearchTask/GlycoProteinParsimony.cs @@ -70,13 +70,13 @@ public GlycoProteinParsimony(string proteinAccess, int proteinPos, char aminoAci if (!localizedGlycan.ContainsKey(proPosId)) { - GlycoProteinParsimony gpp = new GlycoProteinParsimony(gsm.Accession, proteinPos, gsm.BaseSequence[local.Item1-2], local.Item3, gsm.PsmFdrInfo.QValue, gsm.LocalizationLevel, prob); + GlycoProteinParsimony gpp = new GlycoProteinParsimony(gsm.Accession, proteinPos, gsm.BaseSequence[local.Item1-2], local.Item3, gsm.FdrInfo.QValue, gsm.LocalizationLevel, prob); localizedGlycan.Add(proPosId, gpp); } else { bool islocalized = (local.Item3 || localizedGlycan[proPosId].IsLocalized); - double minQValue = localizedGlycan[proPosId].MinQValue > gsm.PsmFdrInfo.QValue ? gsm.PsmFdrInfo.QValue : localizedGlycan[proPosId].MinQValue; + double minQValue = localizedGlycan[proPosId].MinQValue > gsm.FdrInfo.QValue ? gsm.FdrInfo.QValue : localizedGlycan[proPosId].MinQValue; double maxProb = localizedGlycan[proPosId].MaxProbability > prob ? localizedGlycan[proPosId].MaxProbability : prob; var localLevel = localizedGlycan[proPosId].BestLocalizeLevel < gsm.LocalizationLevel ? localizedGlycan[proPosId].BestLocalizeLevel : gsm.LocalizationLevel; diff --git a/MetaMorpheus/TaskLayer/GlycoSearchTask/PostGlycoSearchAnalysisTask.cs b/MetaMorpheus/TaskLayer/GlycoSearchTask/PostGlycoSearchAnalysisTask.cs index 6d9bbbaf7..6578a1790 100644 --- a/MetaMorpheus/TaskLayer/GlycoSearchTask/PostGlycoSearchAnalysisTask.cs +++ b/MetaMorpheus/TaskLayer/GlycoSearchTask/PostGlycoSearchAnalysisTask.cs @@ -44,7 +44,7 @@ public MyTaskResults Run(string OutputFolder, List dbFilenameList, Li //This is all psms for all files including glyco- and non-glyco psms. SingleFDRAnalysis(allPSMs, commonParameters, new List { taskId }); - List filteredGsms = allPSMs.Where(p => p.PsmFdrInfo.QValue < 0.01).ToList(); + List filteredGsms = allPSMs.Where(p => p.FdrInfo.QValue < 0.01).ToList(); //write individual file results if (Parameters.GlycoSearchParameters.WriteIndividualFiles) @@ -263,7 +263,7 @@ private void GlycoAccessionAnalysis(List gsms, string indivi } //get non-ambigous psms above cutoffs that are normally used for parsimony - List _filteredPsms = psmsForProteinParsimony.Where(p => p.FullSequence != null && p.PsmFdrInfo.QValue <= 0.01 && p.PsmFdrInfo.QValueNotch <= 0.01).ToList(); + List _filteredPsms = psmsForProteinParsimony.Where(p => p.FullSequence != null && p.FdrInfo.QValue <= 0.01 && p.FdrInfo.QValueNotch <= 0.01).ToList(); // if there are no peptides observed, there are no proteins; return an empty list of protein groups if (_filteredPsms.Count != 0) @@ -366,8 +366,8 @@ private void QuantificationAnalysis() // get PSMs to pass to FlashLFQ var unambiguousPsmsBelowOnePercentFdr = Parameters.AllPsms.Where(p => - p.PsmFdrInfo.QValue <= 0.01 - && p.PsmFdrInfo.QValueNotch <= 0.01 + p.FdrInfo.QValue <= 0.01 + && p.FdrInfo.QValueNotch <= 0.01 && !p.IsDecoy && p.FullSequence != null).ToList(); diff --git a/MetaMorpheus/TaskLayer/MbrAnalysis/SpectralRecoveryRunner.cs b/MetaMorpheus/TaskLayer/MbrAnalysis/SpectralRecoveryRunner.cs index d8a7307a5..34e5a591d 100644 --- a/MetaMorpheus/TaskLayer/MbrAnalysis/SpectralRecoveryRunner.cs +++ b/MetaMorpheus/TaskLayer/MbrAnalysis/SpectralRecoveryRunner.cs @@ -118,7 +118,7 @@ public static SpectralRecoveryResults RunSpectralRecoveryAlgorithm( { List allPsms = parameters.AllPsms. OrderByDescending(p => p.Score). - ThenBy(p => p.PsmFdrInfo.QValue). + ThenBy(p => p.FdrInfo.QValue). ThenBy(p => p.FullFilePath). ThenBy(x => x.ScanNumber). ThenBy(p => p.FullSequence). @@ -158,11 +158,11 @@ private static List GetAllPeptides( double qValueCutoff = 0.01; if (parameters.AllPsms.Count > 100)//PEP is not computed when there are fewer than 100 psms { - peptides.RemoveAll(p => p.PsmFdrInfo.PEP_QValue > qValueCutoff); + peptides.RemoveAll(p => p.FdrInfo.PEP_QValue > qValueCutoff); } else { - peptides.RemoveAll(p => p.PsmFdrInfo.QValue > qValueCutoff); + peptides.RemoveAll(p => p.FdrInfo.QValue > qValueCutoff); } return peptides; @@ -191,7 +191,7 @@ private static SpectralMatch BestPsmForMbrPeak(IEnumerable peptid private static void AssignEstimatedPsmQvalue(ConcurrentDictionary bestMbrMatches, List allPsms) { double[] allScores = allPsms.Select(s => s.Score).OrderByDescending(s => s).ToArray(); - double[] allQValues = allPsms.OrderByDescending(s => s.Score).Select(q => q.PsmFdrInfo.QValue).ToArray(); + double[] allQValues = allPsms.OrderByDescending(s => s.Score).Select(q => q.FdrInfo.QValue).ToArray(); foreach (SpectralRecoveryPSM match in bestMbrMatches.Values) { @@ -204,12 +204,12 @@ private static void AssignEstimatedPsmQvalue(ConcurrentDictionary pepValues = bestMbrMatches. Select(p => p.Value.spectralLibraryMatch). Where(p => p != null). - OrderBy(p => p.PsmFdrInfo.PEP). - Select(p => p.PsmFdrInfo.PEP). + OrderBy(p => p.FdrInfo.PEP). + Select(p => p.FdrInfo.PEP). ToList(); foreach (SpectralRecoveryPSM match in bestMbrMatches.Values) @@ -271,18 +271,18 @@ private static void AssignEstimatedPsmPepQValue(ConcurrentDictionary psms, List databas search_score = new pepXML.Generated.nameValueType[] { new pepXML.Generated.nameValueType{ name = "Score", value = psm.Score.ToString()}, - new pepXML.Generated.nameValueType{ name = "Qvalue", value = psm.PsmFdrInfo.QValue.ToString() } + new pepXML.Generated.nameValueType{ name = "Qvalue", value = psm.FdrInfo.QValue.ToString() } }, }; searchHits.Add(searchHit); diff --git a/MetaMorpheus/TaskLayer/SearchTask/MzIdentMLWriter.cs b/MetaMorpheus/TaskLayer/SearchTask/MzIdentMLWriter.cs index 833e129ec..4b842483d 100644 --- a/MetaMorpheus/TaskLayer/SearchTask/MzIdentMLWriter.cs +++ b/MetaMorpheus/TaskLayer/SearchTask/MzIdentMLWriter.cs @@ -418,7 +418,7 @@ public static void WriteMzIdentMl(IEnumerable psms, List p.Peptide).Distinct().Count()], @@ -436,7 +436,7 @@ public static void WriteMzIdentMl(IEnumerable psms, List dbFilenameList, Li ComputeXlinkQandPValues(allPsms, intraCsms, interCsms, commonParameters, taskId); WriteCsvFiles(outputFolder, interCsms, intraCsms, singlePsms, loopPsms, deadendPsms, taskId, xlSearchParameters); - MyTaskResults.AddTaskSummaryText("Target inter-crosslinks within 1% FDR: " + interCsms.Count(p => p.PsmFdrInfo.QValue <= 0.01 && !p.IsDecoy && !p.BetaPeptide.IsDecoy)); - MyTaskResults.AddTaskSummaryText("Target intra-crosslinks within 1% FDR: " + intraCsms.Count(p => p.PsmFdrInfo.QValue <= 0.01 && !p.IsDecoy && !p.BetaPeptide.IsDecoy)); - MyTaskResults.AddTaskSummaryText("Target single peptides within 1% FDR: " + singlePsms.Count(p => p.PsmFdrInfo.QValue <= 0.01 && !p.IsDecoy)); - MyTaskResults.AddTaskSummaryText("Target loop-linked peptides within 1% FDR: " + loopPsms.Count(p => p.PsmFdrInfo.QValue <= 0.01 && !p.IsDecoy)); - MyTaskResults.AddTaskSummaryText("Target deadend peptides within 1% FDR: " + deadendPsms.Count(p => p.PsmFdrInfo.QValue <= 0.01 && !p.IsDecoy)); + MyTaskResults.AddTaskSummaryText("Target inter-crosslinks within 1% FDR: " + interCsms.Count(p => p.FdrInfo.QValue <= 0.01 && !p.IsDecoy && !p.BetaPeptide.IsDecoy)); + MyTaskResults.AddTaskSummaryText("Target intra-crosslinks within 1% FDR: " + intraCsms.Count(p => p.FdrInfo.QValue <= 0.01 && !p.IsDecoy && !p.BetaPeptide.IsDecoy)); + MyTaskResults.AddTaskSummaryText("Target single peptides within 1% FDR: " + singlePsms.Count(p => p.FdrInfo.QValue <= 0.01 && !p.IsDecoy)); + MyTaskResults.AddTaskSummaryText("Target loop-linked peptides within 1% FDR: " + loopPsms.Count(p => p.FdrInfo.QValue <= 0.01 && !p.IsDecoy)); + MyTaskResults.AddTaskSummaryText("Target deadend peptides within 1% FDR: " + deadendPsms.Count(p => p.FdrInfo.QValue <= 0.01 && !p.IsDecoy)); // write pepXML if (xlSearchParameters.WritePepXml) { List writeToXml = new List(); - writeToXml.AddRange(intraCsms.Where(p => !p.IsDecoy && !p.BetaPeptide.IsDecoy && p.PsmFdrInfo.QValue <= 0.05)); - writeToXml.AddRange(interCsms.Where(p => !p.IsDecoy && !p.BetaPeptide.IsDecoy && p.PsmFdrInfo.QValue <= 0.05)); - writeToXml.AddRange(singlePsms.Where(p => !p.IsDecoy && p.PsmFdrInfo.QValue <= 0.05)); - writeToXml.AddRange(loopPsms.Where(p => !p.IsDecoy && p.PsmFdrInfo.QValue <= 0.05)); - writeToXml.AddRange(deadendPsms.Where(p => !p.IsDecoy && p.PsmFdrInfo.QValue <= 0.05)); + writeToXml.AddRange(intraCsms.Where(p => !p.IsDecoy && !p.BetaPeptide.IsDecoy && p.FdrInfo.QValue <= 0.05)); + writeToXml.AddRange(interCsms.Where(p => !p.IsDecoy && !p.BetaPeptide.IsDecoy && p.FdrInfo.QValue <= 0.05)); + writeToXml.AddRange(singlePsms.Where(p => !p.IsDecoy && p.FdrInfo.QValue <= 0.05)); + writeToXml.AddRange(loopPsms.Where(p => !p.IsDecoy && p.FdrInfo.QValue <= 0.05)); + writeToXml.AddRange(deadendPsms.Where(p => !p.IsDecoy && p.FdrInfo.QValue <= 0.05)); writeToXml = writeToXml.OrderBy(p => p.ScanNumber).ToList(); foreach (var fullFilePath in currentRawFileList) @@ -106,11 +106,11 @@ public List RunSingleFile(string outputFolder, string ta WriteCsvFiles(outputFolder, interCsms, intraCsms, singlePsms, loopPsms, deadendPsms, taskId, xlSearchParameters); - allFilterPsms.AddRange(interCsms.Where(p=>p.PsmFdrInfo.QValue <= 0.2)); - allFilterPsms.AddRange(intraCsms.Where(p => p.PsmFdrInfo.QValue <= 0.2)); - allFilterPsms.AddRange(singlePsms.Where(p => p.PsmFdrInfo.QValue <= 0.2)); - allFilterPsms.AddRange(loopPsms.Where(p => p.PsmFdrInfo.QValue <= 0.2)); - allFilterPsms.AddRange(deadendPsms.Where(p => p.PsmFdrInfo.QValue <= 0.2)); + allFilterPsms.AddRange(interCsms.Where(p=>p.FdrInfo.QValue <= 0.2)); + allFilterPsms.AddRange(intraCsms.Where(p => p.FdrInfo.QValue <= 0.2)); + allFilterPsms.AddRange(singlePsms.Where(p => p.FdrInfo.QValue <= 0.2)); + allFilterPsms.AddRange(loopPsms.Where(p => p.FdrInfo.QValue <= 0.2)); + allFilterPsms.AddRange(deadendPsms.Where(p => p.FdrInfo.QValue <= 0.2)); return allFilterPsms; } @@ -228,8 +228,8 @@ private void DoCrosslinkFdrAnalysis(List csms) double qValue = Math.Min(1, (double)cumulativeDecoy / (cumulativeTarget <= 0 ? 1 : cumulativeTarget)); double qValueNotch = 0; //maybe we should assign this some day? - double pep = csm.PsmFdrInfo == null ? double.NaN : csm.PsmFdrInfo.PEP; - double pepQValue = csm.PsmFdrInfo == null ? double.NaN : csm.PsmFdrInfo.PEP_QValue; + double pep = csm.FdrInfo == null ? double.NaN : csm.FdrInfo.PEP; + double pepQValue = csm.FdrInfo == null ? double.NaN : csm.FdrInfo.PEP_QValue; csm.SetFdrValues(cumulativeTarget, cumulativeDecoy, qValue, 0, 0, qValueNotch, pep, pepQValue); } @@ -247,7 +247,7 @@ private void WriteXlSpectralLibrary( foreach (var csmGroups in linkedCsms.Where(c => !c.IsDecoy - && c.PsmFdrInfo.QValueNotch < 0.05) + && c.FdrInfo.QValueNotch < 0.05) .GroupBy(c => (c.UniqueSequence, c.ScanPrecursorCharge))) { CrosslinkSpectralMatch bestCsm = csmGroups.MaxBy(c => c.XLTotalScore); @@ -263,7 +263,7 @@ private void WriteXlSpectralLibrary( foreach (var singlePsmGroup in singlePsms.Where(c => !c.IsDecoy - && c.PsmFdrInfo.QValueNotch < 0.05) + && c.FdrInfo.QValueNotch < 0.05) .GroupBy(c => (c.FullSequence, c.ScanPrecursorCharge))) { CrosslinkSpectralMatch bestPsm = singlePsmGroup.MaxBy(c => c.Score); diff --git a/MetaMorpheus/TaskLayer/XLSearchTask/WriteXlFile.cs b/MetaMorpheus/TaskLayer/XLSearchTask/WriteXlFile.cs index cbd5c1840..65800667a 100644 --- a/MetaMorpheus/TaskLayer/XLSearchTask/WriteXlFile.cs +++ b/MetaMorpheus/TaskLayer/XLSearchTask/WriteXlFile.cs @@ -232,7 +232,7 @@ public static void WritePepXML_xl(List items, List items, List items, List items, List Date: Tue, 18 Jun 2024 11:27:45 -0500 Subject: [PATCH 5/7] Fewer files changed --- .../EngineLayer/CrosslinkSearch/CrosslinkSpectralMatch.cs | 8 ++++---- MetaMorpheus/TaskLayer/MbrAnalysis/SpectralRecoveryPSM.cs | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/MetaMorpheus/EngineLayer/CrosslinkSearch/CrosslinkSpectralMatch.cs b/MetaMorpheus/EngineLayer/CrosslinkSearch/CrosslinkSpectralMatch.cs index e8d92e202..4bdf2522c 100644 --- a/MetaMorpheus/EngineLayer/CrosslinkSearch/CrosslinkSpectralMatch.cs +++ b/MetaMorpheus/EngineLayer/CrosslinkSearch/CrosslinkSpectralMatch.cs @@ -492,18 +492,18 @@ public override string ToString() sb.Append((IsDecoy || BetaPeptide.IsDecoy) ? "D" : (IsContaminant || BetaPeptide.IsContaminant) ? "C" : "T"); sb.Append("\t"); } - if (PsmFdrInfo == null) + if (FdrInfo == null) { sb.Append("0\t0\t0\t"); } else { - sb.Append(PsmFdrInfo.QValue.ToString()); + sb.Append(FdrInfo.QValue.ToString()); sb.Append("\t"); - sb.Append(PsmFdrInfo.PEP.ToString()); sb.Append("\t"); + sb.Append(FdrInfo.PEP.ToString()); sb.Append("\t"); - sb.Append(PsmFdrInfo.PEP_QValue.ToString()); sb.Append("\t"); + sb.Append(FdrInfo.PEP_QValue.ToString()); sb.Append("\t"); } return sb.ToString(); } diff --git a/MetaMorpheus/TaskLayer/MbrAnalysis/SpectralRecoveryPSM.cs b/MetaMorpheus/TaskLayer/MbrAnalysis/SpectralRecoveryPSM.cs index 96379eba5..1ee92d8b6 100644 --- a/MetaMorpheus/TaskLayer/MbrAnalysis/SpectralRecoveryPSM.cs +++ b/MetaMorpheus/TaskLayer/MbrAnalysis/SpectralRecoveryPSM.cs @@ -65,9 +65,9 @@ public static string TabSeparatedHeader public override string ToString() { - string originalPsmQ = originalSpectralMatch == null ? "Spectrum Not Found" : originalSpectralMatch.PsmFdrInfo.QValue.ToString(); - string originalPsmPEP = originalSpectralMatch == null ? "" : originalSpectralMatch.PsmFdrInfo.PEP.ToString(); - string originalPsmPEPQ = originalSpectralMatch == null ? "" : originalSpectralMatch.PsmFdrInfo.PEP_QValue.ToString(); + string originalPsmQ = originalSpectralMatch == null ? "Spectrum Not Found" : originalSpectralMatch.FdrInfo.QValue.ToString(); + string originalPsmPEP = originalSpectralMatch == null ? "" : originalSpectralMatch.FdrInfo.PEP.ToString(); + string originalPsmPEPQ = originalSpectralMatch == null ? "" : originalSpectralMatch.FdrInfo.PEP_QValue.ToString(); var sb = new StringBuilder(); sb.Append(spectralLibraryMatch); sb.Append('\t'); From 30b1bf54823add15b295b21b1b1c26db322e5ff7 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 28 Aug 2024 11:14:38 -0500 Subject: [PATCH 6/7] pretty broken --- .../EngineLayer/PsmTsv/PsmTsvWriter.cs | 20 +++++++++---------- .../TaskLayer/SearchTask/SearchParameters.cs | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/MetaMorpheus/EngineLayer/PsmTsv/PsmTsvWriter.cs b/MetaMorpheus/EngineLayer/PsmTsv/PsmTsvWriter.cs index 4c31717e9..4278c5357 100644 --- a/MetaMorpheus/EngineLayer/PsmTsv/PsmTsvWriter.cs +++ b/MetaMorpheus/EngineLayer/PsmTsv/PsmTsvWriter.cs @@ -316,7 +316,7 @@ internal static void AddMatchedIonsData(Dictionary s, List s, SpectralMatch peptide, bool writePeptideLevelFdr = false) + internal static void AddMatchScoreData(Dictionary s, SpectralMatch peptide) { string spectralAngle = peptide == null ? " " : peptide.SpectralAngle.ToString("F4"); string localizedScores = " "; @@ -339,16 +339,16 @@ internal static void AddMatchScoreData(Dictionary s, SpectralMat string PEP = " "; string PEP_Qvalue = " "; - if (peptide != null && peptide.GetFdrInfo(writePeptideLevelFdr) != null) + if (peptide != null && peptide.FdrInfo != null) { - cumulativeTarget = peptide.GetFdrInfo(writePeptideLevelFdr).CumulativeTarget.ToString(CultureInfo.InvariantCulture); - cumulativeDecoy = peptide.GetFdrInfo(writePeptideLevelFdr).CumulativeDecoy.ToString(CultureInfo.InvariantCulture); - qValue = peptide.GetFdrInfo(writePeptideLevelFdr).QValue.ToString("F6", CultureInfo.InvariantCulture); - cumulativeTargetNotch = peptide.GetFdrInfo(writePeptideLevelFdr).CumulativeTargetNotch.ToString(CultureInfo.InvariantCulture); - cumulativeDecoyNotch = peptide.GetFdrInfo(writePeptideLevelFdr).CumulativeDecoyNotch.ToString(CultureInfo.InvariantCulture); - qValueNotch = peptide.GetFdrInfo(writePeptideLevelFdr).QValueNotch.ToString("F6", CultureInfo.InvariantCulture); - PEP = peptide.GetFdrInfo(writePeptideLevelFdr).PEP.ToString(); - PEP_Qvalue = peptide.GetFdrInfo(writePeptideLevelFdr).PEP_QValue.ToString(); + cumulativeTarget = peptide.FdrInfo.CumulativeTarget.ToString(CultureInfo.InvariantCulture); + cumulativeDecoy = peptide.FdrInfo.CumulativeDecoy.ToString(CultureInfo.InvariantCulture); + qValue = peptide.FdrInfo.QValue.ToString("F6", CultureInfo.InvariantCulture); + cumulativeTargetNotch = peptide.FdrInfo.CumulativeTargetNotch.ToString(CultureInfo.InvariantCulture); + cumulativeDecoyNotch = peptide.FdrInfo.CumulativeDecoyNotch.ToString(CultureInfo.InvariantCulture); + qValueNotch = peptide.FdrInfo.QValueNotch.ToString("F6", CultureInfo.InvariantCulture); + PEP = peptide.FdrInfo.PEP.ToString(); + PEP_Qvalue = peptide.FdrInfo.PEP_QValue.ToString(); } s[PsmTsvHeader.CumulativeTarget] = cumulativeTarget; diff --git a/MetaMorpheus/TaskLayer/SearchTask/SearchParameters.cs b/MetaMorpheus/TaskLayer/SearchTask/SearchParameters.cs index 34b113bcd..c3d8bee93 100644 --- a/MetaMorpheus/TaskLayer/SearchTask/SearchParameters.cs +++ b/MetaMorpheus/TaskLayer/SearchTask/SearchParameters.cs @@ -70,6 +70,7 @@ public SearchParameters() public bool MatchBetweenRuns { get; set; } public bool Normalize { get; set; } public double QuantifyPpmTol { get; set; } + public bool OnlyQuantifyConfidentPeptides { get; set; } public bool DoHistogramAnalysis { get; set; } public bool SearchTarget { get; set; } public DecoyType DecoyType { get; set; } @@ -79,7 +80,6 @@ public SearchParameters() public bool DoLocalizationAnalysis { get; set; } public bool DoLabelFreeQuantification { get; set; } public bool UseSharedPeptidesForLFQ { get; set; } - public bool OnlyQuantifyConfidentPeptides { get; set; } public bool DoMultiplexQuantification { get; set; } public string MultiplexModId { get; set; } public bool DoSpectralRecovery { get; set; } From eb5b703d1fd5efc40ea0233392684144eb11fcaa Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 28 Aug 2024 13:59:08 -0500 Subject: [PATCH 7/7] Fixed broken things --- .../FdrAnalysis/FdrAnalysisEngine.cs | 4 ++ .../EngineLayer/PsmTsv/PsmTsvWriter.cs | 20 ++++----- MetaMorpheus/TaskLayer/FilteredPsms.cs | 2 +- .../SearchTask/PostSearchAnalysisTask.cs | 42 +++++++++---------- .../Test/EverythingRunnerEngineTestCase.cs | 1 + .../Test/PostSearchAnalysisTaskTests.cs | 14 +++++-- 6 files changed, 46 insertions(+), 37 deletions(-) diff --git a/MetaMorpheus/EngineLayer/FdrAnalysis/FdrAnalysisEngine.cs b/MetaMorpheus/EngineLayer/FdrAnalysis/FdrAnalysisEngine.cs index 454bcebbc..cb9a62ca6 100644 --- a/MetaMorpheus/EngineLayer/FdrAnalysis/FdrAnalysisEngine.cs +++ b/MetaMorpheus/EngineLayer/FdrAnalysis/FdrAnalysisEngine.cs @@ -38,6 +38,10 @@ private void AddPsmAndPeptideFdrInfoIfNotPresent() foreach (var psm in AllPsms.Where(p => p.PeptideFdrInfo == null)) { psm.PeptideFdrInfo = new FdrInfo(); + // If peptide level Q-values default to zero, it is hard to determine which PSMs have a legitimate peptide-level Q-value of zero and + // which just weren't included in peptide-level calculations. Hence, we set q values to one here. + psm.PeptideFdrInfo.QValue = 1; + psm.PeptideFdrInfo.PEP_QValue = 1; } } diff --git a/MetaMorpheus/EngineLayer/PsmTsv/PsmTsvWriter.cs b/MetaMorpheus/EngineLayer/PsmTsv/PsmTsvWriter.cs index 4278c5357..4c31717e9 100644 --- a/MetaMorpheus/EngineLayer/PsmTsv/PsmTsvWriter.cs +++ b/MetaMorpheus/EngineLayer/PsmTsv/PsmTsvWriter.cs @@ -316,7 +316,7 @@ internal static void AddMatchedIonsData(Dictionary s, List s, SpectralMatch peptide) + internal static void AddMatchScoreData(Dictionary s, SpectralMatch peptide, bool writePeptideLevelFdr = false) { string spectralAngle = peptide == null ? " " : peptide.SpectralAngle.ToString("F4"); string localizedScores = " "; @@ -339,16 +339,16 @@ internal static void AddMatchScoreData(Dictionary s, SpectralMat string PEP = " "; string PEP_Qvalue = " "; - if (peptide != null && peptide.FdrInfo != null) + if (peptide != null && peptide.GetFdrInfo(writePeptideLevelFdr) != null) { - cumulativeTarget = peptide.FdrInfo.CumulativeTarget.ToString(CultureInfo.InvariantCulture); - cumulativeDecoy = peptide.FdrInfo.CumulativeDecoy.ToString(CultureInfo.InvariantCulture); - qValue = peptide.FdrInfo.QValue.ToString("F6", CultureInfo.InvariantCulture); - cumulativeTargetNotch = peptide.FdrInfo.CumulativeTargetNotch.ToString(CultureInfo.InvariantCulture); - cumulativeDecoyNotch = peptide.FdrInfo.CumulativeDecoyNotch.ToString(CultureInfo.InvariantCulture); - qValueNotch = peptide.FdrInfo.QValueNotch.ToString("F6", CultureInfo.InvariantCulture); - PEP = peptide.FdrInfo.PEP.ToString(); - PEP_Qvalue = peptide.FdrInfo.PEP_QValue.ToString(); + cumulativeTarget = peptide.GetFdrInfo(writePeptideLevelFdr).CumulativeTarget.ToString(CultureInfo.InvariantCulture); + cumulativeDecoy = peptide.GetFdrInfo(writePeptideLevelFdr).CumulativeDecoy.ToString(CultureInfo.InvariantCulture); + qValue = peptide.GetFdrInfo(writePeptideLevelFdr).QValue.ToString("F6", CultureInfo.InvariantCulture); + cumulativeTargetNotch = peptide.GetFdrInfo(writePeptideLevelFdr).CumulativeTargetNotch.ToString(CultureInfo.InvariantCulture); + cumulativeDecoyNotch = peptide.GetFdrInfo(writePeptideLevelFdr).CumulativeDecoyNotch.ToString(CultureInfo.InvariantCulture); + qValueNotch = peptide.GetFdrInfo(writePeptideLevelFdr).QValueNotch.ToString("F6", CultureInfo.InvariantCulture); + PEP = peptide.GetFdrInfo(writePeptideLevelFdr).PEP.ToString(); + PEP_Qvalue = peptide.GetFdrInfo(writePeptideLevelFdr).PEP_QValue.ToString(); } s[PsmTsvHeader.CumulativeTarget] = cumulativeTarget; diff --git a/MetaMorpheus/TaskLayer/FilteredPsms.cs b/MetaMorpheus/TaskLayer/FilteredPsms.cs index 869a477a5..25426d8da 100644 --- a/MetaMorpheus/TaskLayer/FilteredPsms.cs +++ b/MetaMorpheus/TaskLayer/FilteredPsms.cs @@ -84,7 +84,7 @@ public static FilteredPsms Filter(IEnumerable psms, pepQValueThreshold ??= commonParams.PepQValueThreshold; double filterThreshold = Math.Min((double)qValueThreshold, (double)pepQValueThreshold); bool filteringNotPerformed = false; - List filteredPsms = new List(); + List filteredPsms = new(); // set the filter type string filterType = "q-value"; diff --git a/MetaMorpheus/TaskLayer/SearchTask/PostSearchAnalysisTask.cs b/MetaMorpheus/TaskLayer/SearchTask/PostSearchAnalysisTask.cs index 6db15ef23..6bdddb22c 100644 --- a/MetaMorpheus/TaskLayer/SearchTask/PostSearchAnalysisTask.cs +++ b/MetaMorpheus/TaskLayer/SearchTask/PostSearchAnalysisTask.cs @@ -72,7 +72,6 @@ public MyTaskResults Run() } ConstructResultsDictionary(); DoMassDifferenceLocalizationAnalysis(); - PeptideAnalysis(); ProteinAnalysis(); QuantificationAnalysis(); @@ -502,8 +501,23 @@ private void QuantificationAnalysis() } } + // Determine if only select, high confidence peptides will be used for quant + List peptideSequencesForQuantification = null; + if (Parameters.SearchParameters.SilacLabels == null && Parameters.SearchParameters.OnlyQuantifyConfidentPeptides) + { + peptideSequencesForQuantification = FilteredPsms.Filter(Parameters.AllPsms, + CommonParameters, + includeDecoys: false, + includeContaminants: true, + includeAmbiguous: false, + includeAmbiguousMods: false, + includeHighQValuePsms: false, + filterAtPeptideLevel: true) + .Select(pep => pep.FullSequence).Distinct().ToList(); + } + // run FlashLFQ - var flashLfqEngine = new FlashLfqEngine( + var flashLfqEngine = new FlashLfqEngine( allIdentifications: flashLFQIdentifications, normalize: Parameters.SearchParameters.Normalize, ppmTolerance: Parameters.SearchParameters.QuantifyPpmTol, @@ -512,9 +526,7 @@ private void QuantificationAnalysis() useSharedPeptidesForProteinQuant: Parameters.SearchParameters.UseSharedPeptidesForLFQ, silent: true, maxThreads: CommonParameters.MaxThreadsToUsePerFile, - peptideSequencesToUse: (Parameters.SearchParameters.SilacLabels == null && Parameters.SearchParameters.OnlyQuantifyConfidentPeptides) - ? _filteredPeptides.Select(peptide => peptide.FullSequence).ToList() - : null); + peptideSequencesToUse: peptideSequencesForQuantification); if (flashLFQIdentifications.Any()) { @@ -588,7 +600,7 @@ protected void WritePsmsToTsv(IEnumerable psms, string filePath, .SelectMany(pwsm => pwsm.Peptide.AllModsOneIsNterminus.Values) .Any(mod => mod.OriginalId.Equals(Parameters.MultiplexModification.OriginalId)))) { - WritePsmPlusMultiplexIons(psms, filePath, asPeptide); + WritePsmPlusMultiplexIons(psms, filePath); } else { @@ -1302,20 +1314,6 @@ private void WritePrunedDatabase() } } - private void PeptideAnalysis() - { - List peptides = Parameters.AllPsms - .GroupBy(b => b.FullSequence) - .Select(b => b.FirstOrDefault()).ToList(); - - new FdrAnalysisEngine(peptides, Parameters.NumNotches, CommonParameters, - FileSpecificParameters, new List { Parameters.SearchTaskId }, - "Peptide", peptideLevelFdr: true, doPEP: false).Run(); - - FilterSpecificPsms(peptides, out int psmOrPeptideCountForResults, peptideLevelFiltering: true); - FilterAllPeptides(peptides); // This sets the _allPeptides field - } - private void WritePsmPlusMultiplexIons(IEnumerable psms, string filePath, bool writePeptideLevelResults = false) { PpmTolerance ionTolerance = new PpmTolerance(10); @@ -1445,9 +1443,9 @@ private void WriteVariantResults() new List { Parameters.SearchTaskId }, "variant_PSMs", doPEP: false).Run(); possibleVariantPsms - .OrderBy(p => p.PsmFdrInfo.QValue) + .OrderBy(p => p.FdrInfo.QValue) .ThenByDescending(p => p.Score) - .ThenBy(p => p.PsmFdrInfo.CumulativeTarget) + .ThenBy(p => p.FdrInfo.CumulativeTarget) .ToList(); WritePsmsToTsv(possibleVariantPsms, variantPsmFile); diff --git a/MetaMorpheus/Test/EverythingRunnerEngineTestCase.cs b/MetaMorpheus/Test/EverythingRunnerEngineTestCase.cs index 699b4fe67..64ffddaff 100644 --- a/MetaMorpheus/Test/EverythingRunnerEngineTestCase.cs +++ b/MetaMorpheus/Test/EverythingRunnerEngineTestCase.cs @@ -126,6 +126,7 @@ static EverythingRunnerEngineTestCase() @"TestData\TaGe_SA_A549_3_snip_2.mzML"); string myDatabase = Path.Combine(TestContext.CurrentContext.TestDirectory, @"TestData\TaGe_SA_A549_3_snip.fasta"); + searchTaskLoaded.SearchParameters.OnlyQuantifyConfidentPeptides = true; _cases.Add(EverythingRunnerEngineTestCases.BottomUpQValue, new EverythingRunnerEngineTestCase(EverythingRunnerEngineTestCases.BottomUpQValue, new List<(string, MetaMorpheusTask)> { ("postSearchAnalysisTaskTestOutput", searchTaskLoaded) }, diff --git a/MetaMorpheus/Test/PostSearchAnalysisTaskTests.cs b/MetaMorpheus/Test/PostSearchAnalysisTaskTests.cs index 1c79f0d1e..f06652821 100644 --- a/MetaMorpheus/Test/PostSearchAnalysisTaskTests.cs +++ b/MetaMorpheus/Test/PostSearchAnalysisTaskTests.cs @@ -101,6 +101,16 @@ public static void AllResultsAndResultsTxtContainsCorrectValues_PepQValue_Bottom Assert.AreEqual("TaGe_SA_A549_3_snip_2 - Target protein groups within 1 % FDR: 155", results[15]); } + [Test] + public static void TestPeptideQuantificationResults() + { + EverythingRunnerEngineTestCase.TryGetTestCase(EverythingRunnerEngineTestCases.BottomUpQValue, out var testCase); + + string peptideQuantFile = Path.Combine(testCase.OutputDirectory, @"postSearchAnalysisTaskTestOutput\AllQuantifiedPeptides.tsv"); + string[] peptideQuant = File.ReadAllLines(peptideQuantFile); + Assert.AreEqual(175, peptideQuant.Length); // 174 peptides + header. Make sure that we're quantifying only those peptides with q-value <= 0.01 + } + /// /// Ensures that there is the proper ratio of summary and individual lines in the result.txt file and that peptides and proteoforms are distinct /// @@ -115,10 +125,6 @@ public static void AllResultTxtContainsCorrectNumberOfResultLines(EverythingRunn int expectedSummaryLines = 1; var allResultTxtLines = File.ReadAllLines(Path.Combine(testCase.OutputDirectory, @"allResults.txt")); - string peptideQuantFile = Path.Combine(outputFolder, "postSearchAnalysisTaskTestOutput", "AllQuantifiedPeptides.tsv"); - string[] peptideQuant = File.ReadAllLines(peptideQuantFile); - Assert.AreEqual(175, peptideQuant.Length); // 174 peptides + header. Make sure that we're quantifying only those peptides with q-value <= 0.01 - var summaryPsmLines = allResultTxtLines.Where(p => p.Contains("All target PSMs")).ToArray(); var individualPsmLines = allResultTxtLines.Where(p => p.Contains("Target PSMs") && !p.Contains("All")).ToArray();