Merge branch 'SPS' into master

Author: nielshulstaert

MainClass.cs

@@ -19,7 +19,7 @@ public static class MainClass
         private static readonly ILog Log =
             LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);
 
-        public const string Version = "1.3.1";
+        public const string Version = "1.3.2";
 
         public static void Main(string[] args)
         {
@@ -424,6 +424,10 @@ private static void RegularParametersParsing(string[] args)
                     "e|ignoreInstrumentErrors", "Ignore missing properties by the instrument.",
                     v => parseInput.IgnoreInstrumentErrors = v != null
                 },
+                {
+                    "x|includeExceptionData", "Include reference and exception data",
+                    v => parseInput.ExData = v != null
+                },
                 {
                     "L=|msLevel=",
                     "Select MS levels (MS1, MS2, etc) included in the output, should be a comma-separated list of integers ( 1,2,3 ) and/or intervals ( 1-3 ), open-end intervals ( 1- ) are allowed",
@@ -778,7 +782,7 @@ private static HashSet<int> ParseMsLevel(string inputString)
             if (!valid.IsMatch(inputString))
                 throw new OptionException("Invalid characters in msLevel key", "msLevel");
 
-            foreach (var piece in inputString.Split(new char[] { ',' }))
+            foreach (var piece in inputString.Split(new char[] {','}))
             {
                 try
                 {
@@ -821,7 +825,8 @@ private static HashSet<int> ParseMsLevel(string inputString)
 
                 catch (Exception ex)
                 {
-                    throw new OptionException(String.Format("Cannot parse part of msLevel input: '{0}'", piece), "msLevel", ex);
+                    throw new OptionException(String.Format("Cannot parse part of msLevel input: '{0}'", piece),
+                        "msLevel", ex);
                 }
             }
 

ParseInput.cs

@@ -75,6 +75,8 @@ public string RawFilePath
 
         public bool IgnoreInstrumentErrors { get; set; }
 
+        public bool ExData { get; set; }
+
         public HashSet<int> MsLevel { get; set; }
 
         public bool MGFPrecursor { get; set; }

Properties/AssemblyInfo.cs

@@ -5,7 +5,7 @@
 // set of attributes. Change these attribute values to modify the information
 // associated with an assembly.
 [assembly: AssemblyTitle("ThermoRawFileParser")]
-[assembly: AssemblyDescription("")]
+[assembly: AssemblyDescription("Modular, Scalable, and Cross-Platform RAW File Conversion [PMID 31755270]")]
 [assembly: AssemblyConfiguration("")]
 [assembly: AssemblyCompany("")]
 [assembly: AssemblyProduct("ThermoRawFileParser")]
@@ -31,7 +31,7 @@
 // You can specify all the values or you can default the Build and Revision Numbers 
 // by using the '*' as shown below:
 // [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("1.3.1.0")]
-[assembly: AssemblyFileVersion("1.3.1.0")]
+[assembly: AssemblyVersion("1.3.2.0")]
+[assembly: AssemblyFileVersion("1.3.2.0")]
 
 [assembly: log4net.Config.XmlConfigurator(ConfigFile = "log4net.config")]

README.md

@@ -2,7 +2,7 @@
 
 Wrapper around the .net (C#) ThermoFisher ThermoRawFileReader library for running on Linux with mono (works on Windows too). It takes a thermo RAW file as input and outputs a metadata file and the spectra in 3 possible formats:
 * MGF: MS2 and MS3 spectra
-* mzML and indexed mzML: both MS1, MS2 and MS3 spectra
+* mzML and indexed mzML: MS1, MS2 and MS3 spectra
 * Apache Parquet: under development
 
 As of version 1.2.0, 2 subcommands are available (shoutout to the [eubic 2020 developers meeting](https://eubic-ms.org/events/2020-developers-meeting/), see [usage](#usage) for examples):
@@ -76,6 +76,7 @@ optional subcommands are xic|query (use [subcommand] -h for more info]):
                                verbose.
   -e, --ignoreInstrumentErrors
                              Ignore missing properties by the instrument.
+  -x, --includeExceptionData Include reference and exception data
   -L, --msLevel=VALUE        Select MS levels (MS1, MS2, etc) included in the
                                output, should be a comma-separated list of
                                integers ( 1,2,3 ) and/or intervals ( 1-3 ),
@@ -112,7 +113,7 @@ usage is:
                                Thermo library. By default peak picking is
                                enabled.
   -s, --stdout               Pipes the output into standard output. Logging is
-                               being turned off
+                               being turned off.
 ```
 ### xic subcommand
 Return one or more chromatograms based on query JSON input.

RawFileParser.cs

@@ -1,5 +1,6 @@
 using System;
 using System.IO;
+using System.Linq;
 using ThermoFisher.CommonCore.Data;
 using ThermoFisher.CommonCore.Data.Business;
 using ThermoFisher.CommonCore.Data.Interfaces;
@@ -23,9 +24,12 @@ public static void Parse(ParseInput parseInput)
             {
                 Log.Info("Started analyzing folder " + parseInput.RawDirectoryPath);
 
-                var rawFilesPath =
-                    Directory.EnumerateFiles(parseInput.RawDirectoryPath);
-                if (Directory.GetFiles(parseInput.RawDirectoryPath, "*", SearchOption.TopDirectoryOnly).Length == 0)
+                var rawFilesPath = Directory
+                    .EnumerateFiles(parseInput.RawDirectoryPath, "*", SearchOption.TopDirectoryOnly)
+                    .Where(s => s.ToLower().EndsWith("raw")).ToArray();
+                Log.Info(String.Format("The folder contains {0} RAW files", rawFilesPath.Length));
+
+                if (rawFilesPath.Length == 0)
                 {
                     Log.Debug("No raw files found in folder");
                     throw new RawFileParserException("No raw files found in folder!");
@@ -99,6 +103,8 @@ private static void ProcessFile(ParseInput parseInput)
                 // selected instrument to the MS instrument, first instance of it
                 rawFile.SelectInstrument(Device.MS, 1);
 
+                rawFile.IncludeReferenceAndExceptionData = parseInput.ExData;
+
                 // Get the first and last scan from the RAW file
                 var firstScanNumber = rawFile.RunHeaderEx.FirstSpectrum;
                 var lastScanNumber = rawFile.RunHeaderEx.LastSpectrum;

ThermoRawFileParser.csproj

@@ -56,6 +56,9 @@
     <CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
     <Prefer32Bit>false</Prefer32Bit>
   </PropertyGroup>
+  <PropertyGroup>
+    <StartupObject>ThermoRawFileParser.MainClass</StartupObject>
+  </PropertyGroup>
   <ItemGroup>
     <Reference Include="AWS.Logger.Core, Version=1.4.0.0, Culture=neutral, PublicKeyToken=885c28607f98e604">
       <HintPath>packages\AWS.Logger.Core.1.4.0\lib\net45\AWS.Logger.Core.dll</HintPath>

ThermoRawFileParserTest/Data/TestFolderMgfs/NOTRAWFILE

@@ -0,0 +1 @@
+NOTRAWFILE

ThermoRawFileParserTest/WriterTests.cs

@@ -35,18 +35,25 @@ public void TestMgf()
         public void TestFolderMgfs()
         {
             // Get temp path for writing the test MGF
-            var tempFilePath = Path.GetTempPath();
+            var tempFilePath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
+
+            Directory.CreateDirectory(tempFilePath);
 
             var testRawFolder = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, @"Data/TestFolderMgfs");
             var parseInput = new ParseInput(null, testRawFolder, tempFilePath, OutputFormat.MGF);
 
             RawFileParser.Parse(parseInput);
 
+            var numFiles = Directory.GetFiles(tempFilePath, "*.mgf");
+            Assert.AreEqual(numFiles.Length, 2);
+
             var mgfData = Mgf.LoadAllStaticData(Path.Combine(tempFilePath, "small1.mgf"));
             Assert.AreEqual(34, mgfData.NumSpectra);
 
             var mgfData2 = Mgf.LoadAllStaticData(Path.Combine(tempFilePath, "small2.mgf"));
             Assert.AreEqual(34, mgfData2.NumSpectra);
+
+            Directory.Delete(tempFilePath, true);
         }
 
         [Test]

Writer/MzMlSpectrumWriter.cs

@@ -27,7 +27,12 @@ public class MzMlSpectrumWriter : SpectrumWriter
         private static readonly ILog Log =
             LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
 
-        private const string FilterStringIsolationMzPattern = @"ms2 (.*?)@";
+        private readonly Regex FilterStringIsolationMzPattern = new Regex(@"ms2 (.*?)@");
+
+        //tune version < 3 produces multiple trailer entry like "SPS Mass [number]"
+        private readonly Regex SPSentry = new Regex(@"SPS Mass\s+\d+:");
+        //tune version == 3 produces trailer entry "SPS Masses/Continued"
+        private readonly Regex SPSentry3 = new Regex(@"SPS Masses(?:\s+Continued)?:");
 
         private IRawDataPlus _rawFile;
 
@@ -1146,7 +1151,7 @@ private SpectrumType ConstructMSSpectrum(int scanNumber)
             int? charge = null;
             double? monoisotopicMz = null;
             double? ionInjectionTime = null;
-            double? isolationWidth = null;
+            List<double> SPSMasses = new List<double>();
             for (var i = 0; i < trailerData.Length; i++)
             {
                 if (trailerData.Labels[i] == "Charge State:")
@@ -1169,10 +1174,21 @@ private SpectrumType ConstructMSSpectrum(int scanNumber)
                         CultureInfo.CurrentCulture);
                 }
 
-                if (trailerData.Labels[i] == "MS" + (int) scanFilter.MSOrder + " Isolation Width:")
+                //tune version < 3 produced trailer entry like "SPS Mass #", one entry per mass
+                if (SPSentry.IsMatch(trailerData.Labels[i]))
                 {
-                    isolationWidth = double.Parse(trailerData.Values[i], NumberStyles.Any,
-                        CultureInfo.CurrentCulture);
+                    var mass = double.Parse(trailerData.Values[i]);
+                    if (mass > 0)  SPSMasses.Add(mass); //zero means mass does not exist
+                }
+
+                //tune version == 3 produces trailer entry "SPS Masses", comma separated list of masses 
+                if (SPSentry3.IsMatch(trailerData.Labels[i]))
+                {
+                    foreach (var mass in trailerData.Values[i].Trim().Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries))
+                    {
+                        SPSMasses.Add(double.Parse(mass));
+                    }
+                    
                 }
             }
 
@@ -1206,7 +1222,7 @@ private SpectrumType ConstructMSSpectrum(int scanNumber)
                     });
 
                     // Keep track of scan number and isolation m/z for precursor reference                   
-                    var result = Regex.Match(scanEvent.ToString(), FilterStringIsolationMzPattern);
+                    var result = FilterStringIsolationMzPattern.Match(scanEvent.ToString());
                     if (result.Success)
                     {
                         if (_precursorMs2ScanNumbers.ContainsKey(result.Groups[1].Value))
@@ -1219,7 +1235,7 @@ private SpectrumType ConstructMSSpectrum(int scanNumber)
 
                     // Construct and set the precursor list element of the spectrum                    
                     var precursorListType =
-                        ConstructPrecursorList(scanEvent, charge, scanFilter.MSOrder, monoisotopicMz, isolationWidth);
+                        ConstructPrecursorList(scanEvent, charge, scanFilter.MSOrder, monoisotopicMz, SPSMasses);
                     spectrum.precursorList = precursorListType;
                     break;
                 case MSOrderType.Ms3:
@@ -1230,8 +1246,7 @@ private SpectrumType ConstructMSSpectrum(int scanNumber)
                         name = "MSn spectrum",
                         value = ""
                     });
-                    precursorListType = ConstructPrecursorList(scanEvent, charge, scanFilter.MSOrder, monoisotopicMz,
-                        isolationWidth);
+                    precursorListType = ConstructPrecursorList(scanEvent, charge, scanFilter.MSOrder, monoisotopicMz, SPSMasses);
                     spectrum.precursorList = precursorListType;
                     break;
                 default:
@@ -1772,19 +1787,20 @@ private SpectrumType ConstructPDASpectrum(int scanNumber, int instrumentNumber)
         /// <param name="isolationWidth">the isolation width</param>
         /// <returns>the precursor list</returns>
         private PrecursorListType ConstructPrecursorList(IScanEventBase scanEvent, int? charge, MSOrderType msLevel,
-            double? monoisotopicMz, double? isolationWidth)
+            double? monoisotopicMz, List<double> SPSMasses)
         {
             // Construct the precursor
             var precursorList = new PrecursorListType
             {
-                count = "1",
-                precursor = new PrecursorType[1]
+                count = (Math.Max(SPSMasses.Count, 1)).ToString(),
+                precursor = new PrecursorType[Math.Max(SPSMasses.Count, 1)]
             };
 
             var spectrumRef = "";
             int precursorScanNumber = _precursorMs1ScanNumber;
             IReaction reaction = null;
             var precursorMz = 0.0;
+            double? isolationWidth = null;
             try
             {
                 switch (msLevel)
@@ -1814,6 +1830,7 @@ private PrecursorListType ConstructPrecursorList(IScanEventBase scanEvent, int?
                 }
 
                 precursorMz = reaction.PrecursorMass;
+                isolationWidth = reaction.IsolationWidth;
             }
             catch (ArgumentOutOfRangeException)
             {
@@ -1823,15 +1840,11 @@ private PrecursorListType ConstructPrecursorList(IScanEventBase scanEvent, int?
             var precursor = new PrecursorType
             {
                 selectedIonList =
-                    new SelectedIonListType {count = 1.ToString(), selectedIon = new ParamGroupType[1]},
+                    new SelectedIonListType {count = "1", selectedIon = new ParamGroupType[1]},
                 spectrumRef = spectrumRef
             };
 
-            precursor.selectedIonList.selectedIon[0] =
-                new ParamGroupType
-                {
-                    cvParam = new CVParamType[3]
-                };
+            precursor.selectedIonList.selectedIon[0] = new ParamGroupType();
 
             // Selected ion MZ
             var selectedIonMz = CalculateSelectedIonMz(reaction, monoisotopicMz, isolationWidth);
@@ -1886,6 +1899,7 @@ private PrecursorListType ConstructPrecursorList(IScanEventBase scanEvent, int?
                 {
                     cvParam = new CVParamType[3]
                 };
+
             precursor.isolationWindow.cvParam[0] =
                 new CVParamType
                 {
@@ -2022,6 +2036,44 @@ private PrecursorListType ConstructPrecursorList(IScanEventBase scanEvent, int?
 
             precursorList.precursor[0] = precursor;
 
+            //the first SPS mass seems to be the same as the one from reaction or scan filter
+            for (int n = 1; n < SPSMasses.Count; n++)
+            {
+                var SPSPrecursor = new PrecursorType
+                {
+                    selectedIonList =
+                    new SelectedIonListType { count = "1", selectedIon = new ParamGroupType[1] },
+                    spectrumRef = spectrumRef
+                };
+
+                // Selected ion MZ only
+                SPSPrecursor.selectedIonList.selectedIon[0] =
+                new ParamGroupType
+                {
+                    cvParam = new CVParamType[]
+                    {
+                        new CVParamType {
+                            name = "selected ion m/z",
+                            value = SPSMasses[n].ToString(),
+                            accession = "MS:1000744",
+                            cvRef = "MS",
+                            unitCvRef = "MS",
+                            unitAccession = "MS:1000040",
+                            unitName = "m/z"
+                        }
+                    }
+                };
+
+                //All SPS masses have the same activation (i.e. it was calculated above)
+                SPSPrecursor.activation =
+                new ParamGroupType
+                {
+                    cvParam = activationCvParams.ToArray()
+                };
+
+                precursorList.precursor[n] = SPSPrecursor;
+            }
+
             return precursorList;
         }
 

Writer/OntologyMapping.cs

@@ -612,12 +612,8 @@ public static List<CVParamType> GetDetectors(string instrumentAccession)
                 case "MS:1002732":
                 // ORBITRAP ECLIPSE    
                 case "MS:1003029":
-                // ORBITRAP EXPLORIS 120
-                case "MS:1003095":
-                // ORBITRAP EXPLORIS 240
-                case "MS:1003094":
-                // ORBITRAP EXPLORIS 480
-                case "MS:1003028":
+                // ORBITRAP ID-X
+                case "MS:1003112":
                     detectors = new List<CVParamType>
                     {
                         new CVParamType
@@ -648,6 +644,12 @@ public static List<CVParamType> GetDetectors(string instrumentAccession)
                 case "MS:1002877":
                 // Q EXACTIVE PLUS    
                 case "MS:1002634":
+                // ORBITRAP EXPLORIS 120
+                case "MS:1003095":
+                // ORBITRAP EXPLORIS 240
+                case "MS:1003094":
+                // ORBITRAP EXPLORIS 480
+                case "MS:1003028":
                     detectors = new List<CVParamType>
                     {
                         new CVParamType