Merge 25.2 to develop

Author: labkey-teamcity

src/org/labkey/targetedms/parser/SkylineBinaryParser.java

@@ -15,19 +15,16 @@
  */
 package org.labkey.targetedms.parser;
 
-import com.google.protobuf.CodedInputStream;
 import org.apache.commons.io.IOUtils;
 import org.apache.logging.log4j.Logger;
 import org.jetbrains.annotations.NotNull;
 import org.labkey.api.exp.api.DataType;
-import org.labkey.api.pipeline.PipelineJobException;
 import org.labkey.targetedms.parser.proto.ChromatogramGroupDataOuterClass;
 import org.labkey.targetedms.parser.skyd.CacheFormat;
 import org.labkey.targetedms.parser.skyd.CacheFormatVersion;
 import org.labkey.targetedms.parser.skyd.CacheHeaderStruct;
 import org.labkey.targetedms.parser.skyd.CachedFileHeaderStruct;
 import org.labkey.targetedms.parser.skyd.ChromGroupHeaderInfo;
-import org.labkey.targetedms.parser.skyd.ChromPeak;
 import org.labkey.targetedms.parser.skyd.ChromTransition;
 import org.labkey.targetedms.parser.skyd.StructSerializer;
 
@@ -39,7 +36,6 @@
 import java.nio.ByteBuffer;
 import java.nio.channels.Channels;
 import java.nio.channels.FileChannel;
-import java.nio.channels.SeekableByteChannel;
 import java.util.EnumSet;
 import java.util.List;
 import java.util.Objects;
@@ -64,7 +60,6 @@ public class SkylineBinaryParser
     private CacheHeaderStruct _cacheHeaderStruct;
 
     private ChromGroupHeaderInfo[] _chromatograms;
-    private float[] _allPeaksRt;
     private byte[] _seqBytes;
     private List<ChromatogramGroupId> _chromatogramGroupIds;
 
@@ -122,7 +117,6 @@ public void parse() throws IOException
 
         parseChromatogramGroupIds();
         parseFiles();
-        parsePeaks();
         _log.debug("Starting to load chromatogram headers");
         parseChromatograms();
         _log.debug("Done loading chromatogram headers");
@@ -202,19 +196,6 @@ private String readStringOfByteLength(InputStream inputStream, int length) throw
         return new String(buffer, _cacheFormat.getCharset());
     }
 
-    private void parsePeaks() throws IOException
-    {
-        _channel.position(_cacheHeaderStruct.getLocationPeaks());
-        ChromPeak[] chromPeaks = _cacheFormat.chromPeakSerializer()
-                .readArray(Channels.newInputStream(_channel), _cacheHeaderStruct.getNumPeaks());
-        _allPeaksRt = new float[chromPeaks.length];
-
-        for (int i = 0; i < chromPeaks.length; i++)
-        {
-            _allPeaksRt[i] = chromPeaks[i].getRetentionTime();
-        }
-    }
-
     private void parseChromatogramGroupIds() throws IOException
     {
         if (_cacheFormat.getFormatVersion().compareTo(CacheFormatVersion.Eighteen) < 0)
@@ -253,27 +234,17 @@ private void parseChromatograms() throws IOException
                 Channels.newInputStream(_channel), _cacheHeaderStruct.getNumChromatograms());
     }
 
-    public SeekableByteChannel getChannel()
-    {
-        return _channel;
-    }
-
     final int getCacheFileSize()
     {
         return _cacheFiles != null ? _cacheFiles.length : 0;
     }
 
-    public int matchTransitions(ChromGroupHeaderInfo header, List<? extends GeneralTransition> transitions, Double explicitRt, double tolerance, boolean multiMatch)
+    /**
+     * Returns the number of transitions from the Precursor in the Skyline document that have a matching
+     * chromatogram in the ChromGroupHeaderInfo.
+     */
+    public int countTransitionMatches(ChromGroupHeaderInfo header, List<? extends GeneralTransition> transitions, double tolerance)
     {
-        int match = 0;
-
-        if (explicitRt != null)
-        {
-            // We have retention time info, use that in the match
-            if (header.excludesTime(explicitRt))
-                return match;
-        }
-
         var numChromTransitions = header.getNumTransitions();
         ChromTransition[] chromTransitions;
 
@@ -285,44 +256,19 @@ public int matchTransitions(ChromGroupHeaderInfo header, List<? extends GeneralT
         {
             throw new RuntimeException(e);
         }
-
+        int matchCount = 0;
         for (GeneralTransition transition : transitions)
         {
             for (int i = 0; i < numChromTransitions; i++)
             {
-                // Do we need to look through all of the transitions from the .skyd file?
                 if (header.toSignedMz(transition.getMz()).compareTolerant(chromTransitions[i].getProduct(header), tolerance) == 0)
                 {
-                    if (explicitRt == null)
-                    {
-                        match++;
-                        if (!multiMatch)
-                        {
-                            break;  // only one match per transition
-                        }
-                    }
-                    else
-                    {
-                        match = multiMatch ? match + 1 : 1; // Examine all RT values even if we're not multimatch
-                    }
+                    matchCount++;
+                    break;
                 }
             }
         }
-
-        return match;
-    }
-
-    public byte[] readChromatogramBytes(ChromGroupHeaderInfo header) throws DataFormatException, IOException
-    {
-        // Get the compressed bytes
-        ByteBuffer buffer = ByteBuffer.allocate(header.getCompressedSize());
-        getChannel().position(header.getLocationPoints()).read(buffer);
-        buffer.position(0);
-        byte[] result = new byte[header.getCompressedSize()];
-        buffer.get(result);
-        // Make sure it uncompresses successfully so that we don't import bad content into the database
-        uncompress(result, header.getUncompressedSize());
-        return result;
+        return matchCount;
     }
 
     public static byte[] uncompressStoredBytes(byte[] bytes, Integer uncompressedSize, int numPoints, int numTransitions) throws DataFormatException

src/org/labkey/targetedms/parser/SkylineDocumentParser.java

@@ -3179,125 +3179,133 @@ private List<ChromGroupHeaderInfo> tryLoadChromatogram(
             GeneralPrecursor<?> precursor,
             double tolerance)
     {
-        // Add precursor matches to a list, if they match at least 1 transition
-        // in this group, and are potentially the maximal transition match.
-
-        // Using only the maximum works well for the case where there are 2
-        // precursors in the same document that match a single entry.
-        // TODO: But it messes up when there are 2 sets of transitions for
-        //       the same precursor covering different numbers of transitions.
-        //       Skyline never creates this case, but it has been reported
-        // int maxTranMatch = 1;
-
         if (_binaryParser != null && _binaryParser.getChromatograms() != null)
         {
             // ChromatogramCache.TryLoadChromInfo() in Skyline code:
             // Filter the list of chromatograms based on our precursor mZ
-            int i = findEntry(precursor.getSignedMz(), tolerance, _binaryParser.getChromatograms(), 0, _binaryParser.getChromatograms().length - 1);
+            ChromGroupHeaderInfo[] chromHeaders = _binaryParser.getChromatograms();
+            int i = findEntry(precursor.getSignedMz(), tolerance, chromHeaders, 0, chromHeaders.length - 1);
             if (i == -1)
             {
                 return Collections.emptyList();
             }
 
             Double explicitRT = molecule.getExplicitRetentionTime();
 
+            List<ChromGroupHeaderInfo> result = new ArrayList<>();
+
             // Add entries to a list until they no longer match
-            List<ChromGroupHeaderInfo> listChromatograms = new ArrayList<>();
-            while (i < _binaryParser.getChromatograms().length &&
-                    matchMz(precursor.getSignedMz(), _binaryParser.getChromatograms()[i].getPrecursor(), tolerance))
+            while (i < chromHeaders.length &&
+                    matchMz(precursor.getSignedMz(), chromHeaders[i].getPrecursor(), tolerance))
             {
-                ChromGroupHeaderInfo chrom = _binaryParser.getChromatograms()[i++];
+                ChromGroupHeaderInfo chrom = chromHeaders[i++];
+                // If explicit retention time info is available, use that to discard obvious mismatches
+                if (explicitRT != null && chrom.excludesTime(explicitRT))
+                {
+                    continue;
+                }
+
                 // Sequence matching for extracted chromatogram data added in v1.5
                 ChromatogramGroupId chromTextId = _binaryParser.getTextId(chrom);
-                if (chromTextId != null) 
+                if (chromTextId != null)
                 {
+                    // If we match based on textId, consider it a chromatogram worth storing
                     if (!molecule.targetMatches(chromTextId.getTarget()))
+                    {
                         continue;
+                    }
                     try
                     {
                         SpectrumFilter spectrumFilter = SpectrumFilter.fromByteArray(precursor.getSpectrumFilter());
-                        if (!Objects.equals(spectrumFilter, chromTextId.getSpectrumFilter())) 
+                        if (!Objects.equals(spectrumFilter, chromTextId.getSpectrumFilter()))
                         {
                             continue;
                         }
                     }
                     catch (InvalidProtocolBufferException e)
                     {
-                        _log.warn("Error parsing spectrum filter {}", e);
-                        return Collections.emptyList();
+                        _log.warn("Error parsing spectrum filter", e);
+                        continue;
                     }
                 }
-
-                // If explicit retention time info is available, use that to discard obvious mismatches
-                if (explicitRT == null || !chrom.excludesTime(explicitRT))
-                {
-                    listChromatograms.add(chrom);
-                }
+                result.add(chrom);
             }
+            return findChromatogramsWithMostTransitions(precursor.getMz(), transitions, result);
+        }
 
-            // MeasuredResults.TryLoadChromatogram in Skyline code:
-            // Since we are reading and returning chromatograms for all replicates we need to maintain
-            // the number of maximum transition matches for each replicate.
-            // MeasuredResults.TryLoadChromatogram in Skyline reads and returns chromatograms for a single replicate.
-            int[] maxTranMatches = new int[_binaryParser.getCacheFileSize()];
+        return Collections.emptyList();
+    }
 
-            ChromGroupHeaderInfo[] chromArray = new ChromGroupHeaderInfo[_binaryParser.getCacheFileSize()];
 
-            for (ChromGroupHeaderInfo chromInfo : listChromatograms)
-            {
-                // If the chromatogram set has an optimization function then the number
-                // of matching chromatograms per transition is a reflection of better
-                // matching.  Otherwise, we only expect one match per transition.
-                // TODO - do we need this on the Java side?
-                boolean multiMatch = false;//chromatogram.OptimizationFunction != null;
+    /**
+     * Within each replicate, if there is more than one ChromGroupHeaderInfo, find the header infos with the most matching
+     * transitions, and, if there are multiple headers with the same number of matching transitions, then find the ones
+     * that have the closest match to the precursor m/z.
+     */
+    private List<ChromGroupHeaderInfo> findChromatogramsWithMostTransitions(
+            double precursorMz, List<? extends GeneralTransition> transitions, List<ChromGroupHeaderInfo> headerInfos)
+    {
+        Map<String, List<ChromGroupHeaderInfo>> byFile = headerInfos.stream()
+                .collect(Collectors.groupingBy(headerInfo -> _binaryParser.getFilePath(headerInfo)));
 
-                int tranMatch = _binaryParser.matchTransitions(chromInfo, transitions, explicitRT, tolerance, multiMatch);
+        List<ChromGroupHeaderInfo> result = new ArrayList<>();
 
-                int fileIndex = chromInfo.getFileIndex();
-                int maxTranMatch = maxTranMatches[fileIndex];
+        for (SkylineReplicate skylineReplicate : _replicateList)
+        {
+            List<ChromGroupHeaderInfo> candidates = new ArrayList<>();
 
-                if (tranMatch >= maxTranMatch)
+            for (SampleFile sampleFile : skylineReplicate.getSampleFileList())
+            {
+                List<ChromGroupHeaderInfo> listInFile = byFile.get(sampleFile.getFilePath());
+                if (listInFile != null)
                 {
-                    // If new maximum, clear anything collected at the previous maximum
-                    if (tranMatch > maxTranMatch)
-                    {
-                        chromArray[fileIndex] = null;
-                    }
+                    candidates.addAll(listInFile);
+                }
+            }
 
-                    maxTranMatches[fileIndex] = tranMatch;
+            if (candidates.size() <= 1)
+            {
+                result.addAll(candidates);
+                continue;
+            }
 
-                    if(chromArray[fileIndex] != null)
-                    {
-                        // If more than one value was found, ensure that there
-                        // is only one precursor match per file.
-                        // Use the entry with the m/z closest to the target
-                        ChromGroupHeaderInfo currentChromForFileIndex = chromArray[fileIndex];
-                        // Use the entry with the m/z closest to the target
-                        if (Math.abs(precursor.getMz() - chromInfo.getPrecursorMz()) <
-                                Math.abs(precursor.getMz() - currentChromForFileIndex.getPrecursorMz()))
-                        {
-                            chromArray[fileIndex] = chromInfo;
-                        }
-                    }
-                    else
-                    {
-                        chromArray[fileIndex] = chromInfo;
-                    }
+            int[] transitionCounts = new int[candidates.size()];
+            int maxTransitionCount = 0;
+            for (int i = 0; i < candidates.size(); i++)
+            {
+                int transitionCount = _binaryParser.countTransitionMatches(candidates.get(i), transitions, _matchTolerance);
+                transitionCounts[i] = transitionCount;
+                maxTransitionCount = Math.max(transitionCount, maxTransitionCount);
+            }
+            List<ChromGroupHeaderInfo> candidatesWithMostTransitions = new ArrayList<ChromGroupHeaderInfo>();
+            for (int i = 0; i < candidates.size(); i++)
+            {
+                if (transitionCounts[i] == maxTransitionCount)
+                {
+                    candidatesWithMostTransitions.add(candidates.get(i));
                 }
             }
+            if (candidatesWithMostTransitions.size() <= 1)
+            {
+                result.addAll(candidatesWithMostTransitions);
+                continue;
+            }
 
-            List<ChromGroupHeaderInfo> finalList = new ArrayList<>();
-            for (ChromGroupHeaderInfo info : chromArray)
+            // If multiple chromGroups tied for the number of matching transitions, then break the tie using
+            // precursor m/z distance
+            double minMzDelta = candidatesWithMostTransitions.stream()
+                    .mapToDouble(headerInfo -> Math.abs(precursorMz - headerInfo.getPrecursorMz()))
+                    .min().getAsDouble();
+            for (ChromGroupHeaderInfo headerInfo : candidatesWithMostTransitions)
             {
-                if (info != null)
+                if (Math.abs(precursorMz - headerInfo.getPrecursorMz()) == minMzDelta)
                 {
-                    finalList.add(info);
+                    result.add(headerInfo);
                 }
             }
-            return finalList;
         }
 
-        return Collections.emptyList();
+        return result;
     }
 
     public int getPeptideGroupCount()

webapp/TargetedMS/js/QCSummaryPanel.js

@@ -299,7 +299,17 @@ Ext4.define('LABKEY.targetedms.QCSummary', {
     renderContainerSampleFileStats: function (params) {
         let container = params.container;
         let sampleFiles = params.sampleFiles;
-        let metrics = sampleFiles[0].Metrics;
+        let metrics = [];
+        let seenMetrics = {};
+        Ext4.iterate(sampleFiles, function (sampleFile) {
+            Ext4.iterate(sampleFile.Metrics, function (metric) {
+                if (!seenMetrics[metric.MetricId]) {
+                    metrics.push(metric);
+                    seenMetrics[metric.MetricId] = true;
+                }
+            });
+        });
+
         let showMetrics = LABKEY.ActionURL.getAction().toLowerCase() === 'qcSummaryHistory'.toLowerCase();
         let tableWidth = container.width - 100;
         let html = '';
@@ -340,8 +350,17 @@ Ext4.define('LABKEY.targetedms.QCSummary', {
 
 
             if (showMetrics) {
-                Ext4.each(sampleFile.Metrics, function (metric) {
-                    html += '<td><div class="sample-file-item" style="text-align: right">' + Ext4.util.Format.htmlEncode(metric.Value) + '</div></td>';
+                Ext4.each(metrics, function (metric) {
+                    let isMetricPresent = false;
+                    Ext4.each(sampleFile.Metrics, function (item) {
+                        if (metric.MetricId === item.MetricId) {
+                            isMetricPresent = true;
+                            html += '<td><div class="sample-file-item" style="text-align: right">' + Ext4.util.Format.htmlEncode(item.Value) + '</div></td>';
+                        }
+                    });
+                    if (!isMetricPresent) {
+                        html += '<td><div class="sample-file-item" style="text-align: right">N/A</div></td>';
+                    }
                 });
             }
             if (sampleFile.IgnoreForAllMetric) {