[bot] Fast-forward for 25.11.13

resources/queries/targetedms/PTMPercentsGrouped.sql

@@ -18,7 +18,7 @@ SELECT
     MAX(ModificationCount) AS ModificationCount @hidden
 
 FROM
-    PTMPercentsGroupedPrepivot
+    PTMPercentsGroupedPrepivotCache
 GROUP BY
     ReplicateName,
     Sequence,

resources/queries/targetedms/PTMPercentsGroupedPrepivot/.qview.xml

@@ -0,0 +1,7 @@
+<customView xmlns="http://labkey.org/data/xml/queryCustomView" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+    <sorts>
+        <sort column="Modification" />
+        <sort column="Sequence" />
+        <sort column="ReplicateName" />
+    </sorts>
+</customView>

resources/queries/targetedms/PTMPercentsGroupedPrepivotCache.query.xml

@@ -0,0 +1,43 @@
+<query xmlns="http://labkey.org/data/xml/query">
+    <metadata>
+        <tables xmlns="http://labkey.org/data/xml">
+            <table tableName="PTMPercentsGroupedPrepivotCache" tableDbType="TABLE">
+                <columns>
+                    <column columnName="PeptideModifiedSequence">
+                        <displayColumnFactory>
+                            <className>org.labkey.targetedms.query.ModifiedSequenceDisplayColumn$PeptideDisplayColumnFactory</className>
+                            <properties>
+                                <property name="showNextAndPrevious">true</property>
+                                <property name="useParens">true</property>
+                                <property name="modificationSite">SiteLocation</property>
+                            </properties>
+                        </displayColumnFactory>
+                        <columnTitle>Sequence</columnTitle>
+                    </column>
+                    <column columnName="TotalPercentModified">
+                        <formatString>0.00%</formatString>
+                    </column>
+                    <column columnName="PercentModified">
+                        <formatString>0.00%</formatString>
+                    </column>
+                    <column columnName="MaxPercentModified">
+                        <formatString>0.00%</formatString>
+                    </column>
+                    <column columnName="Sequence">
+                        <columnTitle>UnmodifiedSequence</columnTitle>
+                    </column>
+                    <column columnName="IsCdr">
+                        <displayColumnFactory>
+                           <className>org.labkey.targetedms.query.CDRDisplayColumnFactory</className>
+                        </displayColumnFactory>
+                    </column>
+                    <column columnName="Risk">
+                        <displayColumnFactory>
+                           <className>org.labkey.targetedms.query.PTMRiskDisplayColumnFactory</className>
+                        </displayColumnFactory>
+                    </column>
+                </columns>
+            </table>
+        </tables>
+    </metadata>
+</query>

resources/queries/targetedms/PTMPercentsGroupedPrepivotCache/.qview.xml

@@ -0,0 +1,7 @@
+<customView xmlns="http://labkey.org/data/xml/queryCustomView" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+    <sorts>
+        <sort column="Modification" />
+        <sort column="Sequence" />
+        <sort column="ReplicateName" />
+    </sorts>
+</customView>

resources/schemas/dbscripts/postgresql/targetedms-26.001-26.002.sql

@@ -0,0 +1,35 @@
+CREATE TABLE targetedms.PTMPercentsGroupedPrepivotCache
+(
+    Id                  BIGINT NOT NULL, -- PeptideId lookup, not a primary key
+    Container           ENTITYID NOT NULL,
+    RunId               BIGINT NOT NULL,
+    Modification        VARCHAR(300) NOT NULL,
+    TotalPercentModified REAL,
+    PercentModified     REAL,
+    MaxPercentModified  REAL,
+    ModificationCount   INT,
+    PeptideModifiedSequence VARCHAR(300),
+    Sequence            VARCHAR(300),
+    PreviousAA          VARCHAR(2),
+    NextAA              VARCHAR(2),
+    SampleFileId        BIGINT NOT NULL,
+    ReplicateName       VARCHAR(200),
+    AminoAcid           VARCHAR(5),
+    SiteLocation        VARCHAR(50),
+    Location            INT,
+    PeptideGroupId      BIGINT NOT NULL,
+
+    CONSTRAINT FK_PTMPercentsGroupedPrepivotCache_Container FOREIGN KEY (Container) REFERENCES core.Containers(EntityId),
+    CONSTRAINT FK_PTMPercentsGroupedPrepivotCache_Id FOREIGN KEY (Id) REFERENCES targetedms.Peptide(Id),
+    CONSTRAINT FK_PTMPercentsGroupedPrepivotCache_RunId FOREIGN KEY (RunId) REFERENCES targetedms.Runs(Id),
+    CONSTRAINT FK_PTMPercentsGroupedPrepivotCache_SampleFileId FOREIGN KEY (SampleFileId) REFERENCES targetedms.SampleFile(Id),
+    CONSTRAINT FK_PTMPercentsGroupedPrepivotCache_PeptideGroupId FOREIGN KEY (PeptideGroupId) REFERENCES targetedms.PeptideGroup(Id)
+);
+
+CREATE INDEX IDX_PTMPercentsGroupedPrepivotCache_Id ON targetedms.PTMPercentsGroupedPrepivotCache(Id);
+CREATE INDEX IDX_PTMPercentsGroupedPrepivotCache_RunId ON targetedms.PTMPercentsGroupedPrepivotCache(RunId);
+CREATE INDEX IDX_PTMPercentsGroupedPrepivotCache_Container ON targetedms.PTMPercentsGroupedPrepivotCache(Container);
+CREATE INDEX IDX_PTMPercentsGroupedPrepivotCache_SampleFileId ON targetedms.PTMPercentsGroupedPrepivotCache(SampleFileId);
+CREATE INDEX IDX_PTMPercentsGroupedPrepivotCache_PeptideGroupId ON targetedms.PTMPercentsGroupedPrepivotCache(PeptideGroupId);
+
+SELECT core.executeJavaUpgradeCode('populatePTMPercentsGroupedPrepivotCache');

resources/schemas/targetedms.xml

@@ -1950,4 +1950,26 @@
             <column columnName="SeriesLabel"/>
         </columns>
     </table>
+    <table tableName="PTMPercentsGroupedPrepivotCache" tableDbType="TABLE">
+        <columns>
+            <column columnName="Id"/> <!-- PeptideId lookup, not a primary key -->
+            <column columnName="Container"/>
+            <column columnName="RunId"/>
+            <column columnName="Modification"/>
+            <column columnName="TotalPercentModified"/>
+            <column columnName="PercentModified"/>
+            <column columnName="MaxPercentModified"/>
+            <column columnName="ModificationCount"/>
+            <column columnName="PeptideModifiedSequence"/>
+            <column columnName="Sequence"/>
+            <column columnName="PreviousAA"/>
+            <column columnName="NextAA"/>
+            <column columnName="SampleFileId"/>
+            <column columnName="ReplicateName"/>
+            <column columnName="AminoAcid"/>
+            <column columnName="SiteLocation"/>
+            <column columnName="Location"/>
+            <column columnName="PeptideGroupId"/>
+        </columns>
+    </table>
 </tables>

src/org/labkey/targetedms/SkylineDocImporter.java

@@ -514,6 +514,7 @@ else if (folderType == TargetedMSService.FolderType.Library)
             parser.logMissingChromatogramCounts();
 
             TargetedMSManager.updateModifiedAreaProportions(_log, run);
+            TargetedMSManager.populatePTMPercentsGroupedPrepivotCache(run, _user, _container);
 
             if (_pipeRoot.isCloudRoot())
                 copyExtractedFilesToCloud(run);

src/org/labkey/targetedms/TargetedMSManager.java

@@ -602,6 +602,11 @@ public static TableInfo getTableInfoQCMetricCache()
         return getSchema().getTable(TargetedMSSchema.TABLE_QC_METRIC_CACHE);
     }
 
+    public static TableInfo getTableInfoPTMPercentsGroupedPrepivotCache()
+    {
+        return getSchema().getTable(TargetedMSSchema.TABLE_PTM_PERCENTS_GROUPED_PREPIVOT_CACHE);
+    }
+
     public static TableInfo getTableInfoSkylineAuditLogEntry()
     {
         return getSchema().getTable(TargetedMSSchema.TABLE_SKYLINE_AUDITLOG_ENTRY);
@@ -1029,6 +1034,12 @@ public static TargetedMSRun[] getRunsInContainer(Container container)
                 container.getId(), SkylineDocImporter.STATUS_SUCCESS, Boolean.FALSE);
     }
 
+    public static TargetedMSRun[] getAllNonDeletedRuns()
+    {
+        return getRuns("StatusId=? AND deleted=?",
+                SkylineDocImporter.STATUS_SUCCESS, Boolean.FALSE);
+    }
+
     @Nullable
     public static TargetedMSRun getRunByFileName(String fileName, Container container)
     {
@@ -1762,6 +1773,7 @@ private static SQLFragment getTempChromInfoIdsDependentDeleteSql(TableInfo fromT
     /** Actually delete runs that have been marked as deleted from the database */
     private static void purgeDeletedRuns()
     {
+        deleteRunDependent(getTableInfoPTMPercentsGroupedPrepivotCache());
         // Delete from FoldChange
         deleteRunDependent(getTableInfoFoldChange());
         // Delete from CalibrationCurve
@@ -2732,6 +2744,63 @@ public static void updateModifiedAreaProportions(@Nullable Logger log, @NotNull
         executor.execute("DROP TABLE " + areasTableName);
     }
 
+    /**
+     * Pre-compute PTMPercentsGroupedPrepivot results during import and store in PTMPercentsCache.
+     * Only populates cache for ExperimentMAM folders.
+     */
+    public static void populatePTMPercentsGroupedPrepivotCache(@NotNull TargetedMSRun run, @NotNull User user, @NotNull Container container)
+    {
+        // Delete any existing cache rows for this run
+        new SqlExecutor(getSchema()).execute(
+                new SQLFragment("DELETE FROM ").append(getTableInfoPTMPercentsGroupedPrepivotCache()).append(" WHERE RunId = ?").add(run.getId()));
+
+        // Only populate cache for ExperimentMAM folders
+        if (getFolderType(container) != TargetedMSService.FolderType.ExperimentMAM)
+        {
+            return;
+        }
+
+        _log.info("Populating PTMPercentsGroupedPrepivotCache for run " + run.getId());
+
+        String labkeySql = "SELECT\n" +
+                "  Modification,\n" +
+                "  TotalPercentModified,\n" +
+                "  PercentModified,\n" +
+                "  MaxPercentModified,\n" +
+                "  ModificationCount,\n" +
+                "  Id,\n" +
+                "  PeptideModifiedSequence,\n" +
+                "  Sequence,\n" +
+                "  PreviousAA,\n" +
+                "  NextAA,\n" +
+                "  SampleFileId,\n" +
+                "  ReplicateName,\n" +
+                "  AminoAcid,\n" +
+                "  SiteLocation,\n" +
+                "  Location,\n" +
+                "  PeptideGroupId\n" +
+                "FROM PTMPercentsGroupedPrepivot\n" +
+                "WHERE PeptideGroupId.RunId = " + run.getId();
+
+        UserSchema schema = QueryService.get().getUserSchema(user, container, TargetedMSSchema.SCHEMA_KEY);
+        TableInfo tableInfo = QueryService.get().createTable(schema, labkeySql, null, true);
+
+        SQLFragment insertSql = new SQLFragment();
+        insertSql.append("INSERT INTO ").append(getTableInfoPTMPercentsGroupedPrepivotCache());
+        insertSql.append(" (Container, RunId, Modification, TotalPercentModified, PercentModified, MaxPercentModified,");
+        insertSql.append(" ModificationCount, Id, PeptideModifiedSequence, Sequence,");
+        insertSql.append(" PreviousAA, NextAA, SampleFileId, ReplicateName, AminoAcid, SiteLocation, Location, PeptideGroupId)");
+        insertSql.append(" SELECT ?, ?, lk.Modification, lk.TotalPercentModified, lk.PercentModified, lk.MaxPercentModified,");
+        insertSql.append(" lk.ModificationCount, lk.Id, lk.PeptideModifiedSequence, lk.Sequence,");
+        insertSql.append(" lk.PreviousAA, lk.NextAA, lk.SampleFileId, lk.ReplicateName, lk.AminoAcid, lk.SiteLocation, lk.Location, lk.PeptideGroupId");
+        insertSql.append(" FROM ").append(tableInfo, "lk");
+        insertSql.add(container.getEntityId());
+        insertSql.add(run.getId());
+        new SqlExecutor(getSchema()).execute(insertSql);
+
+        _log.info("Finished populating PTMPercentsGroupedPrepivotCache for run " + run.getId());
+    }
+
     /**
      * Returns the Transition Full Scan settings for the given run.  This may be null if the settings were not included
      * in the Skyline document.

src/org/labkey/targetedms/TargetedMSModule.java

@@ -231,7 +231,7 @@ public String getName()
     @Override
     public Double getSchemaVersion()
     {
-        return 26.001;
+        return 26.002;
     }
 
     @Override

src/org/labkey/targetedms/TargetedMSSchema.java

@@ -71,6 +71,7 @@
 import org.labkey.api.security.permissions.AdminPermission;
 import org.labkey.api.targetedms.RepresentativeDataState;
 import org.labkey.api.targetedms.RunRepresentativeDataState;
+import org.labkey.api.targetedms.TargetedMSService;
 import org.labkey.api.util.ContainerContext;
 import org.labkey.api.util.HtmlString;
 import org.labkey.api.util.Pair;
@@ -232,6 +233,7 @@ public class TargetedMSSchema extends UserSchema
     public static final String TABLE_QC_ENABLED_METRICS = "QCEnabledMetrics";
     public static final String TABLE_QC_TRACE_METRIC_VALUES = "QCTraceMetricValues";
     public static final String TABLE_QC_METRIC_CACHE = "QCMetricCache";
+    public static final String TABLE_PTM_PERCENTS_GROUPED_PREPIVOT_CACHE = "PTMPercentsGroupedPrepivotCache";
 
     public static final String TABLE_GUIDE_SET = "GuideSet";
 
@@ -1689,6 +1691,14 @@ else if (TABLE_INSTRUMENT_USAGE_PAYMENT.equalsIgnoreCase(name))
                 result.getMutableColumnOrThrow("ExperimentRunLSID").setFk(QueryForeignKey.from(_expSchema, cf).to("Runs", "LSID", null));
             }
             TargetedMSTable.fixupLookups(result);
+
+            if (name.equalsIgnoreCase(TABLE_PTM_PERCENTS_GROUPED_PREPIVOT_CACHE))
+            {
+                // Inject two null columns to match how the uncached version wires up DisplayColumns
+                result.addColumn(new ExprColumn(result, "Risk", new SQLFragment("CAST(NULL AS VARCHAR)"), JdbcType.VARCHAR));
+                result.addColumn(new ExprColumn(result, "IsCdr", new SQLFragment("CAST(NULL AS VARCHAR)"), JdbcType.VARCHAR));
+            }
+
             return result;
         }
 
@@ -1882,6 +1892,7 @@ private Set<String> getAllTableNames(boolean caseInsensitive)
         hs.add(TABLE_RATE_TYPE);
         hs.add(TABLE_INSTRUMENT_RATE);
         hs.add(TABLE_INSTRUMENT_USAGE_PAYMENT);
+        hs.add(TABLE_PTM_PERCENTS_GROUPED_PREPIVOT_CACHE);
 
         return hs;
     }
@@ -1921,6 +1932,11 @@ private QueryDefinition createRunScopedPTMQuery(String prefix, String baseQueryN
         try
         {
             long runId = Long.parseLong(runIdString);
+            if (TargetedMSManager.getFolderType(getContainer()) != TargetedMSService.FolderType.ExperimentMAM)
+            {
+                throw new IllegalStateException("PTM queries are only supported in ExperimentMAM folders");
+            }
+
             QueryDefinition queryDef = Objects.requireNonNull(getQueryDef(baseQueryName));
             QueryDefinition result = QueryService.get().createQueryDef(getUser(), getContainer(), getSchemaPath(), queryName);
             result.setSql(queryDef.getSql() + " IN (SELECT r.Name FROM targetedms.Replicate r WHERE r.RunId = " + runId + ")");

src/org/labkey/targetedms/TargetedMSUpgradeCode.java

@@ -15,14 +15,19 @@
  */
 package org.labkey.targetedms;
 
+import org.apache.logging.log4j.Logger;
+import org.labkey.api.data.Container;
 import org.labkey.api.data.ContainerManager;
+import org.labkey.api.data.DeferredUpgrade;
 import org.labkey.api.data.SQLFragment;
 import org.labkey.api.data.SqlExecutor;
 import org.labkey.api.data.UpgradeCode;
 import org.labkey.api.module.Module;
 import org.labkey.api.module.ModuleContext;
 import org.labkey.api.module.ModuleLoader;
 import org.labkey.api.security.User;
+import org.labkey.api.targetedms.TargetedMSService;
+import org.labkey.api.util.logging.LogHelper;
 import org.labkey.targetedms.query.QCAnnotationTypeTable;
 
 import java.util.Date;
@@ -36,6 +41,8 @@
  */
 public class TargetedMSUpgradeCode implements UpgradeCode
 {
+    private static final Logger LOG = LogHelper.getLogger(TargetedMSUpgradeCode.class, "TargetedMS schema upgrade operations");
+
     // called at every bootstrap to initialize annotation types
     @SuppressWarnings({"UnusedDeclaration"})
     public void populateDefaultAnnotationTypes(final ModuleContext moduleContext)
@@ -69,4 +76,36 @@ private void insertAnnotationType(String name, String color, User user)
         sql.add(color);
         new SqlExecutor(TargetedMSManager.getSchema()).execute(sql);
     }
+
+    /** Populate PTMPercentsGroupedPrepivotCache for all runs in existing ExperimentMAM folders */
+    @SuppressWarnings("UnusedDeclaration")
+    @DeferredUpgrade
+    public void populatePTMPercentsGroupedPrepivotCache(final ModuleContext moduleContext)
+    {
+        if (moduleContext.isNewInstall())
+        {
+            return;
+        }
+
+        User user = moduleContext.getUpgradeUser();
+        LOG.info("Populating PTMPercentsGroupedPrepivotCache for existing ExperimentMAM folders");
+
+        for (TargetedMSRun run : TargetedMSManager.getAllNonDeletedRuns())
+        {
+            Container container = run.getContainer();
+            if (container != null && TargetedMSManager.getFolderType(container) == TargetedMSService.FolderType.ExperimentMAM)
+            {
+                try
+                {
+                    TargetedMSManager.populatePTMPercentsGroupedPrepivotCache(run, user, container);
+                }
+                catch (Exception e)
+                {
+                    LOG.error("Error populating PTMPercentsGroupedPrepivotCache for run " + run.getId() + " in " + container.getPath(), e);
+                }
+            }
+        }
+
+        LOG.info("Finished populating PTMPercentsGroupedPrepivotCache for existing ExperimentMAM folders");
+    }
 }

test/src/org/labkey/test/tests/targetedms/TargetedMSEarlyStagePTMReportTest.java

@@ -39,10 +39,20 @@ private void doInit()
     @Test
     public void testEarlyStagePrepivot()
     {
+        // Test against the live-query version
         goToProjectHome();
         goToSchemaBrowser();
         DataRegionTable table = viewQueryData("targetedms", "PTMPercentsGroupedPrepivot");
+        verifyPrepivotData(table);
 
+        // Test against the cached version too
+        goToSchemaBrowser();
+        table = viewQueryData("targetedms", "PTMPercentsGroupedPrepivotCache");
+        verifyPrepivotData(table);
+    }
+
+    private void verifyPrepivotData(DataRegionTable table)
+    {
         // Test special-cased peptide
         table.setFilter("PeptideModifiedSequence", "Starts With", "EEQ");
         table = new DataRegionTable("query", this);
@@ -59,7 +69,7 @@ public void testEarlyStagePrepivot()
         table.setFilter("PeptideModifiedSequence", "Starts With", "VTN");
         table = new DataRegionTable("query", this);
         assertEquals(List.of("true", "true"), table.getColumnDataAsText("IsCdr"));
-        assertEquals(List.of("High", "Medium"), table.getColumnDataAsText("Risk"));
+        assertEquals(List.of("Medium", "High"), table.getColumnDataAsText("Risk"));
 
         // Test special-cased N-Term Modification, present on QVTL peptide (Q is modified, so don't use it in the filter)
         table.setFilter("PeptideModifiedSequence", "Contains", "VTL");