diff --git a/resources/queries/targetedms/PTMPercentsGrouped.sql b/resources/queries/targetedms/PTMPercentsGrouped.sql index a196a9cdd..42402a76e 100644 --- a/resources/queries/targetedms/PTMPercentsGrouped.sql +++ b/resources/queries/targetedms/PTMPercentsGrouped.sql @@ -18,7 +18,7 @@ SELECT MAX(ModificationCount) AS ModificationCount @hidden FROM - PTMPercentsGroupedPrepivot + PTMPercentsGroupedPrepivotCache GROUP BY ReplicateName, Sequence, diff --git a/resources/queries/targetedms/PTMPercentsGroupedPrepivot/.qview.xml b/resources/queries/targetedms/PTMPercentsGroupedPrepivot/.qview.xml new file mode 100644 index 000000000..0b110ac4f --- /dev/null +++ b/resources/queries/targetedms/PTMPercentsGroupedPrepivot/.qview.xml @@ -0,0 +1,7 @@ + + + + + + + \ No newline at end of file diff --git a/resources/queries/targetedms/PTMPercentsGroupedPrepivotCache.query.xml b/resources/queries/targetedms/PTMPercentsGroupedPrepivotCache.query.xml new file mode 100644 index 000000000..191fa354a --- /dev/null +++ b/resources/queries/targetedms/PTMPercentsGroupedPrepivotCache.query.xml @@ -0,0 +1,43 @@ + + + + + + + + org.labkey.targetedms.query.ModifiedSequenceDisplayColumn$PeptideDisplayColumnFactory + + true + true + SiteLocation + + + Sequence + + + 0.00% + + + 0.00% + + + 0.00% + + + UnmodifiedSequence + + + + org.labkey.targetedms.query.CDRDisplayColumnFactory + + + + + org.labkey.targetedms.query.PTMRiskDisplayColumnFactory + + + +
+ + + diff --git a/resources/queries/targetedms/PTMPercentsGroupedPrepivotCache/.qview.xml b/resources/queries/targetedms/PTMPercentsGroupedPrepivotCache/.qview.xml new file mode 100644 index 000000000..0b110ac4f --- /dev/null +++ b/resources/queries/targetedms/PTMPercentsGroupedPrepivotCache/.qview.xml @@ -0,0 +1,7 @@ + + + + + + + \ No newline at end of file diff --git a/resources/schemas/dbscripts/postgresql/targetedms-26.001-26.002.sql b/resources/schemas/dbscripts/postgresql/targetedms-26.001-26.002.sql new file mode 100644 index 000000000..59d26d370 --- /dev/null +++ b/resources/schemas/dbscripts/postgresql/targetedms-26.001-26.002.sql @@ -0,0 +1,35 @@ +CREATE TABLE targetedms.PTMPercentsGroupedPrepivotCache +( + Id BIGINT NOT NULL, -- PeptideId lookup, not a primary key + Container ENTITYID NOT NULL, + RunId BIGINT NOT NULL, + Modification VARCHAR(300) NOT NULL, + TotalPercentModified REAL, + PercentModified REAL, + MaxPercentModified REAL, + ModificationCount INT, + PeptideModifiedSequence VARCHAR(300), + Sequence VARCHAR(300), + PreviousAA VARCHAR(2), + NextAA VARCHAR(2), + SampleFileId BIGINT NOT NULL, + ReplicateName VARCHAR(200), + AminoAcid VARCHAR(5), + SiteLocation VARCHAR(50), + Location INT, + PeptideGroupId BIGINT NOT NULL, + + CONSTRAINT FK_PTMPercentsGroupedPrepivotCache_Container FOREIGN KEY (Container) REFERENCES core.Containers(EntityId), + CONSTRAINT FK_PTMPercentsGroupedPrepivotCache_Id FOREIGN KEY (Id) REFERENCES targetedms.Peptide(Id), + CONSTRAINT FK_PTMPercentsGroupedPrepivotCache_RunId FOREIGN KEY (RunId) REFERENCES targetedms.Runs(Id), + CONSTRAINT FK_PTMPercentsGroupedPrepivotCache_SampleFileId FOREIGN KEY (SampleFileId) REFERENCES targetedms.SampleFile(Id), + CONSTRAINT FK_PTMPercentsGroupedPrepivotCache_PeptideGroupId FOREIGN KEY (PeptideGroupId) REFERENCES targetedms.PeptideGroup(Id) +); + +CREATE INDEX IDX_PTMPercentsGroupedPrepivotCache_Id ON targetedms.PTMPercentsGroupedPrepivotCache(Id); +CREATE INDEX IDX_PTMPercentsGroupedPrepivotCache_RunId ON targetedms.PTMPercentsGroupedPrepivotCache(RunId); +CREATE INDEX IDX_PTMPercentsGroupedPrepivotCache_Container ON targetedms.PTMPercentsGroupedPrepivotCache(Container); +CREATE INDEX IDX_PTMPercentsGroupedPrepivotCache_SampleFileId ON targetedms.PTMPercentsGroupedPrepivotCache(SampleFileId); +CREATE INDEX IDX_PTMPercentsGroupedPrepivotCache_PeptideGroupId ON targetedms.PTMPercentsGroupedPrepivotCache(PeptideGroupId); + +SELECT core.executeJavaUpgradeCode('populatePTMPercentsGroupedPrepivotCache'); diff --git a/resources/schemas/targetedms.xml b/resources/schemas/targetedms.xml index 2bd32a2d8..569b890e1 100644 --- a/resources/schemas/targetedms.xml +++ b/resources/schemas/targetedms.xml @@ -1950,4 +1950,26 @@ + + + + + + + + + + + + + + + + + + + + + +
diff --git a/src/org/labkey/targetedms/SkylineDocImporter.java b/src/org/labkey/targetedms/SkylineDocImporter.java index 5e6f30648..3ceca58f6 100644 --- a/src/org/labkey/targetedms/SkylineDocImporter.java +++ b/src/org/labkey/targetedms/SkylineDocImporter.java @@ -514,6 +514,7 @@ else if (folderType == TargetedMSService.FolderType.Library) parser.logMissingChromatogramCounts(); TargetedMSManager.updateModifiedAreaProportions(_log, run); + TargetedMSManager.populatePTMPercentsGroupedPrepivotCache(run, _user, _container); if (_pipeRoot.isCloudRoot()) copyExtractedFilesToCloud(run); diff --git a/src/org/labkey/targetedms/TargetedMSManager.java b/src/org/labkey/targetedms/TargetedMSManager.java index 2d55db0fd..64ed12e47 100644 --- a/src/org/labkey/targetedms/TargetedMSManager.java +++ b/src/org/labkey/targetedms/TargetedMSManager.java @@ -602,6 +602,11 @@ public static TableInfo getTableInfoQCMetricCache() return getSchema().getTable(TargetedMSSchema.TABLE_QC_METRIC_CACHE); } + public static TableInfo getTableInfoPTMPercentsGroupedPrepivotCache() + { + return getSchema().getTable(TargetedMSSchema.TABLE_PTM_PERCENTS_GROUPED_PREPIVOT_CACHE); + } + public static TableInfo getTableInfoSkylineAuditLogEntry() { return getSchema().getTable(TargetedMSSchema.TABLE_SKYLINE_AUDITLOG_ENTRY); @@ -1029,6 +1034,12 @@ public static TargetedMSRun[] getRunsInContainer(Container container) container.getId(), SkylineDocImporter.STATUS_SUCCESS, Boolean.FALSE); } + public static TargetedMSRun[] getAllNonDeletedRuns() + { + return getRuns("StatusId=? AND deleted=?", + SkylineDocImporter.STATUS_SUCCESS, Boolean.FALSE); + } + @Nullable public static TargetedMSRun getRunByFileName(String fileName, Container container) { @@ -1762,6 +1773,7 @@ private static SQLFragment getTempChromInfoIdsDependentDeleteSql(TableInfo fromT /** Actually delete runs that have been marked as deleted from the database */ private static void purgeDeletedRuns() { + deleteRunDependent(getTableInfoPTMPercentsGroupedPrepivotCache()); // Delete from FoldChange deleteRunDependent(getTableInfoFoldChange()); // Delete from CalibrationCurve @@ -2732,6 +2744,63 @@ public static void updateModifiedAreaProportions(@Nullable Logger log, @NotNull executor.execute("DROP TABLE " + areasTableName); } + /** + * Pre-compute PTMPercentsGroupedPrepivot results during import and store in PTMPercentsCache. + * Only populates cache for ExperimentMAM folders. + */ + public static void populatePTMPercentsGroupedPrepivotCache(@NotNull TargetedMSRun run, @NotNull User user, @NotNull Container container) + { + // Delete any existing cache rows for this run + new SqlExecutor(getSchema()).execute( + new SQLFragment("DELETE FROM ").append(getTableInfoPTMPercentsGroupedPrepivotCache()).append(" WHERE RunId = ?").add(run.getId())); + + // Only populate cache for ExperimentMAM folders + if (getFolderType(container) != TargetedMSService.FolderType.ExperimentMAM) + { + return; + } + + _log.info("Populating PTMPercentsGroupedPrepivotCache for run " + run.getId()); + + String labkeySql = "SELECT\n" + + " Modification,\n" + + " TotalPercentModified,\n" + + " PercentModified,\n" + + " MaxPercentModified,\n" + + " ModificationCount,\n" + + " Id,\n" + + " PeptideModifiedSequence,\n" + + " Sequence,\n" + + " PreviousAA,\n" + + " NextAA,\n" + + " SampleFileId,\n" + + " ReplicateName,\n" + + " AminoAcid,\n" + + " SiteLocation,\n" + + " Location,\n" + + " PeptideGroupId\n" + + "FROM PTMPercentsGroupedPrepivot\n" + + "WHERE PeptideGroupId.RunId = " + run.getId(); + + UserSchema schema = QueryService.get().getUserSchema(user, container, TargetedMSSchema.SCHEMA_KEY); + TableInfo tableInfo = QueryService.get().createTable(schema, labkeySql, null, true); + + SQLFragment insertSql = new SQLFragment(); + insertSql.append("INSERT INTO ").append(getTableInfoPTMPercentsGroupedPrepivotCache()); + insertSql.append(" (Container, RunId, Modification, TotalPercentModified, PercentModified, MaxPercentModified,"); + insertSql.append(" ModificationCount, Id, PeptideModifiedSequence, Sequence,"); + insertSql.append(" PreviousAA, NextAA, SampleFileId, ReplicateName, AminoAcid, SiteLocation, Location, PeptideGroupId)"); + insertSql.append(" SELECT ?, ?, lk.Modification, lk.TotalPercentModified, lk.PercentModified, lk.MaxPercentModified,"); + insertSql.append(" lk.ModificationCount, lk.Id, lk.PeptideModifiedSequence, lk.Sequence,"); + insertSql.append(" lk.PreviousAA, lk.NextAA, lk.SampleFileId, lk.ReplicateName, lk.AminoAcid, lk.SiteLocation, lk.Location, lk.PeptideGroupId"); + insertSql.append(" FROM ").append(tableInfo, "lk"); + insertSql.add(container.getEntityId()); + insertSql.add(run.getId()); + new SqlExecutor(getSchema()).execute(insertSql); + + _log.info("Finished populating PTMPercentsGroupedPrepivotCache for run " + run.getId()); + } + /** * Returns the Transition Full Scan settings for the given run. This may be null if the settings were not included * in the Skyline document. diff --git a/src/org/labkey/targetedms/TargetedMSModule.java b/src/org/labkey/targetedms/TargetedMSModule.java index c68243f38..6d6538561 100644 --- a/src/org/labkey/targetedms/TargetedMSModule.java +++ b/src/org/labkey/targetedms/TargetedMSModule.java @@ -231,7 +231,7 @@ public String getName() @Override public Double getSchemaVersion() { - return 26.001; + return 26.002; } @Override diff --git a/src/org/labkey/targetedms/TargetedMSSchema.java b/src/org/labkey/targetedms/TargetedMSSchema.java index db2ef2cb7..a863332d8 100644 --- a/src/org/labkey/targetedms/TargetedMSSchema.java +++ b/src/org/labkey/targetedms/TargetedMSSchema.java @@ -71,6 +71,7 @@ import org.labkey.api.security.permissions.AdminPermission; import org.labkey.api.targetedms.RepresentativeDataState; import org.labkey.api.targetedms.RunRepresentativeDataState; +import org.labkey.api.targetedms.TargetedMSService; import org.labkey.api.util.ContainerContext; import org.labkey.api.util.HtmlString; import org.labkey.api.util.Pair; @@ -232,6 +233,7 @@ public class TargetedMSSchema extends UserSchema public static final String TABLE_QC_ENABLED_METRICS = "QCEnabledMetrics"; public static final String TABLE_QC_TRACE_METRIC_VALUES = "QCTraceMetricValues"; public static final String TABLE_QC_METRIC_CACHE = "QCMetricCache"; + public static final String TABLE_PTM_PERCENTS_GROUPED_PREPIVOT_CACHE = "PTMPercentsGroupedPrepivotCache"; public static final String TABLE_GUIDE_SET = "GuideSet"; @@ -1689,6 +1691,14 @@ else if (TABLE_INSTRUMENT_USAGE_PAYMENT.equalsIgnoreCase(name)) result.getMutableColumnOrThrow("ExperimentRunLSID").setFk(QueryForeignKey.from(_expSchema, cf).to("Runs", "LSID", null)); } TargetedMSTable.fixupLookups(result); + + if (name.equalsIgnoreCase(TABLE_PTM_PERCENTS_GROUPED_PREPIVOT_CACHE)) + { + // Inject two null columns to match how the uncached version wires up DisplayColumns + result.addColumn(new ExprColumn(result, "Risk", new SQLFragment("CAST(NULL AS VARCHAR)"), JdbcType.VARCHAR)); + result.addColumn(new ExprColumn(result, "IsCdr", new SQLFragment("CAST(NULL AS VARCHAR)"), JdbcType.VARCHAR)); + } + return result; } @@ -1882,6 +1892,7 @@ private Set getAllTableNames(boolean caseInsensitive) hs.add(TABLE_RATE_TYPE); hs.add(TABLE_INSTRUMENT_RATE); hs.add(TABLE_INSTRUMENT_USAGE_PAYMENT); + hs.add(TABLE_PTM_PERCENTS_GROUPED_PREPIVOT_CACHE); return hs; } @@ -1921,6 +1932,11 @@ private QueryDefinition createRunScopedPTMQuery(String prefix, String baseQueryN try { long runId = Long.parseLong(runIdString); + if (TargetedMSManager.getFolderType(getContainer()) != TargetedMSService.FolderType.ExperimentMAM) + { + throw new IllegalStateException("PTM queries are only supported in ExperimentMAM folders"); + } + QueryDefinition queryDef = Objects.requireNonNull(getQueryDef(baseQueryName)); QueryDefinition result = QueryService.get().createQueryDef(getUser(), getContainer(), getSchemaPath(), queryName); result.setSql(queryDef.getSql() + " IN (SELECT r.Name FROM targetedms.Replicate r WHERE r.RunId = " + runId + ")"); diff --git a/src/org/labkey/targetedms/TargetedMSUpgradeCode.java b/src/org/labkey/targetedms/TargetedMSUpgradeCode.java index 1654517a3..3035b5fec 100644 --- a/src/org/labkey/targetedms/TargetedMSUpgradeCode.java +++ b/src/org/labkey/targetedms/TargetedMSUpgradeCode.java @@ -15,7 +15,10 @@ */ package org.labkey.targetedms; +import org.apache.logging.log4j.Logger; +import org.labkey.api.data.Container; import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.DeferredUpgrade; import org.labkey.api.data.SQLFragment; import org.labkey.api.data.SqlExecutor; import org.labkey.api.data.UpgradeCode; @@ -23,6 +26,8 @@ import org.labkey.api.module.ModuleContext; import org.labkey.api.module.ModuleLoader; import org.labkey.api.security.User; +import org.labkey.api.targetedms.TargetedMSService; +import org.labkey.api.util.logging.LogHelper; import org.labkey.targetedms.query.QCAnnotationTypeTable; import java.util.Date; @@ -36,6 +41,8 @@ */ public class TargetedMSUpgradeCode implements UpgradeCode { + private static final Logger LOG = LogHelper.getLogger(TargetedMSUpgradeCode.class, "TargetedMS schema upgrade operations"); + // called at every bootstrap to initialize annotation types @SuppressWarnings({"UnusedDeclaration"}) public void populateDefaultAnnotationTypes(final ModuleContext moduleContext) @@ -69,4 +76,36 @@ private void insertAnnotationType(String name, String color, User user) sql.add(color); new SqlExecutor(TargetedMSManager.getSchema()).execute(sql); } + + /** Populate PTMPercentsGroupedPrepivotCache for all runs in existing ExperimentMAM folders */ + @SuppressWarnings("UnusedDeclaration") + @DeferredUpgrade + public void populatePTMPercentsGroupedPrepivotCache(final ModuleContext moduleContext) + { + if (moduleContext.isNewInstall()) + { + return; + } + + User user = moduleContext.getUpgradeUser(); + LOG.info("Populating PTMPercentsGroupedPrepivotCache for existing ExperimentMAM folders"); + + for (TargetedMSRun run : TargetedMSManager.getAllNonDeletedRuns()) + { + Container container = run.getContainer(); + if (container != null && TargetedMSManager.getFolderType(container) == TargetedMSService.FolderType.ExperimentMAM) + { + try + { + TargetedMSManager.populatePTMPercentsGroupedPrepivotCache(run, user, container); + } + catch (Exception e) + { + LOG.error("Error populating PTMPercentsGroupedPrepivotCache for run " + run.getId() + " in " + container.getPath(), e); + } + } + } + + LOG.info("Finished populating PTMPercentsGroupedPrepivotCache for existing ExperimentMAM folders"); + } } diff --git a/test/src/org/labkey/test/tests/targetedms/TargetedMSEarlyStagePTMReportTest.java b/test/src/org/labkey/test/tests/targetedms/TargetedMSEarlyStagePTMReportTest.java index dedbdb2e4..c0b5907b1 100644 --- a/test/src/org/labkey/test/tests/targetedms/TargetedMSEarlyStagePTMReportTest.java +++ b/test/src/org/labkey/test/tests/targetedms/TargetedMSEarlyStagePTMReportTest.java @@ -39,10 +39,20 @@ private void doInit() @Test public void testEarlyStagePrepivot() { + // Test against the live-query version goToProjectHome(); goToSchemaBrowser(); DataRegionTable table = viewQueryData("targetedms", "PTMPercentsGroupedPrepivot"); + verifyPrepivotData(table); + // Test against the cached version too + goToSchemaBrowser(); + table = viewQueryData("targetedms", "PTMPercentsGroupedPrepivotCache"); + verifyPrepivotData(table); + } + + private void verifyPrepivotData(DataRegionTable table) + { // Test special-cased peptide table.setFilter("PeptideModifiedSequence", "Starts With", "EEQ"); table = new DataRegionTable("query", this); @@ -59,7 +69,7 @@ public void testEarlyStagePrepivot() table.setFilter("PeptideModifiedSequence", "Starts With", "VTN"); table = new DataRegionTable("query", this); assertEquals(List.of("true", "true"), table.getColumnDataAsText("IsCdr")); - assertEquals(List.of("High", "Medium"), table.getColumnDataAsText("Risk")); + assertEquals(List.of("Medium", "High"), table.getColumnDataAsText("Risk")); // Test special-cased N-Term Modification, present on QVTL peptide (Q is modified, so don't use it in the filter) table.setFilter("PeptideModifiedSequence", "Contains", "VTL");