From 01ad9dc50f222bac8c8811636bcd1158b61a8b3c Mon Sep 17 00:00:00 2001 From: Vladimir Steshin Date: Fri, 8 May 2026 17:09:39 +0300 Subject: [PATCH 1/6] test impl --- .../snapshot/IgniteSnapshotManager.java | 84 +++++++++++-------- .../IgniteClusterSnapshotMetricsTest.java | 2 +- 2 files changed, 49 insertions(+), 37 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java index 849e87175e346..3f1a8a38959dd 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java @@ -393,7 +393,7 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter private volatile boolean recovered; /** Last seen cluster snapshot operation. */ - private volatile ClusterSnapshotFuture lastSeenSnpFut = new ClusterSnapshotFuture(); + private volatile ClusterSnapshotFuture lastSeenSnpFut; /** Last seen incremental snapshot operation. */ private volatile ClusterSnapshotFuture lastSeenIncSnpFut; @@ -773,9 +773,22 @@ private IgniteInternalFuture initLocalSnapshotStartSt "Another snapshot operation in progress [req=" + req + ", curr=" + curSnpOp + ']')); } + ClusterSnapshotFuture snpFut0 = new ClusterSnapshotFuture(req.reqId, req.snpName, req.incremental() ? req.incrementIndex() : null); + + clusterSnpFut = snpFut0; + + log.error("TEST | set lastSeenSnpFut on " + cctx.localNode().order()); + + if (req.incremental()) + lastSeenIncSnpFut = snpFut0; + else + lastSeenSnpFut = snpFut0; + SnapshotOperation snpOp = new SnapshotOperation(req, new SnapshotFileTree(cctx.kernalContext(), req.snapshotName(), req.snapshotPath())); + log.error("TEST | initLocalSnapshotStartStage() on " + cctx.localNode().order()); + curSnpOp = snpOp; if (req.incremental()) @@ -1116,7 +1129,9 @@ private void processLocalSnapshotStartStageResult(UUID id, Map createSnapshot( } snpFut0 = new ClusterSnapshotFuture(UUID.randomUUID(), name, incIdx); - - clusterSnpFut = snpFut0; - - if (incremental) - lastSeenIncSnpFut = snpFut0; - else - lastSeenSnpFut = snpFut0; } Set cacheGrpNames0 = cacheGrpNames == null ? null : new HashSet<>(cacheGrpNames); @@ -4152,7 +4166,6 @@ public ClusterSnapshotFuture() { rqId = null; name = ""; startTime = 0; - endTime = 0; incIdx = null; } @@ -4165,7 +4178,6 @@ public ClusterSnapshotFuture(String name, Exception err) { this.name = name; startTime = U.currentTimeMillis(); - endTime = 0; rqId = null; incIdx = null; } diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotMetricsTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotMetricsTest.java index e4dcf1c2624a9..292726d2cdb54 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotMetricsTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotMetricsTest.java @@ -71,7 +71,7 @@ public class IgniteClusterSnapshotMetricsTest extends IgniteClusterSnapshotResto private static final String DEDICATED_DIR_PREFIX = "dedicated-"; /** Number of nodes using a separate working directory. */ - private static final int DEDICATED_CNT = 2; + private static final int DEDICATED_CNT = 3; /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { From d6e5f7283f4989684cae402bdef0bdd924ac2c9a Mon Sep 17 00:00:00 2001 From: Vladimir Steshin Date: Fri, 8 May 2026 22:58:00 +0300 Subject: [PATCH 2/6] fix --- .../cache/persistence/snapshot/IgniteSnapshotManager.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java index 3f1a8a38959dd..884260a72ff1a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java @@ -2030,9 +2030,7 @@ public IgniteFutureImpl createSnapshot( int incIdx = -1; synchronized (snpOpMux) { - assert clusterSnpFut != null; - - if (!clusterSnpFut.isDone()) { + if (clusterSnpFut != null && !clusterSnpFut.isDone()) { throw new IgniteException( "Create snapshot request has been rejected. The previous snapshot operation was not completed." ); From d2b89dd8f72cdf9f4498748e0ca091983a33fa58 Mon Sep 17 00:00:00 2001 From: Vladimir Steshin Date: Tue, 12 May 2026 14:24:07 +0300 Subject: [PATCH 3/6] fix --- .../snapshot/IgniteSnapshotManager.java | 26 ++++-- .../IgniteClusterSnapshotSelfTest.java | 91 +++++++------------ .../IgniteClusterSnapshotMetricsTest.java | 2 +- 3 files changed, 53 insertions(+), 66 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java index 884260a72ff1a..5dffc8a6bff3f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java @@ -773,16 +773,17 @@ private IgniteInternalFuture initLocalSnapshotStartSt "Another snapshot operation in progress [req=" + req + ", curr=" + curSnpOp + ']')); } - ClusterSnapshotFuture snpFut0 = new ClusterSnapshotFuture(req.reqId, req.snpName, req.incremental() ? req.incrementIndex() : null); + // Let's keep the metrics on any node. + if (clusterSnpFut == null) { + clusterSnpFut = new ClusterSnapshotFuture(req.reqId, req.snpName, req.incremental() ? req.incrementIndex() : null); - clusterSnpFut = snpFut0; - - log.error("TEST | set lastSeenSnpFut on " + cctx.localNode().order()); + if (req.incremental()) + lastSeenIncSnpFut = clusterSnpFut; + else + lastSeenSnpFut = clusterSnpFut; - if (req.incremental()) - lastSeenIncSnpFut = snpFut0; - else - lastSeenSnpFut = snpFut0; + log.error("TEST | set lastSeenSnpFut on " + cctx.localNode().order()); + } SnapshotOperation snpOp = new SnapshotOperation(req, new SnapshotFileTree(cctx.kernalContext(), req.snapshotName(), req.snapshotPath())); @@ -1446,7 +1447,7 @@ public boolean isSnapshotCreating() { return true; synchronized (snpOpMux) { - return curSnpOp != null || clusterSnpFut != null; + return clusterSnpFut != null; } } @@ -2067,6 +2068,13 @@ public IgniteFutureImpl createSnapshot( } snpFut0 = new ClusterSnapshotFuture(UUID.randomUUID(), name, incIdx); + + clusterSnpFut = snpFut0; + + if (incremental) + lastSeenIncSnpFut = snpFut0; + else + lastSeenSnpFut = snpFut0; } Set cacheGrpNames0 = cacheGrpNames == null ? null : new HashSet<>(cacheGrpNames); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java index ee23e278a7dc1..a11076b9551b7 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java @@ -849,30 +849,31 @@ public void testClusterSnapshotWithExplicitPathError() throws Exception { /** @throws Exception If fails. */ @Test public void testClusterSnapshotMetrics() throws Exception { - String newSnapshotName = SNAPSHOT_NAME + "_new"; CountDownLatch deltaApply = new CountDownLatch(1); CountDownLatch deltaBlock = new CountDownLatch(1); IgniteEx ignite = startGridsWithCache(2, dfltCacheCfg, CACHE_KEYS_RANGE); - MetricRegistry mreg0 = ignite.context().metric().registry(SNAPSHOT_METRICS); - - LongMetric startTime = mreg0.findMetric("LastSnapshotStartTime"); - LongMetric endTime = mreg0.findMetric("LastSnapshotEndTime"); - ObjectGauge snpName = mreg0.findMetric("LastSnapshotName"); - ObjectGauge errMsg = mreg0.findMetric("LastSnapshotErrorMessage"); - ObjectGauge> snpList = mreg0.findMetric("LocalSnapshotNames"); - // Snapshot process will be blocked when delta partition files processing starts. snp(ignite).localSnapshotSenderFactory( blockingLocalSnapshotSender(ignite, deltaApply, deltaBlock)); - assertEquals("Snapshot start time must be undefined prior to snapshot operation started.", - 0, startTime.value()); - assertEquals("Snapshot end time must be undefined to snapshot operation started.", - 0, endTime.value()); - assertTrue("Snapshot name must not exist prior to snapshot operation started.", snpName.value().isEmpty()); - assertTrue("Snapshot error message must null prior to snapshot operation started.", errMsg.value().isEmpty()); - assertTrue("Snapshots on local node must not exist", snpList.value().isEmpty()); + for(Ignite g : G.allGrids()) { + MetricRegistry mreg0 = ((IgniteEx)g).context().metric().registry(SNAPSHOT_METRICS); + + LongMetric startTime = mreg0.findMetric("LastSnapshotStartTime"); + LongMetric endTime = mreg0.findMetric("LastSnapshotEndTime"); + ObjectGauge snpName = mreg0.findMetric("LastSnapshotName"); + ObjectGauge errMsg = mreg0.findMetric("LastSnapshotErrorMessage"); + ObjectGauge> snpList = mreg0.findMetric("LocalSnapshotNames"); + + assertEquals("Snapshot start time must be undefined prior to snapshot operation started.", + 0, startTime.value()); + assertEquals("Snapshot end time must be undefined to snapshot operation started.", + 0, endTime.value()); + assertTrue("Snapshot name must not exist prior to snapshot operation started.", snpName.value().isEmpty()); + assertTrue("Snapshot error message must null prior to snapshot operation started.", errMsg.value().isEmpty()); + assertTrue("Snapshots on local node must not exist", snpList.value().isEmpty()); + } long cutoffStartTime = U.currentTimeMillis(); @@ -880,50 +881,28 @@ public void testClusterSnapshotMetrics() throws Exception { U.await(deltaApply); - assertTrue("Snapshot start time must be set prior to snapshot operation started " + - "[startTime=" + startTime.value() + ", cutoffTime=" + cutoffStartTime + ']', - startTime.value() >= cutoffStartTime); - assertEquals("Snapshot end time must be zero prior to snapshot operation started.", - 0, endTime.value()); - assertEquals("Snapshot name must be set prior to snapshot operation started.", - SNAPSHOT_NAME, snpName.value()); - assertTrue("Snapshot error message must null prior to snapshot operation started.", - errMsg.value().isEmpty()); - - IgniteFuture fut1 = snp(grid(1)).createSnapshot(newSnapshotName, null, false, onlyPrimary); - - assertThrowsWithCause((Callable)fut1::get, IgniteException.class); - - MetricRegistry mreg1 = grid(1).context().metric().registry(SNAPSHOT_METRICS); - - LongMetric startTime1 = mreg1.findMetric("LastSnapshotStartTime"); - LongMetric endTime1 = mreg1.findMetric("LastSnapshotEndTime"); - ObjectGauge snpName1 = mreg1.findMetric("LastSnapshotName"); - ObjectGauge errMsg1 = mreg1.findMetric("LastSnapshotErrorMessage"); - - assertTrue("Snapshot start time must be greater than zero for finished snapshot.", - startTime1.value() > 0); - assertEquals("Snapshot end time must zero for failed on start snapshots.", - 0, endTime1.value()); - assertEquals("Snapshot name must be set when snapshot operation already finished.", - newSnapshotName, snpName1.value()); - assertNotNull("Concurrent snapshot operation must failed.", - errMsg1.value()); + for(Ignite g : G.allGrids()) { + MetricRegistry mreg0 = ((IgniteEx)g).context().metric().registry(SNAPSHOT_METRICS); + + LongMetric startTime = mreg0.findMetric("LastSnapshotStartTime"); + LongMetric endTime = mreg0.findMetric("LastSnapshotEndTime"); + ObjectGauge snpName = mreg0.findMetric("LastSnapshotName"); + ObjectGauge errMsg = mreg0.findMetric("LastSnapshotErrorMessage"); + + assertTrue("Snapshot start time must be set prior to snapshot operation started " + + "[startTime=" + startTime.value() + ", cutoffTime=" + cutoffStartTime + ']', + startTime.value() >= cutoffStartTime); + assertEquals("Snapshot end time must be zero prior to snapshot operation started.", + 0, endTime.value()); + assertEquals("Snapshot name must be set prior to snapshot operation started.", + SNAPSHOT_NAME, snpName.value()); + assertTrue("Snapshot error message must null prior to snapshot operation started.", + errMsg.value().isEmpty()); + } deltaBlock.countDown(); fut0.get(); - - assertTrue("Snapshot start time must be greater than zero for finished snapshot.", - startTime.value() > 0); - assertTrue("Snapshot end time must be greater than zero for finished snapshot.", - endTime.value() > 0); - assertEquals("Snapshot name must be set when snapshot operation already finished.", - SNAPSHOT_NAME, snpName.value()); - assertTrue("Concurrent snapshot operation must finished successfully.", - errMsg.value().isEmpty()); - assertEquals("Only the first snapshot must be created and stored on disk.", - Collections.singletonList(SNAPSHOT_NAME), snpList.value()); } /** @throws Exception If fails. */ diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotMetricsTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotMetricsTest.java index 292726d2cdb54..e4dcf1c2624a9 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotMetricsTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotMetricsTest.java @@ -71,7 +71,7 @@ public class IgniteClusterSnapshotMetricsTest extends IgniteClusterSnapshotResto private static final String DEDICATED_DIR_PREFIX = "dedicated-"; /** Number of nodes using a separate working directory. */ - private static final int DEDICATED_CNT = 3; + private static final int DEDICATED_CNT = 2; /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { From e5ccbaee1334910b41f1e849eb1a28c4ca865150 Mon Sep 17 00:00:00 2001 From: Vladimir Steshin Date: Tue, 12 May 2026 15:23:11 +0300 Subject: [PATCH 4/6] fix --- .../snapshot/IgniteSnapshotManager.java | 4 +- .../IgniteClusterSnapshotSelfTest.java | 40 ++++++++++++------- 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java index 5dffc8a6bff3f..6a20180adef5b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java @@ -393,7 +393,7 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter private volatile boolean recovered; /** Last seen cluster snapshot operation. */ - private volatile ClusterSnapshotFuture lastSeenSnpFut; + private volatile ClusterSnapshotFuture lastSeenSnpFut = new ClusterSnapshotFuture(); /** Last seen incremental snapshot operation. */ private volatile ClusterSnapshotFuture lastSeenIncSnpFut; @@ -4172,6 +4172,7 @@ public ClusterSnapshotFuture() { rqId = null; name = ""; startTime = 0; + endTime = 0; incIdx = null; } @@ -4184,6 +4185,7 @@ public ClusterSnapshotFuture(String name, Exception err) { this.name = name; startTime = U.currentTimeMillis(); + endTime = 0; rqId = null; incIdx = null; } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java index a11076b9551b7..a11611d12878e 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java @@ -857,14 +857,14 @@ public void testClusterSnapshotMetrics() throws Exception { snp(ignite).localSnapshotSenderFactory( blockingLocalSnapshotSender(ignite, deltaApply, deltaBlock)); - for(Ignite g : G.allGrids()) { - MetricRegistry mreg0 = ((IgniteEx)g).context().metric().registry(SNAPSHOT_METRICS); + for (Ignite g : G.allGrids()) { + MetricRegistry mreg = ((IgniteEx)g).context().metric().registry(SNAPSHOT_METRICS); - LongMetric startTime = mreg0.findMetric("LastSnapshotStartTime"); - LongMetric endTime = mreg0.findMetric("LastSnapshotEndTime"); - ObjectGauge snpName = mreg0.findMetric("LastSnapshotName"); - ObjectGauge errMsg = mreg0.findMetric("LastSnapshotErrorMessage"); - ObjectGauge> snpList = mreg0.findMetric("LocalSnapshotNames"); + LongMetric startTime = mreg.findMetric("LastSnapshotStartTime"); + LongMetric endTime = mreg.findMetric("LastSnapshotEndTime"); + ObjectGauge snpName = mreg.findMetric("LastSnapshotName"); + ObjectGauge errMsg = mreg.findMetric("LastSnapshotErrorMessage"); + ObjectGauge> snpList = mreg.findMetric("LocalSnapshotNames"); assertEquals("Snapshot start time must be undefined prior to snapshot operation started.", 0, startTime.value()); @@ -877,17 +877,17 @@ public void testClusterSnapshotMetrics() throws Exception { long cutoffStartTime = U.currentTimeMillis(); - IgniteFuture fut0 = snp(ignite).createSnapshot(SNAPSHOT_NAME, null, false, onlyPrimary); + IgniteFuture fut = snp(ignite).createSnapshot(SNAPSHOT_NAME, null, false, onlyPrimary); U.await(deltaApply); - for(Ignite g : G.allGrids()) { - MetricRegistry mreg0 = ((IgniteEx)g).context().metric().registry(SNAPSHOT_METRICS); + for (Ignite g : G.allGrids()) { + MetricRegistry mreg = ((IgniteEx)g).context().metric().registry(SNAPSHOT_METRICS); - LongMetric startTime = mreg0.findMetric("LastSnapshotStartTime"); - LongMetric endTime = mreg0.findMetric("LastSnapshotEndTime"); - ObjectGauge snpName = mreg0.findMetric("LastSnapshotName"); - ObjectGauge errMsg = mreg0.findMetric("LastSnapshotErrorMessage"); + LongMetric startTime = mreg.findMetric("LastSnapshotStartTime"); + LongMetric endTime = mreg.findMetric("LastSnapshotEndTime"); + ObjectGauge snpName = mreg.findMetric("LastSnapshotName"); + ObjectGauge errMsg = mreg.findMetric("LastSnapshotErrorMessage"); assertTrue("Snapshot start time must be set prior to snapshot operation started " + "[startTime=" + startTime.value() + ", cutoffTime=" + cutoffStartTime + ']', @@ -902,7 +902,17 @@ public void testClusterSnapshotMetrics() throws Exception { deltaBlock.countDown(); - fut0.get(); + fut.get(); + + for (Ignite g : G.allGrids()) { + MetricRegistry mreg = ((IgniteEx)g).context().metric().registry(SNAPSHOT_METRICS); + + LongMetric startTime = mreg.findMetric("LastSnapshotStartTime"); + LongMetric endTime = mreg.findMetric("LastSnapshotEndTime"); + + waitForCondition(() -> endTime.value() != 0L && startTime.value() != 0 && endTime.value() > startTime.value(), + getTestTimeout()); + } } /** @throws Exception If fails. */ From 991d58ea35372ef364645c00c877c2112acc73bd Mon Sep 17 00:00:00 2001 From: Vladimir Steshin Date: Wed, 13 May 2026 15:09:02 +0300 Subject: [PATCH 5/6] fix --- .../cache/persistence/snapshot/IgniteSnapshotManager.java | 6 ------ .../IncrementalSnapshotWarnAtomicCachesTest.java | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java index 6a20180adef5b..0927dd3e79b3e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java @@ -781,15 +781,11 @@ private IgniteInternalFuture initLocalSnapshotStartSt lastSeenIncSnpFut = clusterSnpFut; else lastSeenSnpFut = clusterSnpFut; - - log.error("TEST | set lastSeenSnpFut on " + cctx.localNode().order()); } SnapshotOperation snpOp = new SnapshotOperation(req, new SnapshotFileTree(cctx.kernalContext(), req.snapshotName(), req.snapshotPath())); - log.error("TEST | initLocalSnapshotStartStage() on " + cctx.localNode().order()); - curSnpOp = snpOp; if (req.incremental()) @@ -1404,8 +1400,6 @@ private void processLocalSnapshotEndStageResult(UUID id, Map Date: Wed, 13 May 2026 16:51:29 +0300 Subject: [PATCH 6/6] fix --- .../snapshot/IgniteSnapshotManager.java | 6 ++ ...crementalSnapshotWarnAtomicCachesTest.java | 101 ++++++++++++------ 2 files changed, 77 insertions(+), 30 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java index 0927dd3e79b3e..808e26be6316b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java @@ -2176,6 +2176,12 @@ void warnAtomicCachesInIncrementalSnapshot(String snpName, int incIdx, Collectio if (warnCaches.isEmpty()) return; + System.err.println("TEST | " + "Incremental snapshot [snpName=" + snpName + ", incIdx=" + incIdx + "] contains ATOMIC caches with backups: " + + warnCaches + ". Please note, incremental snapshots doesn't guarantee consistency of restored atomic caches. " + + "It is highly recommended to verify these caches after restoring with the \"idle_verify\" command. " + + "If it is needed it's possible to repair inconsistent partitions with the \"consistency\" command. " + + "Please, check the \"Control Script\" section of Ignite docs for more information about these commands."); + U.warn(log, "Incremental snapshot [snpName=" + snpName + ", incIdx=" + incIdx + "] contains ATOMIC caches with backups: " + warnCaches + ". Please note, incremental snapshots doesn't guarantee consistency of restored atomic caches. " + "It is highly recommended to verify these caches after restoring with the \"idle_verify\" command. " + diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/incremental/IncrementalSnapshotWarnAtomicCachesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/incremental/IncrementalSnapshotWarnAtomicCachesTest.java index 017d593b6ee73..4a9697b55625b 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/incremental/IncrementalSnapshotWarnAtomicCachesTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/incremental/IncrementalSnapshotWarnAtomicCachesTest.java @@ -19,6 +19,7 @@ import java.util.ArrayList; import java.util.Collection; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -32,6 +33,7 @@ import org.apache.ignite.configuration.DataRegionConfiguration; import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteInterruptedCheckedException; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.testframework.ListeningTestLogger; import org.apache.ignite.testframework.LogListener; @@ -39,6 +41,8 @@ import org.jetbrains.annotations.Nullable; import org.junit.Test; +import static org.apache.ignite.testframework.GridTestUtils.waitForCondition; + /** */ public class IncrementalSnapshotWarnAtomicCachesTest extends GridCommonAbstractTest { /** */ @@ -112,8 +116,8 @@ private CacheConfiguration[] prepareCacheConfs(String grp0, St /** */ public void checkCachesSnapshotCreationAndRestore(CacheConfiguration... ccfgs) throws Exception { - List allWarnCaches = new ArrayList<>(); - Map> warnCachesByGrps = new HashMap<>(); + List allWarnCaches = new ArrayList<>(); + Map> warnCachesByGrps = new HashMap<>(); for (CacheConfiguration ccfg: ccfgs) { if (ccfg.getAtomicityMode() == CacheAtomicityMode.ATOMIC && ccfg.getBackups() > 0) { @@ -122,37 +126,27 @@ public void checkCachesSnapshotCreationAndRestore(CacheConfiguration { caches = caches == null ? new ArrayList<>() : caches; - int cacheNum = Integer.parseInt(ccfg.getName().replace("cache", "")); - - caches.add(cacheNum); + caches.add(ccfg.getName()); - allWarnCaches.add(cacheNum); + allWarnCaches.add(ccfg.getName()); return caches; }); } } - checkWarnMessageOnCreateSnapshot(cachesPattern(allWarnCaches), ccfgs); - checkWarnMessageOnRestoreSnapshot(cachesPattern(allWarnCaches), null); + checkWarnMessageOnCreateSnapshot(allWarnCaches, ccfgs); + checkWarnMessageOnRestoreSnapshot(allWarnCaches, null); - for (String grp: warnCachesByGrps.keySet()) - checkWarnMessageOnRestoreSnapshot(cachesPattern(warnCachesByGrps.get(grp)), F.asList(grp)); - } + log.error("TEST | test2"); - /** Transforms cache numbers to cache pattern. For example, [0, 1] -> cache[0,1], cache[0,1]. */ - private @Nullable String cachesPattern(List cacheNums) { - if (cacheNums.isEmpty()) - return null; - - return cacheNums.stream() - .map(c -> "cache" + cacheNums) - .collect(Collectors.joining(", ")); + for (String grp: warnCachesByGrps.keySet()) + checkWarnMessageOnRestoreSnapshot(warnCachesByGrps.get(grp), F.asList(grp)); } /** */ private void checkWarnMessageOnCreateSnapshot( - @Nullable String warnAtomicCaches, + Collection warnAtomicCaches, CacheConfiguration... ccfgs ) throws Exception { this.ccfgs = ccfgs; @@ -170,27 +164,37 @@ private void checkWarnMessageOnCreateSnapshot( lsnLogger.registerListener(lsnr); + log.error("TEST | test00"); + g.snapshot().createSnapshot(SNP).get(getTestTimeout()); - assertTrue(warnAtomicCaches, lsnr.check()); + assertTrue(lsnr.check()); for (CacheConfiguration c: ccfgs) { for (int i = 1_000; i < 2_000; i++) g.cache(c.getName()).put(i, i); } - lsnr = warnLogListener(warnAtomicCaches, warnAtomicCaches == null ? 0 : 3); + lsnr = warnLogListener(warnAtomicCaches, warnAtomicCaches.isEmpty() ? 0 : 3); + + lsnLogger.clearListeners(); lsnLogger.registerListener(lsnr); + log.error("TEST | test01"); + g.snapshot().createIncrementalSnapshot(SNP).get(getTestTimeout()); - assertTrue(warnAtomicCaches, lsnr.check()); + log.error("TEST | test02"); + + waitForLog(lsnr); + + log.error("TEST | test03"); } /** */ private void checkWarnMessageOnRestoreSnapshot( - @Nullable String warnAtomicCaches, + Collection warnAtomicCaches, @Nullable Collection restoreCacheGrps ) throws Exception { stopAllGrids(); @@ -207,30 +211,67 @@ private void checkWarnMessageOnRestoreSnapshot( LogListener lsnr = warnLogListener(warnAtomicCaches, 0); // Should not warn for full snapshots. + lsnLogger.clearListeners(); + lsnLogger.registerListener(lsnr); + log.error("TEST | test4"); + g.snapshot().restoreSnapshot(SNP, restoreCacheGrps).get(getTestTimeout()); - assertTrue(warnAtomicCaches + " " + restoreCacheGrps, lsnr.check()); + waitForLog(lsnr); g.destroyCaches(g.cacheNames()); awaitPartitionMapExchange(); - lsnr = warnLogListener(warnAtomicCaches, warnAtomicCaches == null ? 0 : 1); + LogListener lsnr0 = warnLogListener(warnAtomicCaches, warnAtomicCaches.isEmpty() ? 0 : 1); - lsnLogger.registerListener(lsnr); + lsnLogger.clearListeners(); + + lsnLogger.registerListener(lsnr0); + + log.error("TEST | test5"); g.snapshot().restoreSnapshot(SNP, restoreCacheGrps, 1).get(getTestTimeout()); - assertTrue(warnAtomicCaches + " " + restoreCacheGrps, lsnr.check()); + log.error("TEST | test6"); + + waitForLog(lsnr0); + + log.error("TEST | test7"); + } + + /** */ + private void waitForLog(LogListener lsnr) throws IgniteInterruptedCheckedException { + waitForCondition(()-> { + try { + return lsnr.check(getTestTimeout()); + } + catch (InterruptedException e) { + return false; + } + }, getTestTimeout()); } /** */ - private LogListener warnLogListener(@Nullable String atomicCaches, int times) { + private LogListener warnLogListener(Collection atomicCaches, int times) { + String cachesStr = null; + + if (atomicCaches.size() == 1) + cachesStr = F.first(atomicCaches); + else if (atomicCaches.size() > 1) { + cachesStr = "((" + String.join(", ", atomicCaches) + ')'; + + cachesStr += "|(" + atomicCaches.stream().sorted(Comparator.reverseOrder()).collect(Collectors.joining(", ")) + "))"; + } + Pattern p = Pattern.compile( "Incremental snapshot \\[snpName=" + SNP + ", incIdx=1] contains ATOMIC caches with backups:" - + (atomicCaches == null ? "" : " \\[" + atomicCaches) + ']'); + + (cachesStr == null ? "" : " \\[" + cachesStr) + "]"); + + log.error("TEST | pattern: " + "Incremental snapshot \\[snpName=" + SNP + ", incIdx=1] contains ATOMIC caches with backups:" + + (cachesStr == null ? "" : " \\[" + cachesStr) + "]"); return LogListener.matches(p).times(times).build(); }