fix(yarn-project): retry sqlite3mc vendor download on transient errors

.test_patterns.yml

@@ -172,6 +172,18 @@ tests:
     error_regex: "ContractFunctionExecutionError: The contract function"
     owners:
       - *mitch
+  # Under proposer pipelining each validator votes in its own slot and the votes
+  # don't aggregate into the same round, so the slashing quorum (3) is never
+  # reached within the 414s budget; the test consistently times out at the docker
+  # outer 600s (exit 124). The publisher refactor lands all vote-offenses tx's
+  # on L1 successfully — voteCount on the slasher proposer simply stays at 1
+  # per round. This is a slashing-payload aggregation issue independent of
+  # publisher work; skip until the slashing team addresses it separately.
+  - regex: "e2e_p2p/valid_epoch_pruned_slash.test.ts"
+    skip: true
+    owners:
+      - *mitch
+      - *palla
   - regex: "archiver/src/archiver/archiver.test.ts"
     error_regex: "Received number of calls: 1"
     owners:
@@ -185,14 +197,6 @@ tests:
       - *phil
       - *palla
 
-  # http://ci.aztec-labs.com/64a972aafaa40dd0
-  # ProvingBroker › Retries › does not retry if job is stale — kv-store closes
-  # before the broker's final reportProvingJobError write lands.
-  - regex: "prover-client/src/proving_broker/proving_broker.test.ts"
-    error_regex: "does not retry if job is stale|Store is closed"
-    owners:
-      - *alex
-
   # Nightly GKE tests
   - regex: "spartan/bootstrap.sh"
     owners:

aztec-up/test/amm_flow.sh

@@ -1,6 +1,8 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
+export SEQ_ENABLE_PROPOSER_PIPELINING=true
+
 # Start local network and wait for port to open.
 aztec start --local-network &
 local_network_pid=$!

aztec-up/test/basic_install.sh

@@ -10,6 +10,7 @@ echo
 
 export LOG_LEVEL=silent
 export PXE_PROVER=none
+export SEQ_ENABLE_PROPOSER_PIPELINING=true
 
 # Start local network and wait for port to open.
 aztec start --local-network &

aztec-up/test/bridge_and_claim.sh

@@ -1,6 +1,8 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
+export SEQ_ENABLE_PROPOSER_PIPELINING=true
+
 # Start local network and wait for port to open.
 aztec start --local-network &
 local_network_pid=$!

docs/examples/ts/docker-compose.yml

@@ -28,6 +28,7 @@ services:
       WS_BLOCK_CHECK_INTERVAL_MS: 500
       ARCHIVER_VIEM_POLLING_INTERVAL_MS: 500
       P2P_MIN_TX_POOL_AGE_MS: 0
+      SEQ_ENABLE_PROPOSER_PIPELINING: 'true'
       HARDWARE_CONCURRENCY: ${HARDWARE_CONCURRENCY:-}
 
   docs-examples:

playground/docker-compose.yml

@@ -27,6 +27,7 @@ services:
       WS_BLOCK_CHECK_INTERVAL_MS: 50
       ARCHIVER_VIEM_POLLING_INTERVAL_MS: 500
       P2P_MIN_TX_POOL_AGE_MS: 0
+      SEQ_ENABLE_PROPOSER_PIPELINING: 'true'
     healthcheck:
       test: ['CMD', 'curl', '-fSs', 'http://127.0.0.1:8080/status']
       interval: 3s

spartan/aztec-node/templates/_pod-template.yaml

@@ -237,6 +237,10 @@ spec:
         - name: SLASH_INVALID_BLOCK_PENALTY
           value: {{ .Values.node.slash.invalidBlockPenalty | quote }}
         {{- end }}
+        {{- if .Values.node.slash.invalidCheckpointProposalPenalty }}
+        - name: SLASH_INVALID_CHECKPOINT_PROPOSAL_PENALTY
+          value: {{ .Values.node.slash.invalidCheckpointProposalPenalty | quote }}
+        {{- end }}
         {{- if .Values.node.slash.proposeInvalidAttestationsPenalty }}
         - name: SLASH_PROPOSE_INVALID_ATTESTATIONS_PENALTY
           value: {{ .Values.node.slash.proposeInvalidAttestationsPenalty | quote }}

spartan/aztec-node/values.yaml

@@ -152,6 +152,7 @@ node:
     inactivityPenalty: ""
     inactivityTargetPercentage: ""
     invalidBlockPenalty: ""
+    invalidCheckpointProposalPenalty: ""
     proposeInvalidAttestationsPenalty: ""
     attestDescendantOfInvalidPenalty: ""
     attestInvalidCheckpointProposalPenalty: ""

spartan/environments/network-defaults.yml

@@ -143,6 +143,8 @@ slasher: &slasher
   SLASH_UNKNOWN_PENALTY: 10e18
   # Penalty for broadcasting an invalid block.
   SLASH_INVALID_BLOCK_PENALTY: 10e18
+  # Penalty for broadcasting an invalid checkpoint proposal.
+  SLASH_INVALID_CHECKPOINT_PROPOSAL_PENALTY: 0
   # L2 slots grace period before considering an offense expired.
   SLASH_GRACE_PERIOD_L2_SLOTS: 0
 
@@ -247,6 +249,7 @@ networks:
     SLASH_ATTEST_INVALID_CHECKPOINT_PROPOSAL_PENALTY: 10e18
     SLASH_UNKNOWN_PENALTY: 10e18
     SLASH_INVALID_BLOCK_PENALTY: 10e18
+    SLASH_INVALID_CHECKPOINT_PROPOSAL_PENALTY: 0
     SLASH_GRACE_PERIOD_L2_SLOTS: 0
     ENABLE_VERSION_CHECK: true
 
@@ -293,6 +296,7 @@ networks:
     SLASH_ATTEST_INVALID_CHECKPOINT_PROPOSAL_PENALTY: 10e18
     SLASH_UNKNOWN_PENALTY: 10e18
     SLASH_INVALID_BLOCK_PENALTY: 10e18
+    SLASH_INVALID_CHECKPOINT_PROPOSAL_PENALTY: 0
     SLASH_GRACE_PERIOD_L2_SLOTS: 64
     ENABLE_VERSION_CHECK: true
 
@@ -353,4 +357,5 @@ networks:
     SLASH_ATTEST_INVALID_CHECKPOINT_PROPOSAL_PENALTY: 2000e18
     SLASH_UNKNOWN_PENALTY: 2000e18
     SLASH_INVALID_BLOCK_PENALTY: 2000e18
+    SLASH_INVALID_CHECKPOINT_PROPOSAL_PENALTY: 0
     SLASH_GRACE_PERIOD_L2_SLOTS: 1200

spartan/scripts/deploy_network.sh

@@ -595,6 +595,7 @@ SLASH_ATTEST_DESCENDANT_OF_INVALID_PENALTY = ${SLASH_ATTEST_DESCENDANT_OF_INVALI
 SLASH_ATTEST_INVALID_CHECKPOINT_PROPOSAL_PENALTY = ${SLASH_ATTEST_INVALID_CHECKPOINT_PROPOSAL_PENALTY:-null}
 SLASH_UNKNOWN_PENALTY = ${SLASH_UNKNOWN_PENALTY:-null}
 SLASH_INVALID_BLOCK_PENALTY = ${SLASH_INVALID_BLOCK_PENALTY:-null}
+SLASH_INVALID_CHECKPOINT_PROPOSAL_PENALTY = ${SLASH_INVALID_CHECKPOINT_PROPOSAL_PENALTY:-null}
 SLASH_OFFENSE_EXPIRATION_ROUNDS = ${SLASH_OFFENSE_EXPIRATION_ROUNDS:-null}
 SLASH_MAX_PAYLOAD_SIZE = ${SLASH_MAX_PAYLOAD_SIZE:-null}
 OTEL_COLLECTOR_ENDPOINT = "${OTEL_COLLECTOR_ENDPOINT}"

spartan/terraform/deploy-aztec-infra/main.tf

@@ -209,6 +209,7 @@ locals {
     "validator.slash.attestInvalidCheckpointProposalPenalty"       = var.SLASH_ATTEST_INVALID_CHECKPOINT_PROPOSAL_PENALTY
     "validator.slash.unknownPenalty"                              = var.SLASH_UNKNOWN_PENALTY
     "validator.slash.invalidBlockPenalty"                         = var.SLASH_INVALID_BLOCK_PENALTY
+    "validator.slash.invalidCheckpointProposalPenalty"             = var.SLASH_INVALID_CHECKPOINT_PROPOSAL_PENALTY
     "validator.slash.offenseExpirationRounds"                     = var.SLASH_OFFENSE_EXPIRATION_ROUNDS
     "validator.slash.maxPayloadSize"                              = var.SLASH_MAX_PAYLOAD_SIZE
     "validator.node.env.TRANSACTIONS_DISABLED"                    = var.TRANSACTIONS_DISABLED

spartan/terraform/deploy-aztec-infra/variables.tf

@@ -520,6 +520,12 @@ variable "SLASH_INVALID_BLOCK_PENALTY" {
   nullable    = true
 }
 
+variable "SLASH_INVALID_CHECKPOINT_PROPOSAL_PENALTY" {
+  description = "The slash invalid checkpoint proposal penalty"
+  type        = string
+  nullable    = true
+}
+
 variable "SLASH_OFFENSE_EXPIRATION_ROUNDS" {
   description = "The slash offense expiration rounds"
   type        = string

yarn-project/archiver/src/store/block_store.ts

@@ -13,7 +13,10 @@ import {
   BlockHash,
   Body,
   CommitteeAttestation,
+  GENESIS_CHECKPOINT_HEADER_HASH,
   L2Block,
+  type L2TipId,
+  type L2Tips,
   type ValidateCheckpointResult,
   deserializeValidateCheckpointResult,
   serializeValidateCheckpointResult,
@@ -1129,6 +1132,174 @@ export class BlockStore {
     return typeof lastBlockNumber === 'number' ? BlockNumber(lastBlockNumber) : BlockNumber(INITIAL_L2_BLOCK_NUM - 1);
   }
 
+  /**
+   * Resolves all five L2 chain tips (proposed, proposedCheckpoint, checkpointed, proven, finalized)
+   * in a single read-only transaction so the snapshot is internally consistent. Each underlying
+   * record is read at most once: latest block, latest confirmed checkpoint, and latest pending
+   * checkpoint are each loaded directly (no separate "find the number, then look up data" hop),
+   * the proven/finalized checkpoint singletons are read once and their storage entries are
+   * reused if they coincide with the latest checkpoint, and per-tip block hashes are deduped
+   * when two tips land on the same block (e.g. finalized == proven, or proposedCheckpoint falls
+   * back to checkpointed when no pending checkpoint exists).
+   *
+   * The result is guaranteed to satisfy `finalized <= proven <= checkpointed <= proposed` (by
+   * block number). Genesis is represented by `(INITIAL_L2_BLOCK_NUM - 1)` and the supplied
+   * `genesisBlockHash`, paired with the synthetic genesis checkpoint id.
+   *
+   * @param genesisBlockHash - Block hash to report for the synthetic pre-initial block (used when
+   *   a tip is still at genesis).
+   */
+  async getL2TipsData(genesisBlockHash: BlockHash): Promise<L2Tips> {
+    return await this.db.transactionAsync(async () => {
+      // Define genesis tips
+      const genesisBlockNumber = BlockNumber(INITIAL_L2_BLOCK_NUM - 1);
+      const genesisCheckpointNumber = CheckpointNumber(INITIAL_CHECKPOINT_NUMBER - 1);
+      const genesisBlockId = { number: genesisBlockNumber, hash: genesisBlockHash.toString() };
+      const genesisCheckpointId = {
+        number: genesisCheckpointNumber,
+        hash: GENESIS_CHECKPOINT_HEADER_HASH.toString(),
+      };
+      const genesisTip: L2TipId = { block: genesisBlockId, checkpoint: genesisCheckpointId };
+
+      // Load latest block and checkpoint entries
+      const [latestBlockEntry] = await toArray(this.#blocks.entriesAsync({ reverse: true, limit: 1 }));
+      const [proposedCheckpointEntry] = await toArray(
+        this.#proposedCheckpoints.entriesAsync({ reverse: true, limit: 1 }),
+      );
+      const [latestCheckpointEntry] = await toArray(this.#checkpoints.entriesAsync({ reverse: true, limit: 1 }));
+      const latestCheckpointNumber = latestCheckpointEntry
+        ? CheckpointNumber(latestCheckpointEntry[0])
+        : genesisCheckpointNumber;
+
+      // Load proven and finalized checkpoint number pointers
+      const [provenRaw, finalizedRaw] = await Promise.all([
+        this.#lastProvenCheckpoint.getAsync(),
+        this.#lastFinalizedCheckpoint.getAsync(),
+      ]);
+
+      // Clamp to enforce finalized <= proven <= checkpointed.
+      const provenCheckpointNumber = CheckpointNumber(Math.min(provenRaw ?? 0, latestCheckpointNumber));
+      const finalizedCheckpointNumber = CheckpointNumber(Math.min(finalizedRaw ?? 0, provenCheckpointNumber));
+
+      // Avoid loading the same checkpoint more than once
+      const checkpointStorageCache = new Map<CheckpointNumber, CheckpointStorage>();
+      if (latestCheckpointEntry) {
+        checkpointStorageCache.set(CheckpointNumber(latestCheckpointEntry[0]), latestCheckpointEntry[1]);
+      }
+      const loadCheckpointStorage = async (n: CheckpointNumber): Promise<CheckpointStorage | undefined> => {
+        if (n === 0) {
+          return undefined;
+        }
+        if (!checkpointStorageCache.has(n)) {
+          const checkpointStorage = await this.#checkpoints.getAsync(n);
+          if (!checkpointStorage) {
+            throw new CheckpointNotFoundError(n);
+          }
+          checkpointStorageCache.set(n, checkpointStorage);
+        }
+        return checkpointStorageCache.get(n)!;
+      };
+
+      // Load proven and finalized checkpoint storage entries
+      const provenCheckpoint = await loadCheckpointStorage(provenCheckpointNumber);
+      const finalizedCheckpoint = await loadCheckpointStorage(finalizedCheckpointNumber);
+
+      // Avoid loading the same block hash multiple times when tips land on the same block
+      const blockHashCache = new Map<number, string>();
+      blockHashCache.set(genesisBlockNumber, genesisBlockHash.toString());
+      if (latestBlockEntry) {
+        blockHashCache.set(latestBlockEntry[0], BlockHash.fromBuffer(latestBlockEntry[1].blockHash).toString());
+      }
+      const loadBlockHash = async (n: BlockNumber): Promise<string> => {
+        if (!blockHashCache.has(n)) {
+          const blockStorage = await this.#blocks.getAsync(n);
+          if (!blockStorage) {
+            throw new BlockNotFoundError(n);
+          }
+          const blockHash = BlockHash.fromBuffer(blockStorage.blockHash).toString();
+          blockHashCache.set(n, blockHash);
+        }
+        return blockHashCache.get(n)!;
+      };
+
+      // Build proposed chain tip (this one has block only, no checkpoint)
+      const proposedBlockId =
+        latestBlockEntry === undefined
+          ? genesisBlockId
+          : {
+              number: BlockNumber(latestBlockEntry[0]),
+              hash: BlockHash.fromBuffer(latestBlockEntry[1].blockHash).toString(),
+            };
+
+      // Build other tips from checkpoint data, reading corresponding block data from the cache
+      const buildTipFromCheckpoint = async (
+        stored: ProposedCheckpointStorage | CheckpointStorage | undefined,
+      ): Promise<L2TipId> => {
+        if (!stored) {
+          return genesisTip;
+        }
+        const blockNumber = BlockNumber(stored.startBlock + stored.blockCount - 1);
+        const blockHash = await loadBlockHash(blockNumber);
+        const header = CheckpointHeader.fromBuffer(stored.header);
+        return {
+          block: { number: blockNumber, hash: blockHash },
+          checkpoint: { number: CheckpointNumber(stored.checkpointNumber), hash: header.hash().toString() },
+        };
+      };
+
+      const checkpointedTip = await buildTipFromCheckpoint(latestCheckpointEntry?.[1]);
+      const provenTip = await buildTipFromCheckpoint(provenCheckpoint);
+      const finalizedTip = await buildTipFromCheckpoint(finalizedCheckpoint);
+
+      // Proposed checkpoint falls back to the checkpoint tip if it's not set. And if local storage is
+      // inconsistent and the proposed checkpoint is behind the checkpointed tip, we patch that and
+      // report the checkpointed tip as the proposed checkpoint to maintain the invariant.
+      const proposedCheckpointTip =
+        proposedCheckpointEntry === undefined || proposedCheckpointEntry[0] <= latestCheckpointNumber
+          ? checkpointedTip
+          : await buildTipFromCheckpoint(proposedCheckpointEntry[1]);
+
+      // A checkpointed block past the latest stored block would mean a checkpoint
+      // references blocks that aren't in blocks.
+      if (proposedBlockId.number < checkpointedTip.block.number) {
+        throw new Error(
+          `Inconsistent block store: latest block ${proposedBlockId.number} is behind checkpointed block ${checkpointedTip.block.number}`,
+        );
+      }
+
+      // Assert that checkpoint numbers are increasing
+      if (
+        finalizedTip.checkpoint.number > provenTip.checkpoint.number ||
+        provenTip.checkpoint.number > checkpointedTip.checkpoint.number ||
+        checkpointedTip.checkpoint.number > proposedCheckpointTip.checkpoint.number
+      ) {
+        throw new Error(
+          `Inconsistent checkpoint numbers in chain tips: finalized=${finalizedTip.checkpoint.number} proven=${provenTip.checkpoint.number} checkpointed=${checkpointedTip.checkpoint.number} proposed=${proposedCheckpointTip.checkpoint.number}`,
+        );
+      }
+
+      // Assert block numbers are increasing
+      if (
+        finalizedTip.block.number > provenTip.block.number ||
+        provenTip.block.number > checkpointedTip.block.number ||
+        checkpointedTip.block.number > proposedCheckpointTip.block.number ||
+        proposedCheckpointTip.block.number > proposedBlockId.number
+      ) {
+        throw new Error(
+          `Inconsistent block numbers in chain tips: finalized=${finalizedTip.block.number} proven=${provenTip.block.number} checkpointed=${checkpointedTip.block.number} proposedCheckpoint=${proposedCheckpointTip.block.number} proposed=${proposedBlockId.number}`,
+        );
+      }
+
+      return {
+        proposed: proposedBlockId,
+        proposedCheckpoint: proposedCheckpointTip,
+        checkpointed: checkpointedTip,
+        proven: provenTip,
+        finalized: finalizedTip,
+      };
+    });
+  }
+
   /**
    * Gets the most recent L1 block processed.
    * @returns The L1 block that published the latest L2 block
@@ -1188,13 +1359,15 @@ export class BlockStore {
   }
 
   async getProvenCheckpointNumber(): Promise<CheckpointNumber> {
-    const [latestCheckpointNumber, provenCheckpointNumber] = await Promise.all([
-      this.getLatestCheckpointNumber(),
-      this.#lastProvenCheckpoint.getAsync(),
-    ]);
-    return (provenCheckpointNumber ?? 0) > latestCheckpointNumber
-      ? latestCheckpointNumber
-      : CheckpointNumber(provenCheckpointNumber ?? 0);
+    return await this.db.transactionAsync(async () => {
+      const [latestCheckpointNumber, provenCheckpointNumber] = await Promise.all([
+        this.getLatestCheckpointNumber(),
+        this.#lastProvenCheckpoint.getAsync(),
+      ]);
+      return (provenCheckpointNumber ?? 0) > latestCheckpointNumber
+        ? latestCheckpointNumber
+        : CheckpointNumber(provenCheckpointNumber ?? 0);
+    });
   }
 
   async setProvenCheckpointNumber(checkpointNumber: CheckpointNumber) {
@@ -1203,13 +1376,15 @@ export class BlockStore {
   }
 
   async getFinalizedCheckpointNumber(): Promise<CheckpointNumber> {
-    const [latestCheckpointNumber, finalizedCheckpointNumber] = await Promise.all([
-      this.getLatestCheckpointNumber(),
-      this.#lastFinalizedCheckpoint.getAsync(),
-    ]);
-    return (finalizedCheckpointNumber ?? 0) > latestCheckpointNumber
-      ? latestCheckpointNumber
-      : CheckpointNumber(finalizedCheckpointNumber ?? 0);
+    return await this.db.transactionAsync(async () => {
+      const [provenCheckpointNumber, finalizedCheckpointNumber] = await Promise.all([
+        this.getProvenCheckpointNumber(),
+        this.#lastFinalizedCheckpoint.getAsync(),
+      ]);
+      return (finalizedCheckpointNumber ?? 0) > provenCheckpointNumber
+        ? provenCheckpointNumber
+        : CheckpointNumber(finalizedCheckpointNumber ?? 0);
+    });
   }
 
   setFinalizedCheckpointNumber(checkpointNumber: CheckpointNumber) {