TSAN fixes: Refactor mutex handling

src/consensus/aft/raft.h

@@ -207,14 +207,39 @@ namespace aft
     std::unique_ptr<LedgerProxy> ledger;
     std::shared_ptr<ccf::NodeToNode> channels;
 
+    enum class StartupRole : std::uint8_t
+    {
+      Primary,
+      Backup,
+    };
+
+    // Describes the initial role and state for this node at construction
+    // time, before any other thread can observe it (so no lock is needed).
+    struct StartupState
+    {
+      StartupRole role;
+
+      // State to apply before becoming primary/backup. When nullopt for
+      // a primary, the node starts from scratch (genesis).
+      struct StateInfo
+      {
+        Index index = 0;
+        Term term = 0;
+        std::vector<Index> view_history;
+        Index recovery_start_index = 0;
+      };
+      std::optional<StateInfo> info = std::nullopt;
+    };
+
     Aft(
       const ccf::consensus::Configuration& settings_,
       std::unique_ptr<Store> store_,
       std::unique_ptr<LedgerProxy> ledger_,
       std::shared_ptr<ccf::NodeToNode> channels_,
       std::shared_ptr<aft::State> state_,
       std::shared_ptr<ccf::NodeClient> rpc_request_context_,
-      bool public_only_ = false) :
+      bool public_only_ = false,
+      std::optional<StartupState> startup = std::nullopt) :
       store(std::move(store_)),
 
       timeout_elapsed(0),
@@ -236,7 +261,38 @@ namespace aft
 
       ledger(std::move(ledger_)),
       channels(std::move(channels_))
-    {}
+    {
+      if (startup.has_value())
+      {
+        const auto& s = startup.value();
+        if (s.info.has_value())
+        {
+          const auto& si = s.info.value();
+          if (s.role == StartupRole::Primary)
+          {
+            state->current_view = si.term;
+            state->last_idx = si.index;
+            state->commit_idx = si.index;
+            state->view_history.initialise(si.view_history);
+            state->view_history.update(si.index, si.term);
+          }
+          else
+          {
+            state->last_idx = si.index;
+            state->commit_idx = si.index;
+            state->view_history.initialise(si.view_history);
+            ledger->init(si.index, si.recovery_start_index);
+            become_aware_of_new_term(si.term);
+          }
+        }
+
+        if (s.role == StartupRole::Primary)
+        {
+          state->current_view += starting_view_change;
+          become_leader(true);
+        }
+      }
+    }
 
     ~Aft() override = default;
 

src/enclave/rpc_handler.h

@@ -14,6 +14,8 @@
 namespace ccf::kv
 {
   class CommittableTx;
+  class Consensus;
+  class TxHistory;
 }
 
 namespace ccf
@@ -33,6 +35,8 @@ namespace ccf
     virtual void tick(std::chrono::milliseconds /*elapsed*/) {}
     virtual void open() = 0;
     virtual bool is_open() = 0;
+    virtual void set_consensus_and_history(
+      ccf::kv::Consensus* consensus, ccf::kv::TxHistory* history) = 0;
 
     // Used by rpcendpoint to process incoming client RPCs
     virtual void process(std::shared_ptr<RpcContextImpl> ctx) = 0;

src/node/ledger_secrets.h

@@ -170,10 +170,10 @@ namespace ccf
 
     VersionedLedgerSecret get_latest(ccf::kv::ReadOnlyTx& tx)
     {
-      std::lock_guard<ccf::pal::Mutex> guard(lock);
-
       take_dependency_on_secrets(tx);
 
+      std::lock_guard<ccf::pal::Mutex> guard(lock);
+
       if (ledger_secrets.empty())
       {
         throw std::logic_error(
@@ -186,10 +186,10 @@ namespace ccf
     std::pair<VersionedLedgerSecret, std::optional<VersionedLedgerSecret>>
     get_latest_and_penultimate(ccf::kv::ReadOnlyTx& tx)
     {
-      std::lock_guard<ccf::pal::Mutex> guard(lock);
-
       take_dependency_on_secrets(tx);
 
+      std::lock_guard<ccf::pal::Mutex> guard(lock);
+
       if (ledger_secrets.empty())
       {
         throw std::logic_error(
@@ -209,10 +209,10 @@ namespace ccf
       ccf::kv::ReadOnlyTx& tx,
       std::optional<ccf::kv::Version> up_to = std::nullopt)
     {
-      std::lock_guard<ccf::pal::Mutex> guard(lock);
-
       take_dependency_on_secrets(tx);
 
+      std::lock_guard<ccf::pal::Mutex> guard(lock);
+
       if (!up_to.has_value())
       {
         return ledger_secrets;