Merge #206: feat: add redb as an alternative to sqlite

83e6dd6 refactor: add docs,version and change visibility (codingp110)
7bcbcf0 feat: add redb as an alternative to sqlite (codingp110)

Pull request description:

  <!-- You can erase any parts of this template not applicable to your Pull Request. -->

  ### Description
  We currently only support `sqlite` persistence. This PR adds an alternative `redb` persistence by leveraging Summer Of Bitcoin work on [`bdk_redb`](https://github.com/110CodingP/bdk_redb).
  <!-- Describe the purpose of this PR, what's being adding and/or fixed -->

  ### Notes to the reviewers
  The CI does not pass since `bdk_redb` has a MSRV of 1.85.0 which is in turn due to `redb`. Also `bdk_redb` is yet to published as a crate so currently we use the GitHub version of the same.

  The following script tests a simple scenario of creating a wallet with bitcoind-rpc as chain source , sending funds to an address controlled by the wallet and creating a transaction using the wallet:
  ```
  rm -rf ~/.bitcoin/regtest
  rm -rf ~/.bdk-bitcoin

  bitcoin-cli -rpcuser=alice -rpcpassword=password createwallet alice

  export NETWORK=regtest

  export EXT_DESCRIPTOR=$(bitcoin-cli -rpcwallet=alice -rpcuser=alice -rpcpassword=password listdescriptors true | jq -r '.descriptors | .[0] | .desc')

  export INT_DESCRIPTOR=$(bitcoin-cli -rpcwallet=alice -rpcuser=alice -rpcpassword=password listdescriptors true | jq -r '.descriptors | .[1] | .desc')

  export DATABASE_TYPE=redb
  export CLIENT_TYPE=rpc
  export SERVER_URL=127.0.0.1:18443

  addr=$(bdk-cli wallet -w alice -a alice:password new_address | jq -r '.address')

  bitcoin-cli -rpcuser=alice -rpcpassword=password -rpcwallet=alice generatetoaddress 102 $addr

  bdk-cli wallet -w alice -a alice:password full_scan
  bdk-cli wallet -w alice -a alice:password balance

  recipient=$(bitcoin-cli  -rpcwallet=alice getrawchangeaddress)
  psbt=$(bdk-cli wallet -w alice -a alice:password create_tx --to "$recipient:5000" | jq -r '.psbt')

  signed_psbt=$(bdk-cli wallet -w alice -a alice:password sign $psbt | jq -r '.psbt')
  bdk-cli wallet -w alice -a alice:password broadcast --psbt $signed_psbt

  unset $addr

  addr=$(bdk-cli wallet -w alice -a alice:password new_address | jq -r '.address')
  bitcoin-cli -rpcuser=alice -rpcpassword=password -rpcwallet=alice generatetoaddress 1 $addr

  bdk-cli wallet -w alice -a alice:password sync
  bdk-cli wallet -w alice -a alice:password balance
  ```

  the conf is as follows:
  ```
  regtest=1

  [regtest]
  server=1
  rpcuser=alice
  rpcpassword=password
  ```

  ~~Also removed `sqlite` from default features since we now have an alternative.~~

  <!-- In this section you can include notes directed to the reviewers, like explaining why some parts
  of the PR were done in a specific way -->

  ## Changelog notice

  <!-- Notice the release manager should include in the release tag message changelog -->
  <!-- See https://keepachangelog.com/en/1.0.0/ for examples -->

  ### Checklists

  #### All Submissions:

  * [x] I've signed all my commits
  * [x] I followed the [contribution guidelines](https://github.com/bitcoindevkit/bdk-cli/blob/master/CONTRIBUTING.md)
  * [x] I ran `cargo fmt` and `cargo clippy` before committing

  #### New Features:

   * [x] I've added tests for the new feature (test is in the PR description)
   * [x] I've added docs for the new feature
   * [x] I've updated `CHANGELOG.md`

ACKs for top commit:
  notmandatory:
    utACK 83e6dd6

Tree-SHA512: 50481b0cc293437145c5b1c3671e21b4a2bed32733e86a05a3404e13b70c77bd04754c53411656c7e84a4fc6efcf92659949f2c02f2135fabb0612b010ef4861

Author: tvpeter

Cargo.lock

@@ -26,6 +26,7 @@ bdk_bitcoind_rpc = { version = "0.20.0", optional = true }
 bdk_electrum = { version = "0.23.0", optional = true }
 bdk_esplora = { version = "0.22.0", features = ["async-https", "tokio"], optional = true }
 bdk_kyoto = { version = "0.11.0", optional = true }
+bdk_redb = { version = "0.1.0", optional = true }
 shlex = {  version = "1.3.0", optional = true }
 tracing = "0.1.41"
 tracing-subscriber = "0.3.19"
@@ -38,12 +39,13 @@ repl = ["shlex"]
 
 # Available database options
 sqlite = ["bdk_wallet/rusqlite"]
+redb = ["bdk_redb"]
 
 # Available blockchain client options
 cbf = ["bdk_kyoto"]
 electrum = ["bdk_electrum"]
 esplora = ["bdk_esplora"]
-rpc = ["bdk_bitcoind_rpc"] 
+rpc = ["bdk_bitcoind_rpc"]
 
 # Use this to consensus verify transactions at sync time
 verify = []

Cargo.toml

@@ -126,6 +126,9 @@ pub enum DatabaseType {
     /// Sqlite database
     #[cfg(feature = "sqlite")]
     Sqlite,
+    /// Redb database
+    #[cfg(feature = "redb")]
+    Redb,
 }
 
 #[cfg(any(
@@ -169,7 +172,7 @@ pub struct WalletOpts {
     ))]
     #[arg(env = "CLIENT_TYPE", short = 'c', long, value_enum, required = true)]
     pub client_type: ClientType,
-    #[cfg(feature = "sqlite")]
+    #[cfg(any(feature = "sqlite", feature = "redb"))]
     #[arg(env = "DATABASE_TYPE", short = 'd', long, value_enum, required = true)]
     pub database_type: DatabaseType,
     /// Sets the server url.

src/commands.rs

@@ -57,9 +57,18 @@ pub enum BDKCliError {
     #[error("PsbtError: {0}")]
     PsbtError(#[from] bdk_wallet::bitcoin::psbt::Error),
 
+    #[cfg(feature = "sqlite")]
     #[error("Rusqlite error: {0}")]
     RusqliteError(#[from] bdk_wallet::rusqlite::Error),
 
+    #[cfg(feature = "redb")]
+    #[error("Redb StoreError: {0}")]
+    RedbStoreError(#[from] bdk_redb::error::StoreError),
+
+    #[cfg(feature = "redb")]
+    #[error("Redb dabtabase error: {0}")]
+    RedbDatabaseError(#[from] bdk_redb::redb::DatabaseError),
+
     #[error("Serde json error: {0}")]
     SerdeJson(#[from] serde_json::Error),
 

src/error.rs

@@ -13,9 +13,13 @@
 use crate::commands::OfflineWalletSubCommand::*;
 use crate::commands::*;
 use crate::error::BDKCliError as Error;
+#[cfg(any(feature = "sqlite", feature = "redb"))]
+use crate::persister::Persister;
 #[cfg(feature = "cbf")]
 use crate::utils::BlockchainClient::KyotoClient;
 use crate::utils::*;
+#[cfg(feature = "redb")]
+use bdk_redb::Store as RedbStore;
 use bdk_wallet::bip39::{Language, Mnemonic};
 use bdk_wallet::bitcoin::Network;
 use bdk_wallet::bitcoin::bip32::{DerivationPath, KeySource};
@@ -51,6 +55,8 @@ use crate::utils::BlockchainClient::Electrum;
 #[cfg(feature = "cbf")]
 use bdk_kyoto::{Info, LightClient};
 use bdk_wallet::bitcoin::base64::prelude::*;
+#[cfg(feature = "redb")]
+use std::sync::Arc;
 #[cfg(feature = "cbf")]
 use tokio::select;
 #[cfg(any(
@@ -751,15 +757,28 @@ pub(crate) async fn handle_command(cli_opts: CliOpts) -> Result<String, Error> {
             let home_dir = prepare_home_dir(cli_opts.datadir)?;
             let wallet_name = &wallet_opts.wallet;
             let database_path = prepare_wallet_db_dir(wallet_name, &home_dir)?;
-            #[cfg(feature = "sqlite")]
+
+            #[cfg(any(feature = "sqlite", feature = "redb"))]
             let result = {
-                let mut persister = match &wallet_opts.database_type {
+                let mut persister: Persister = match &wallet_opts.database_type {
                     #[cfg(feature = "sqlite")]
                     DatabaseType::Sqlite => {
                         let db_file = database_path.join("wallet.sqlite");
                         let connection = Connection::open(db_file)?;
                         log::debug!("Sqlite database opened successfully");
-                        connection
+                        Persister::Connection(connection)
+                    }
+                    #[cfg(feature = "redb")]
+                    DatabaseType::Redb => {
+                        let db = Arc::new(bdk_redb::redb::Database::create(
+                            home_dir.join("wallet.redb"),
+                        )?);
+                        let store = RedbStore::new(
+                            db,
+                            wallet_name.as_deref().unwrap_or("wallet").to_string(),
+                        )?;
+                        log::debug!("Redb database opened successfully");
+                        Persister::RedbStore(store)
                     }
                 };
 
@@ -776,7 +795,7 @@ pub(crate) async fn handle_command(cli_opts: CliOpts) -> Result<String, Error> {
                 wallet.persist(&mut persister)?;
                 result
             };
-            #[cfg(not(any(feature = "sqlite")))]
+            #[cfg(not(any(feature = "sqlite", feature = "redb")))]
             let result = {
                 let wallet = new_wallet(network, &wallet_opts)?;
                 let blockchain_client =
@@ -792,18 +811,30 @@ pub(crate) async fn handle_command(cli_opts: CliOpts) -> Result<String, Error> {
             subcommand: WalletSubCommand::OfflineWalletSubCommand(offline_subcommand),
         } => {
             let network = cli_opts.network;
-            #[cfg(feature = "sqlite")]
+            #[cfg(any(feature = "sqlite", feature = "redb"))]
             let result = {
                 let home_dir = prepare_home_dir(cli_opts.datadir)?;
                 let wallet_name = &wallet_opts.wallet;
-                let database_path = prepare_wallet_db_dir(wallet_name, &home_dir)?;
-                let mut persister = match &wallet_opts.database_type {
+                let mut persister: Persister = match &wallet_opts.database_type {
                     #[cfg(feature = "sqlite")]
                     DatabaseType::Sqlite => {
+                        let database_path = prepare_wallet_db_dir(wallet_name, &home_dir)?;
                         let db_file = database_path.join("wallet.sqlite");
                         let connection = Connection::open(db_file)?;
                         log::debug!("Sqlite database opened successfully");
-                        connection
+                        Persister::Connection(connection)
+                    }
+                    #[cfg(feature = "redb")]
+                    DatabaseType::Redb => {
+                        let db = Arc::new(bdk_redb::redb::Database::create(
+                            home_dir.join("wallet.redb"),
+                        )?);
+                        let store = RedbStore::new(
+                            db,
+                            wallet_name.as_deref().unwrap_or("wallet").to_string(),
+                        )?;
+                        log::debug!("Redb database opened successfully");
+                        Persister::RedbStore(store)
                     }
                 };
 
@@ -816,7 +847,7 @@ pub(crate) async fn handle_command(cli_opts: CliOpts) -> Result<String, Error> {
                 wallet.persist(&mut persister)?;
                 result
             };
-            #[cfg(not(any(feature = "sqlite")))]
+            #[cfg(not(any(feature = "sqlite", feature = "redb")))]
             let result = {
                 let mut wallet = new_wallet(network, &wallet_opts)?;
                 handle_offline_wallet_subcommand(&mut wallet, &wallet_opts, offline_subcommand)?
@@ -840,27 +871,38 @@ pub(crate) async fn handle_command(cli_opts: CliOpts) -> Result<String, Error> {
         #[cfg(feature = "repl")]
         CliSubCommand::Repl { wallet_opts } => {
             let network = cli_opts.network;
-            #[cfg(feature = "sqlite")]
+            #[cfg(any(feature = "sqlite", feature = "redb"))]
             let (mut wallet, mut persister) = {
                 let wallet_name = &wallet_opts.wallet;
 
                 let home_dir = prepare_home_dir(cli_opts.datadir.clone())?;
 
-                let database_path = prepare_wallet_db_dir(wallet_name, &home_dir)?;
-
-                let mut persister = match &wallet_opts.database_type {
+                let mut persister: Persister = match &wallet_opts.database_type {
                     #[cfg(feature = "sqlite")]
                     DatabaseType::Sqlite => {
+                        let database_path = prepare_wallet_db_dir(wallet_name, &home_dir)?;
                         let db_file = database_path.join("wallet.sqlite");
                         let connection = Connection::open(db_file)?;
                         log::debug!("Sqlite database opened successfully");
-                        connection
+                        Persister::Connection(connection)
+                    }
+                    #[cfg(feature = "redb")]
+                    DatabaseType::Redb => {
+                        let db = Arc::new(bdk_redb::redb::Database::create(
+                            home_dir.join("wallet.redb"),
+                        )?);
+                        let store = RedbStore::new(
+                            db,
+                            wallet_name.as_deref().unwrap_or("wallet").to_string(),
+                        )?;
+                        log::debug!("Redb database opened successfully");
+                        Persister::RedbStore(store)
                     }
                 };
                 let wallet = new_persisted_wallet(network, &mut persister, &wallet_opts)?;
                 (wallet, persister)
             };
-            #[cfg(not(any(feature = "sqlite")))]
+            #[cfg(not(any(feature = "sqlite", feature = "redb")))]
             let mut wallet = new_wallet(network, &wallet_opts)?;
             let home_dir = prepare_home_dir(cli_opts.datadir.clone())?;
             let database_path = prepare_wallet_db_dir(&wallet_opts.wallet, &home_dir)?;
@@ -880,7 +922,7 @@ pub(crate) async fn handle_command(cli_opts: CliOpts) -> Result<String, Error> {
                     database_path.clone(),
                 )
                 .await;
-                #[cfg(feature = "sqlite")]
+                #[cfg(any(feature = "sqlite", feature = "redb"))]
                 wallet.persist(&mut persister)?;
 
                 match result {

src/handlers.rs

@@ -13,6 +13,8 @@
 mod commands;
 mod error;
 mod handlers;
+#[cfg(any(feature = "sqlite", feature = "redb"))]
+mod persister;
 mod utils;
 
 use bdk_wallet::bitcoin::Network;

src/main.rs

@@ -0,0 +1,40 @@
+use crate::error::BDKCliError;
+use bdk_wallet::WalletPersister;
+
+// Types of Persistence backends supported by bdk-cli
+pub(crate) enum Persister {
+    #[cfg(feature = "sqlite")]
+    Connection(bdk_wallet::rusqlite::Connection),
+    #[cfg(feature = "redb")]
+    RedbStore(bdk_redb::Store),
+}
+
+impl WalletPersister for Persister {
+    type Error = BDKCliError;
+
+    fn initialize(persister: &mut Self) -> Result<bdk_wallet::ChangeSet, Self::Error> {
+        match persister {
+            #[cfg(feature = "sqlite")]
+            Persister::Connection(connection) => {
+                WalletPersister::initialize(connection).map_err(BDKCliError::from)
+            }
+            #[cfg(feature = "redb")]
+            Persister::RedbStore(store) => {
+                WalletPersister::initialize(store).map_err(BDKCliError::from)
+            }
+        }
+    }
+
+    fn persist(persister: &mut Self, changeset: &bdk_wallet::ChangeSet) -> Result<(), Self::Error> {
+        match persister {
+            #[cfg(feature = "sqlite")]
+            Persister::Connection(connection) => {
+                WalletPersister::persist(connection, changeset).map_err(BDKCliError::from)
+            }
+            #[cfg(feature = "redb")]
+            Persister::RedbStore(store) => {
+                WalletPersister::persist(store, changeset).map_err(BDKCliError::from)
+            }
+        }
+    }
+}