diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 6d0056e9aa..cd3980b9af 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -43,4 +43,4 @@ jobs: echo "ELECTRS_EXE=$( pwd )/bin/electrs-${{ runner.os }}-${{ runner.arch }}" >> "$GITHUB_ENV" - name: Run benchmarks run: | - cargo bench + RUSTFLAGS="--cfg tokio_unstable" cargo bench diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index b2575aca1f..16064fa45c 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -84,7 +84,7 @@ jobs: - name: Test on Rust ${{ matrix.toolchain }} if: "matrix.platform != 'windows-latest'" run: | - RUSTFLAGS="--cfg no_download --cfg cycle_tests" cargo test + RUSTFLAGS="--cfg no_download --cfg cycle_tests --cfg tokio_unstable" cargo test - name: Test with UniFFI support on Rust ${{ matrix.toolchain }} if: "matrix.platform != 'windows-latest' && matrix.build-uniffi" run: | @@ -114,4 +114,4 @@ jobs: - uses: actions/checkout@v6 - uses: dtolnay/rust-toolchain@nightly - uses: dtolnay/install@cargo-docs-rs - - run: cargo docs-rs \ No newline at end of file + - run: cargo docs-rs diff --git a/Cargo.toml b/Cargo.toml index 2a60c8cedc..72db969b5a 100755 --- a/Cargo.toml +++ b/Cargo.toml @@ -196,3 +196,17 @@ harness = false #lightning-liquidity = { path = "../rust-lightning/lightning-liquidity" } #lightning-macros = { path = "../rust-lightning/lightning-macros" } #lightning-dns-resolver = { path = "../rust-lightning/lightning-dns-resolver" } + +[patch."https://github.com/lightningdevkit/rust-lightning"] +lightning = { git = "https://github.com/tnull/rust-lightning", rev = "2611766e6a5913d6e33afdc1932485575ee8ea4e" } +lightning-types = { git = "https://github.com/tnull/rust-lightning", rev = "2611766e6a5913d6e33afdc1932485575ee8ea4e" } +lightning-invoice = { git = "https://github.com/tnull/rust-lightning", rev = "2611766e6a5913d6e33afdc1932485575ee8ea4e" } +lightning-net-tokio = { git = "https://github.com/tnull/rust-lightning", rev = "2611766e6a5913d6e33afdc1932485575ee8ea4e" } +lightning-persister = { git = "https://github.com/tnull/rust-lightning", rev = "2611766e6a5913d6e33afdc1932485575ee8ea4e" } +lightning-background-processor = { git = "https://github.com/tnull/rust-lightning", rev = "2611766e6a5913d6e33afdc1932485575ee8ea4e" } +lightning-rapid-gossip-sync = { git = "https://github.com/tnull/rust-lightning", rev = "2611766e6a5913d6e33afdc1932485575ee8ea4e" } +lightning-block-sync = { git = "https://github.com/tnull/rust-lightning", rev = "2611766e6a5913d6e33afdc1932485575ee8ea4e" } +lightning-transaction-sync = { git = "https://github.com/tnull/rust-lightning", rev = "2611766e6a5913d6e33afdc1932485575ee8ea4e" } +lightning-liquidity = { git = "https://github.com/tnull/rust-lightning", rev = "2611766e6a5913d6e33afdc1932485575ee8ea4e" } +lightning-macros = { git = "https://github.com/tnull/rust-lightning", rev = "2611766e6a5913d6e33afdc1932485575ee8ea4e" } +lightning-dns-resolver = { git = "https://github.com/tnull/rust-lightning", rev = "2611766e6a5913d6e33afdc1932485575ee8ea4e" } diff --git a/scripts/uniffi_bindgen_generate_kotlin.sh b/scripts/uniffi_bindgen_generate_kotlin.sh index dc0237ba68..f82d5c0d0d 100755 --- a/scripts/uniffi_bindgen_generate_kotlin.sh +++ b/scripts/uniffi_bindgen_generate_kotlin.sh @@ -5,6 +5,11 @@ PROJECT_DIR="ldk-node-jvm" PACKAGE_DIR="org/lightningdevkit/ldknode" UNIFFI_BINDGEN_BIN="cargo run --manifest-path bindings/uniffi-bindgen/Cargo.toml" +case " ${RUSTFLAGS:-} " in + *" --cfg tokio_unstable "*|*" --cfg=tokio_unstable "*) ;; + *) export RUSTFLAGS="${RUSTFLAGS:+$RUSTFLAGS }--cfg tokio_unstable" ;; +esac + if [[ "$OSTYPE" == "linux-gnu"* ]]; then rustup target add x86_64-unknown-linux-gnu || exit 1 cargo build --release --target x86_64-unknown-linux-gnu --features uniffi || exit 1 diff --git a/scripts/uniffi_bindgen_generate_kotlin_android.sh b/scripts/uniffi_bindgen_generate_kotlin_android.sh index 161292857c..d0eb8654d3 100755 --- a/scripts/uniffi_bindgen_generate_kotlin_android.sh +++ b/scripts/uniffi_bindgen_generate_kotlin_android.sh @@ -5,6 +5,11 @@ TARGET_DIR="target" PROJECT_DIR="ldk-node-android" UNIFFI_BINDGEN_BIN="cargo run --manifest-path bindings/uniffi-bindgen/Cargo.toml" +case " ${RUSTFLAGS:-} " in + *" --cfg tokio_unstable "*|*" --cfg=tokio_unstable "*) RUSTFLAGS_WITH_TOKIO_UNSTABLE="${RUSTFLAGS:-}" ;; + *) RUSTFLAGS_WITH_TOKIO_UNSTABLE="${RUSTFLAGS:+$RUSTFLAGS }--cfg tokio_unstable" ;; +esac + export_variable_if_not_present() { local name="$1" local value="$2" @@ -35,9 +40,9 @@ case "$OSTYPE" in PATH="$ANDROID_NDK_ROOT/toolchains/llvm/prebuilt/$LLVM_ARCH_PATH/bin:$PATH" rustup target add x86_64-linux-android aarch64-linux-android armv7-linux-androideabi -RUSTFLAGS="-C link-args=-Wl,-z,max-page-size=16384,-z,common-page-size=16384" CFLAGS="-D__ANDROID_MIN_SDK_VERSION__=21" AR=llvm-ar CARGO_TARGET_X86_64_LINUX_ANDROID_LINKER="x86_64-linux-android21-clang" CC="x86_64-linux-android21-clang" cargo build --profile release-smaller --features uniffi --target x86_64-linux-android || exit 1 -RUSTFLAGS="-C link-args=-Wl,-z,max-page-size=16384,-z,common-page-size=16384" CFLAGS="-D__ANDROID_MIN_SDK_VERSION__=21" AR=llvm-ar CARGO_TARGET_ARMV7_LINUX_ANDROIDEABI_LINKER="armv7a-linux-androideabi21-clang" CC="armv7a-linux-androideabi21-clang" cargo build --profile release-smaller --features uniffi --target armv7-linux-androideabi || exit 1 -RUSTFLAGS="-C link-args=-Wl,-z,max-page-size=16384,-z,common-page-size=16384" CFLAGS="-D__ANDROID_MIN_SDK_VERSION__=21" AR=llvm-ar CARGO_TARGET_AARCH64_LINUX_ANDROID_LINKER="aarch64-linux-android21-clang" CC="aarch64-linux-android21-clang" cargo build --profile release-smaller --features uniffi --target aarch64-linux-android || exit 1 +RUSTFLAGS="$RUSTFLAGS_WITH_TOKIO_UNSTABLE -C link-args=-Wl,-z,max-page-size=16384,-z,common-page-size=16384" CFLAGS="-D__ANDROID_MIN_SDK_VERSION__=21" AR=llvm-ar CARGO_TARGET_X86_64_LINUX_ANDROID_LINKER="x86_64-linux-android21-clang" CC="x86_64-linux-android21-clang" cargo build --profile release-smaller --features uniffi --target x86_64-linux-android || exit 1 +RUSTFLAGS="$RUSTFLAGS_WITH_TOKIO_UNSTABLE -C link-args=-Wl,-z,max-page-size=16384,-z,common-page-size=16384" CFLAGS="-D__ANDROID_MIN_SDK_VERSION__=21" AR=llvm-ar CARGO_TARGET_ARMV7_LINUX_ANDROIDEABI_LINKER="armv7a-linux-androideabi21-clang" CC="armv7a-linux-androideabi21-clang" cargo build --profile release-smaller --features uniffi --target armv7-linux-androideabi || exit 1 +RUSTFLAGS="$RUSTFLAGS_WITH_TOKIO_UNSTABLE -C link-args=-Wl,-z,max-page-size=16384,-z,common-page-size=16384" CFLAGS="-D__ANDROID_MIN_SDK_VERSION__=21" AR=llvm-ar CARGO_TARGET_AARCH64_LINUX_ANDROID_LINKER="aarch64-linux-android21-clang" CC="aarch64-linux-android21-clang" cargo build --profile release-smaller --features uniffi --target aarch64-linux-android || exit 1 $UNIFFI_BINDGEN_BIN generate bindings/ldk_node.udl --lib-file "$TARGET_DIR"/x86_64-linux-android/release-smaller/libldk_node.so --language kotlin --config uniffi-android.toml -o "$BINDINGS_DIR"/"$PROJECT_DIR"/lib/src/main/kotlin || exit 1 JNI_LIB_DIR="$BINDINGS_DIR"/"$PROJECT_DIR"/lib/src/main/jniLibs/ diff --git a/scripts/uniffi_bindgen_generate_python.sh b/scripts/uniffi_bindgen_generate_python.sh index 50ba450b76..abc4088ffa 100755 --- a/scripts/uniffi_bindgen_generate_python.sh +++ b/scripts/uniffi_bindgen_generate_python.sh @@ -2,6 +2,11 @@ BINDINGS_DIR="./bindings/python/src/ldk_node" UNIFFI_BINDGEN_BIN="cargo run --manifest-path bindings/uniffi-bindgen/Cargo.toml" +case " ${RUSTFLAGS:-} " in + *" --cfg tokio_unstable "*|*" --cfg=tokio_unstable "*) ;; + *) export RUSTFLAGS="${RUSTFLAGS:+$RUSTFLAGS }--cfg tokio_unstable" ;; +esac + if [[ "$OSTYPE" == "linux-gnu"* ]]; then DYNAMIC_LIB_PATH="./target/release-smaller/libldk_node.so" else diff --git a/scripts/uniffi_bindgen_generate_swift.sh b/scripts/uniffi_bindgen_generate_swift.sh index d4c900e405..d69ac1fbea 100755 --- a/scripts/uniffi_bindgen_generate_swift.sh +++ b/scripts/uniffi_bindgen_generate_swift.sh @@ -4,6 +4,11 @@ set -eox pipefail BINDINGS_DIR="./bindings/swift" UNIFFI_BINDGEN_BIN="cargo run --manifest-path bindings/uniffi-bindgen/Cargo.toml" +case " ${RUSTFLAGS:-} " in + *" --cfg tokio_unstable "*|*" --cfg=tokio_unstable "*) ;; + *) export RUSTFLAGS="${RUSTFLAGS:+$RUSTFLAGS }--cfg tokio_unstable" ;; +esac + mkdir -p $BINDINGS_DIR # Install rust target toolchains diff --git a/src/builder.rs b/src/builder.rs index 03ded494f9..c88c867cc1 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -81,11 +81,11 @@ use crate::tx_broadcaster::TransactionBroadcaster; use crate::types::{ AsyncPersister, ChainMonitor, ChannelManager, DynStore, DynStoreRef, DynStoreWrapper, GossipSync, Graph, HRNResolver, KeysManager, MessageRouter, OnionMessenger, PaymentStore, - PeerManager, PendingPaymentStore, SyncAndAsyncKVStore, + PeerManager, PendingPaymentStore, }; use crate::wallet::persist::KVStoreWalletPersister; use crate::wallet::Wallet; -use crate::{Node, NodeMetrics}; +use crate::{Node, NodeMetrics, PersistedNodeMetrics}; const LSPS_HARDENED_CHILD_INDEX: u32 = 577; const PERSISTER_MAX_PENDING_UPDATES: u64 = 100; @@ -176,17 +176,17 @@ pub enum BuildError { RuntimeSetupFailed, /// We failed to read data from the [`KVStore`]. /// - /// [`KVStore`]: lightning::util::persist::KVStoreSync + /// [`KVStore`]: lightning::util::persist::KVStore ReadFailed, /// We failed to write data to the [`KVStore`]. /// - /// [`KVStore`]: lightning::util::persist::KVStoreSync + /// [`KVStore`]: lightning::util::persist::KVStore WriteFailed, /// We failed to access the given `storage_dir_path`. StoragePathAccessFailed, /// We failed to setup our [`KVStore`]. /// - /// [`KVStore`]: lightning::util::persist::KVStoreSync + /// [`KVStore`]: lightning::util::persist::KVStore KVStoreSetupFailed, /// We failed to setup the onchain wallet. WalletSetupFailed, @@ -697,11 +697,12 @@ impl NodeBuilder { /// [`FilesystemStoreV2`]: lightning_persister::fs_store::v2::FilesystemStoreV2 pub fn build_with_fs_store(&self, node_entropy: NodeEntropy) -> Result { let logger = setup_logger(&self.log_writer_config, &self.config)?; + let runtime = self.setup_runtime(&logger)?; let mut storage_dir_path: PathBuf = self.config.storage_dir_path.clone().into(); storage_dir_path.push("fs_store"); - let kv_store = open_or_migrate_fs_store(storage_dir_path)?; - self.build_with_store_and_logger(node_entropy, kv_store, logger) + let kv_store = runtime.block_on(open_or_migrate_fs_store(storage_dir_path))?; + self.build_with_store_runtime_and_logger(node_entropy, kv_store, runtime, logger) } /// Builds a [`Node`] instance with a [VSS] backend and according to the options @@ -825,7 +826,7 @@ impl NodeBuilder { } /// Builds a [`Node`] instance according to the options previously configured. - pub fn build_with_store( + pub fn build_with_store( &self, node_entropy: NodeEntropy, kv_store: S, ) -> Result { let logger = setup_logger(&self.log_writer_config, &self.config)?; @@ -844,14 +845,14 @@ impl NodeBuilder { } } - fn build_with_store_and_logger( + fn build_with_store_and_logger( &self, node_entropy: NodeEntropy, kv_store: S, logger: Arc, ) -> Result { let runtime = self.setup_runtime(&logger)?; self.build_with_store_runtime_and_logger(node_entropy, kv_store, runtime, logger) } - fn build_with_store_runtime_and_logger( + fn build_with_store_runtime_and_logger( &self, node_entropy: NodeEntropy, kv_store: S, runtime: Arc, logger: Arc, ) -> Result { let seed_bytes = node_entropy.to_seed_bytes(); @@ -1345,7 +1346,7 @@ impl ArcedNodeBuilder { /// Builds a [`Node`] instance according to the options previously configured. // Note that the generics here don't actually work for Uniffi, but we don't currently expose // this so its not needed. - pub fn build_with_store( + pub fn build_with_store( &self, node_entropy: Arc, kv_store: S, ) -> Result, BuildError> { self.inner.read().expect("lock").build_with_store(*node_entropy, kv_store).map(Arc::new) @@ -1415,10 +1416,10 @@ fn build_with_store_internal( // Initialize the status fields. let node_metrics = match node_metris_res { - Ok(metrics) => Arc::new(RwLock::new(metrics)), + Ok(metrics) => Arc::new(PersistedNodeMetrics::new(metrics)), Err(e) => { if e.kind() == std::io::ErrorKind::NotFound { - Arc::new(RwLock::new(NodeMetrics::default())) + Arc::new(PersistedNodeMetrics::new(NodeMetrics::default())) } else { log_error!(logger, "Failed to read node metrics from store: {}", e); return Err(BuildError::ReadFailed); @@ -1539,12 +1540,16 @@ fn build_with_store_internal( let change_descriptor = Bip84(xprv, KeychainKind::Internal); let mut wallet_persister = KVStoreWalletPersister::new(Arc::clone(&kv_store), Arc::clone(&logger)); - let wallet_opt = BdkWallet::load() - .descriptor(KeychainKind::External, Some(descriptor.clone())) - .descriptor(KeychainKind::Internal, Some(change_descriptor.clone())) - .extract_keys() - .check_network(config.network) - .load_wallet(&mut wallet_persister) + let wallet_opt = runtime + .block_on(async { + BdkWallet::load() + .descriptor(KeychainKind::External, Some(descriptor.clone())) + .descriptor(KeychainKind::Internal, Some(change_descriptor.clone())) + .extract_keys() + .check_network(config.network) + .load_wallet_async(&mut wallet_persister) + .await + }) .map_err(|e| match e { bdk_wallet::LoadWithPersistError::InvalidChangeSet( bdk_wallet::LoadError::Mismatch(bdk_wallet::LoadMismatch::Network { @@ -1568,9 +1573,13 @@ fn build_with_store_internal( let bdk_wallet = match wallet_opt { Some(wallet) => wallet, None => { - let mut wallet = BdkWallet::create(descriptor, change_descriptor) - .network(config.network) - .create_wallet(&mut wallet_persister) + let mut wallet = runtime + .block_on(async { + BdkWallet::create(descriptor, change_descriptor) + .network(config.network) + .create_wallet_async(&mut wallet_persister) + .await + }) .map_err(|e| { log_error!(logger, "Failed to set up wallet: {}", e); BuildError::WalletSetupFailed @@ -1619,6 +1628,7 @@ fn build_with_store_internal( Arc::clone(&fee_estimator), Arc::clone(&chain_source), Arc::clone(&payment_store), + Arc::clone(&runtime), Arc::clone(&config), Arc::clone(&logger), Arc::clone(&pending_payment_store), diff --git a/src/chain/bitcoind.rs b/src/chain/bitcoind.rs index 2582f32f64..6bfa8ffd27 100644 --- a/src/chain/bitcoind.rs +++ b/src/chain/bitcoind.rs @@ -42,7 +42,7 @@ use crate::fee_estimator::{ use crate::io::utils::update_and_persist_node_metrics; use crate::logger::{log_bytes, log_debug, log_error, log_info, log_trace, LdkLogger, Logger}; use crate::types::{ChainMonitor, ChannelManager, DynStore, Sweeper, Wallet}; -use crate::{Error, NodeMetrics}; +use crate::{Error, PersistedNodeMetrics}; const CHAIN_POLLING_INTERVAL_SECS: u64 = 2; const CHAIN_POLLING_TIMEOUT_SECS: u64 = 10; @@ -55,14 +55,14 @@ pub(super) struct BitcoindChainSource { kv_store: Arc, config: Arc, logger: Arc, - node_metrics: Arc>, + node_metrics: Arc, } impl BitcoindChainSource { pub(crate) fn new_rpc( rpc_host: String, rpc_port: u16, rpc_user: String, rpc_password: String, fee_estimator: Arc, kv_store: Arc, config: Arc, - logger: Arc, node_metrics: Arc>, + logger: Arc, node_metrics: Arc, ) -> Self { let api_client = Arc::new(BitcoindClient::new_rpc( rpc_host.clone(), @@ -89,7 +89,7 @@ impl BitcoindChainSource { rpc_host: String, rpc_port: u16, rpc_user: String, rpc_password: String, fee_estimator: Arc, kv_store: Arc, config: Arc, rest_client_config: BitcoindRestClientConfig, logger: Arc, - node_metrics: Arc>, + node_metrics: Arc, ) -> Self { let api_client = Arc::new(BitcoindClient::new_rest( rest_client_config.rest_host, @@ -204,6 +204,7 @@ impl BitcoindChainSource { m.latest_onchain_wallet_sync_timestamp = unix_time_secs_opt; }, ) + .await .unwrap_or_else(|e| { log_error!(self.logger, "Failed to persist node metrics: {}", e); }); @@ -451,7 +452,8 @@ impl BitcoindChainSource { update_and_persist_node_metrics(&self.node_metrics, &*self.kv_store, &*self.logger, |m| { m.latest_lightning_wallet_sync_timestamp = unix_time_secs_opt; m.latest_onchain_wallet_sync_timestamp = unix_time_secs_opt; - })?; + }) + .await?; Ok(()) } @@ -563,7 +565,8 @@ impl BitcoindChainSource { SystemTime::now().duration_since(UNIX_EPOCH).ok().map(|d| d.as_secs()); update_and_persist_node_metrics(&self.node_metrics, &*self.kv_store, &*self.logger, |m| { m.latest_fee_rate_cache_update_timestamp = unix_time_secs_opt - })?; + }) + .await?; Ok(()) } diff --git a/src/chain/electrum.rs b/src/chain/electrum.rs index 54e7fff0ca..ad0ef1b7ba 100644 --- a/src/chain/electrum.rs +++ b/src/chain/electrum.rs @@ -34,7 +34,7 @@ use crate::io::utils::update_and_persist_node_metrics; use crate::logger::{log_bytes, log_debug, log_error, log_trace, LdkLogger, Logger}; use crate::runtime::Runtime; use crate::types::{ChainMonitor, ChannelManager, DynStore, Sweeper, Wallet}; -use crate::NodeMetrics; +use crate::PersistedNodeMetrics; const BDK_ELECTRUM_CLIENT_BATCH_SIZE: usize = 5; const ELECTRUM_CLIENT_NUM_RETRIES: u8 = 3; @@ -49,14 +49,14 @@ pub(super) struct ElectrumChainSource { kv_store: Arc, config: Arc, logger: Arc, - node_metrics: Arc>, + node_metrics: Arc, } impl ElectrumChainSource { pub(super) fn new( server_url: String, sync_config: ElectrumSyncConfig, fee_estimator: Arc, kv_store: Arc, config: Arc, - logger: Arc, node_metrics: Arc>, + logger: Arc, node_metrics: Arc, ) -> Self { let electrum_runtime_status = RwLock::new(ElectrumRuntimeStatus::new()); let onchain_wallet_sync_status = Mutex::new(WalletSyncStatus::Completed); @@ -129,31 +129,6 @@ impl ElectrumChainSource { let incremental_sync = self.node_metrics.read().expect("lock").latest_onchain_wallet_sync_timestamp.is_some(); - let apply_wallet_update = - |update_res: Result, now: Instant| match update_res { - Ok(update) => match onchain_wallet.apply_update(update) { - Ok(()) => { - log_debug!( - self.logger, - "{} of on-chain wallet finished in {}ms.", - if incremental_sync { "Incremental sync" } else { "Sync" }, - now.elapsed().as_millis() - ); - let unix_time_secs_opt = - SystemTime::now().duration_since(UNIX_EPOCH).ok().map(|d| d.as_secs()); - update_and_persist_node_metrics( - &self.node_metrics, - &*self.kv_store, - &*self.logger, - |m| m.latest_onchain_wallet_sync_timestamp = unix_time_secs_opt, - )?; - Ok(()) - }, - Err(e) => Err(e), - }, - Err(e) => Err(e), - }; - let cached_txs = onchain_wallet.get_cached_txs(); let res = if incremental_sync { @@ -162,20 +137,62 @@ impl ElectrumChainSource { .get_incremental_sync_wallet_update(incremental_sync_request, cached_txs); let now = Instant::now(); - let update_res = incremental_sync_fut.await.map(|u| u.into()); - apply_wallet_update(update_res, now) + let update_res: Result = incremental_sync_fut.await.map(|u| u.into()); + self.apply_onchain_wallet_update( + onchain_wallet.as_ref(), + incremental_sync, + update_res, + now, + ) + .await } else { let full_scan_request = onchain_wallet.get_full_scan_request(); let full_scan_fut = electrum_client.get_full_scan_wallet_update(full_scan_request, cached_txs); let now = Instant::now(); - let update_res = full_scan_fut.await.map(|u| u.into()); - apply_wallet_update(update_res, now) + let update_res: Result = full_scan_fut.await.map(|u| u.into()); + self.apply_onchain_wallet_update( + onchain_wallet.as_ref(), + incremental_sync, + update_res, + now, + ) + .await }; res } + async fn apply_onchain_wallet_update( + &self, onchain_wallet: &Wallet, incremental_sync: bool, + update_res: Result, now: Instant, + ) -> Result<(), Error> { + match update_res { + Ok(update) => match onchain_wallet.apply_update(update) { + Ok(()) => { + log_debug!( + self.logger, + "{} of on-chain wallet finished in {}ms.", + if incremental_sync { "Incremental sync" } else { "Sync" }, + now.elapsed().as_millis() + ); + let unix_time_secs_opt = + SystemTime::now().duration_since(UNIX_EPOCH).ok().map(|d| d.as_secs()); + update_and_persist_node_metrics( + &self.node_metrics, + &*self.kv_store, + &*self.logger, + |m| m.latest_onchain_wallet_sync_timestamp = unix_time_secs_opt, + ) + .await?; + Ok(()) + }, + Err(e) => Err(e), + }, + Err(e) => Err(e), + } + } + pub(crate) async fn sync_lightning_wallet( &self, channel_manager: Arc, chain_monitor: Arc, output_sweeper: Arc, @@ -239,7 +256,8 @@ impl ElectrumChainSource { &*self.kv_store, &*self.logger, |m| m.latest_lightning_wallet_sync_timestamp = unix_time_secs_opt, - )?; + ) + .await?; } res @@ -270,7 +288,8 @@ impl ElectrumChainSource { SystemTime::now().duration_since(UNIX_EPOCH).ok().map(|d| d.as_secs()); update_and_persist_node_metrics(&self.node_metrics, &*self.kv_store, &*self.logger, |m| { m.latest_fee_rate_cache_update_timestamp = unix_time_secs_opt - })?; + }) + .await?; Ok(()) } diff --git a/src/chain/esplora.rs b/src/chain/esplora.rs index 5825a09849..eb23a395d3 100644 --- a/src/chain/esplora.rs +++ b/src/chain/esplora.rs @@ -6,7 +6,7 @@ // accordance with one or both of these licenses. use std::collections::HashMap; -use std::sync::{Arc, Mutex, RwLock}; +use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use bdk_esplora::EsploraAsyncExt; @@ -25,7 +25,7 @@ use crate::fee_estimator::{ use crate::io::utils::update_and_persist_node_metrics; use crate::logger::{log_bytes, log_debug, log_error, log_trace, LdkLogger, Logger}; use crate::types::{ChainMonitor, ChannelManager, DynStore, Sweeper, Wallet}; -use crate::{Error, NodeMetrics}; +use crate::{Error, PersistedNodeMetrics}; pub(super) struct EsploraChainSource { pub(super) sync_config: EsploraSyncConfig, @@ -37,14 +37,14 @@ pub(super) struct EsploraChainSource { kv_store: Arc, config: Arc, logger: Arc, - node_metrics: Arc>, + node_metrics: Arc, } impl EsploraChainSource { pub(crate) fn new( server_url: String, headers: HashMap, sync_config: EsploraSyncConfig, fee_estimator: Arc, kv_store: Arc, config: Arc, - logger: Arc, node_metrics: Arc>, + logger: Arc, node_metrics: Arc, ) -> Result { let mut client_builder = esplora_client::Builder::new(&server_url); client_builder = @@ -127,7 +127,8 @@ impl EsploraChainSource { &*self.kv_store, &*self.logger, |m| m.latest_onchain_wallet_sync_timestamp = unix_time_secs_opt, - )?; + ) + .await?; Ok(()) }, Err(e) => Err(e), @@ -265,7 +266,8 @@ impl EsploraChainSource { &*self.kv_store, &*self.logger, |m| m.latest_lightning_wallet_sync_timestamp = unix_time_secs_opt, - )?; + ) + .await?; Ok(()) }, Err(e) => { @@ -347,7 +349,8 @@ impl EsploraChainSource { SystemTime::now().duration_since(UNIX_EPOCH).ok().map(|d| d.as_secs()); update_and_persist_node_metrics(&self.node_metrics, &*self.kv_store, &*self.logger, |m| { m.latest_fee_rate_cache_update_timestamp = unix_time_secs_opt - })?; + }) + .await?; Ok(()) } diff --git a/src/chain/mod.rs b/src/chain/mod.rs index cb8541be6a..92c4bdb641 100644 --- a/src/chain/mod.rs +++ b/src/chain/mod.rs @@ -10,7 +10,7 @@ mod electrum; mod esplora; use std::collections::HashMap; -use std::sync::{Arc, Mutex, RwLock}; +use std::sync::{Arc, Mutex}; use std::time::Duration; use bitcoin::{Script, Txid}; @@ -27,7 +27,7 @@ use crate::fee_estimator::OnchainFeeEstimator; use crate::logger::{log_debug, log_info, log_trace, LdkLogger, Logger}; use crate::runtime::Runtime; use crate::types::{Broadcaster, ChainMonitor, ChannelManager, DynStore, Sweeper, Wallet}; -use crate::{Error, NodeMetrics}; +use crate::{Error, PersistedNodeMetrics}; pub(crate) enum WalletSyncStatus { Completed, @@ -100,7 +100,7 @@ impl ChainSource { server_url: String, headers: HashMap, sync_config: EsploraSyncConfig, fee_estimator: Arc, tx_broadcaster: Arc, kv_store: Arc, config: Arc, logger: Arc, - node_metrics: Arc>, + node_metrics: Arc, ) -> Result<(Self, Option), ()> { let esplora_chain_source = EsploraChainSource::new( server_url, @@ -121,7 +121,7 @@ impl ChainSource { server_url: String, sync_config: ElectrumSyncConfig, fee_estimator: Arc, tx_broadcaster: Arc, kv_store: Arc, config: Arc, logger: Arc, - node_metrics: Arc>, + node_metrics: Arc, ) -> (Self, Option) { let electrum_chain_source = ElectrumChainSource::new( server_url, @@ -141,7 +141,7 @@ impl ChainSource { rpc_host: String, rpc_port: u16, rpc_user: String, rpc_password: String, fee_estimator: Arc, tx_broadcaster: Arc, kv_store: Arc, config: Arc, logger: Arc, - node_metrics: Arc>, + node_metrics: Arc, ) -> (Self, Option) { let bitcoind_chain_source = BitcoindChainSource::new_rpc( rpc_host, @@ -164,7 +164,7 @@ impl ChainSource { rpc_host: String, rpc_port: u16, rpc_user: String, rpc_password: String, fee_estimator: Arc, tx_broadcaster: Arc, kv_store: Arc, config: Arc, rest_client_config: BitcoindRestClientConfig, - logger: Arc, node_metrics: Arc>, + logger: Arc, node_metrics: Arc, ) -> (Self, Option) { let bitcoind_chain_source = BitcoindChainSource::new_rest( rpc_host, diff --git a/src/data_store.rs b/src/data_store.rs index f80ec08915..c080667bac 100644 --- a/src/data_store.rs +++ b/src/data_store.rs @@ -9,7 +9,7 @@ use std::collections::{hash_map, HashMap}; use std::ops::Deref; use std::sync::{Arc, Mutex}; -use lightning::util::persist::KVStoreSync; +use lightning::util::persist::KVStore; use lightning::util::ser::{Readable, Writeable}; use crate::logger::{log_error, LdkLogger}; @@ -45,6 +45,7 @@ where L::Target: LdkLogger, { objects: Mutex>, + mutation_lock: tokio::sync::Mutex<()>, primary_namespace: String, secondary_namespace: String, kv_store: Arc, @@ -61,50 +62,64 @@ where ) -> Self { let objects = Mutex::new(HashMap::from_iter(objects.into_iter().map(|obj| (obj.id(), obj)))); - Self { objects, primary_namespace, secondary_namespace, kv_store, logger } + Self { + objects, + mutation_lock: tokio::sync::Mutex::new(()), + primary_namespace, + secondary_namespace, + kv_store, + logger, + } } - pub(crate) fn insert(&self, object: SO) -> Result { - let mut locked_objects = self.objects.lock().expect("lock"); + pub(crate) async fn insert(&self, object: SO) -> Result { + let _guard = self.mutation_lock.lock().await; - self.persist(&object)?; + self.persist(&object).await?; + let mut locked_objects = self.objects.lock().expect("lock"); let updated = locked_objects.insert(object.id(), object).is_some(); Ok(updated) } - pub(crate) fn insert_or_update(&self, object: SO) -> Result { - let mut locked_objects = self.objects.lock().expect("lock"); + pub(crate) async fn insert_or_update(&self, object: SO) -> Result { + let _guard = self.mutation_lock.lock().await; + let (updated, data_to_persist) = { + let mut locked_objects = self.objects.lock().expect("lock"); + match locked_objects.entry(object.id()) { + hash_map::Entry::Occupied(mut e) => { + let update = object.to_update(); + let updated = e.get_mut().update(update); + let data_to_persist = + if updated { Some(Self::encode_object(e.get())) } else { None }; + (updated, data_to_persist) + }, + hash_map::Entry::Vacant(e) => { + let data_to_persist = Self::encode_object(&object); + e.insert(object); + (true, Some(data_to_persist)) + }, + } + }; - let updated; - match locked_objects.entry(object.id()) { - hash_map::Entry::Occupied(mut e) => { - let update = object.to_update(); - updated = e.get_mut().update(update); - if updated { - self.persist(&e.get())?; - } - }, - hash_map::Entry::Vacant(e) => { - e.insert(object.clone()); - self.persist(&object)?; - updated = true; - }, + if let Some((store_key, data)) = data_to_persist { + self.persist_encoded(store_key, data).await?; } - Ok(updated) } - pub(crate) fn remove(&self, id: &SO::Id) -> Result<(), Error> { - let removed = self.objects.lock().expect("lock").remove(id).is_some(); + pub(crate) async fn remove(&self, id: &SO::Id) -> Result<(), Error> { + let _guard = self.mutation_lock.lock().await; + let removed = { self.objects.lock().expect("lock").remove(id).is_some() }; if removed { let store_key = id.encode_to_hex_str(); - KVStoreSync::remove( + KVStore::remove( &*self.kv_store, &self.primary_namespace, &self.secondary_namespace, &store_key, false, ) + .await .map_err(|e| { log_error!( self.logger, @@ -124,36 +139,49 @@ where self.objects.lock().expect("lock").get(id).cloned() } - pub(crate) fn update(&self, update: SO::Update) -> Result { - let mut locked_objects = self.objects.lock().expect("lock"); - - if let Some(object) = locked_objects.get_mut(&update.id()) { - let updated = object.update(update); - if updated { - self.persist(&object)?; - Ok(DataStoreUpdateResult::Updated) + pub(crate) async fn update(&self, update: SO::Update) -> Result { + let _guard = self.mutation_lock.lock().await; + let (res, data_to_persist) = { + let mut locked_objects = self.objects.lock().expect("lock"); + if let Some(object) = locked_objects.get_mut(&update.id()) { + let updated = object.update(update); + if updated { + (DataStoreUpdateResult::Updated, Some(Self::encode_object(object))) + } else { + (DataStoreUpdateResult::Unchanged, None) + } } else { - Ok(DataStoreUpdateResult::Unchanged) + (DataStoreUpdateResult::NotFound, None) } - } else { - Ok(DataStoreUpdateResult::NotFound) + }; + if let Some((store_key, data)) = data_to_persist { + self.persist_encoded(store_key, data).await?; } + Ok(res) } pub(crate) fn list_filter bool>(&self, f: F) -> Vec { self.objects.lock().expect("lock").values().filter(f).cloned().collect::>() } - fn persist(&self, object: &SO) -> Result<(), Error> { - let store_key = object.id().encode_to_hex_str(); - let data = object.encode(); - KVStoreSync::write( + async fn persist(&self, object: &SO) -> Result<(), Error> { + let (store_key, data) = Self::encode_object(object); + self.persist_encoded(store_key, data).await + } + + fn encode_object(object: &SO) -> (String, Vec) { + (object.id().encode_to_hex_str(), object.encode()) + } + + async fn persist_encoded(&self, store_key: String, data: Vec) -> Result<(), Error> { + KVStore::write( &*self.kv_store, &self.primary_namespace, &self.secondary_namespace, &store_key, data, ) + .await .map_err(|e| { log_error!( self.logger, @@ -175,7 +203,7 @@ where #[cfg(test)] mod tests { - use lightning::impl_writeable_tlv_based; + use lightning::impl_ser_tlv_based; use lightning::util::test_utils::TestLogger; use super::*; @@ -193,7 +221,7 @@ mod tests { hex_utils::to_string(&self.id) } } - impl_writeable_tlv_based!(TestObjectId, { (0, id, required) }); + impl_ser_tlv_based!(TestObjectId, { (0, id, required) }); struct TestObjectUpdate { id: TestObjectId, @@ -233,13 +261,13 @@ mod tests { } } - impl_writeable_tlv_based!(TestObject, { + impl_ser_tlv_based!(TestObject, { (0, id, required), (2, data, required), }); - #[test] - fn data_is_persisted() { + #[tokio::test] + async fn data_is_persisted() { let store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); let logger = Arc::new(TestLogger::new()); let primary_namespace = "datastore_test_primary".to_string(); @@ -258,47 +286,49 @@ mod tests { let store_key = id.encode_to_hex_str(); // Check we start empty. - assert!(KVStoreSync::read(&*store, &primary_namespace, &secondary_namespace, &store_key) + assert!(KVStore::read(&*store, &primary_namespace, &secondary_namespace, &store_key) + .await .is_err()); // Check we successfully store an object and return `false` let object = TestObject { id, data: [23u8; 3] }; - assert_eq!(Ok(false), data_store.insert(object.clone())); + assert_eq!(Ok(false), data_store.insert(object.clone()).await); assert_eq!(Some(object), data_store.get(&id)); - assert!(KVStoreSync::read(&*store, &primary_namespace, &secondary_namespace, &store_key) + assert!(KVStore::read(&*store, &primary_namespace, &secondary_namespace, &store_key) + .await .is_ok()); // Test re-insertion returns `true` let mut override_object = object.clone(); override_object.data = [24u8; 3]; - assert_eq!(Ok(true), data_store.insert(override_object)); + assert_eq!(Ok(true), data_store.insert(override_object).await); assert_eq!(Some(override_object), data_store.get(&id)); // Check update returns `Updated` let update = TestObjectUpdate { id, data: [25u8; 3] }; - assert_eq!(Ok(DataStoreUpdateResult::Updated), data_store.update(update)); + assert_eq!(Ok(DataStoreUpdateResult::Updated), data_store.update(update).await); assert_eq!(data_store.get(&id).unwrap().data, [25u8; 3]); // Check no-op update yields `Unchanged` let update = TestObjectUpdate { id, data: [25u8; 3] }; - assert_eq!(Ok(DataStoreUpdateResult::Unchanged), data_store.update(update)); + assert_eq!(Ok(DataStoreUpdateResult::Unchanged), data_store.update(update).await); // Check bogus update yields `NotFound` let bogus_id = TestObjectId { id: [84u8; 4] }; let update = TestObjectUpdate { id: bogus_id, data: [12u8; 3] }; - assert_eq!(Ok(DataStoreUpdateResult::NotFound), data_store.update(update)); + assert_eq!(Ok(DataStoreUpdateResult::NotFound), data_store.update(update).await); // Check `insert_or_update` inserts unknown objects let iou_id = TestObjectId { id: [55u8; 4] }; let iou_object = TestObject { id: iou_id, data: [34u8; 3] }; - assert_eq!(Ok(true), data_store.insert_or_update(iou_object.clone())); + assert_eq!(Ok(true), data_store.insert_or_update(iou_object.clone()).await); // Check `insert_or_update` doesn't update the same object - assert_eq!(Ok(false), data_store.insert_or_update(iou_object.clone())); + assert_eq!(Ok(false), data_store.insert_or_update(iou_object.clone()).await); // Check `insert_or_update` updates if object changed let mut new_iou_object = iou_object; new_iou_object.data[0] += 1; - assert_eq!(Ok(true), data_store.insert_or_update(new_iou_object)); + assert_eq!(Ok(true), data_store.insert_or_update(new_iou_object).await); } } diff --git a/src/event.rs b/src/event.rs index 7d23be99a6..bed662cebd 100644 --- a/src/event.rs +++ b/src/event.rs @@ -29,7 +29,7 @@ use lightning::util::config::{ChannelConfigOverrides, ChannelConfigUpdate}; use lightning::util::errors::APIError; use lightning::util::persist::KVStore; use lightning::util::ser::{Readable, ReadableArgs, Writeable, Writer}; -use lightning::{impl_writeable_tlv_based, impl_writeable_tlv_based_enum}; +use lightning::{impl_ser_tlv_based, impl_ser_tlv_based_enum}; use lightning_liquidity::lsps2::utils::compute_opening_fee; use lightning_types::payment::{PaymentHash, PaymentPreimage}; @@ -78,7 +78,7 @@ pub struct HTLCLocator { pub node_id: Option, } -impl_writeable_tlv_based!(HTLCLocator, { +impl_ser_tlv_based!(HTLCLocator, { (1, channel_id, required), (3, user_channel_id, option), (5, node_id, option), @@ -294,7 +294,7 @@ pub enum Event { }, } -impl_writeable_tlv_based_enum!(Event, +impl_ser_tlv_based_enum!(Event, (0, PaymentSuccessful) => { (0, payment_hash, required), (1, fee_paid_msat, option), @@ -581,7 +581,7 @@ where } } - fn fail_claimable_payment( + async fn fail_claimable_payment( &self, payment_id: PaymentId, payment_hash: &PaymentHash, ) -> Result<(), ReplayEvent> { self.channel_manager.fail_htlc_backwards(payment_hash); @@ -591,7 +591,7 @@ where status: Some(PaymentStatus::Failed), ..PaymentDetailsUpdate::new(payment_id) }; - match self.payment_store.update(update) { + match self.payment_store.update(update).await { Ok(_) => Ok(()), Err(e) => { log_error!(self.logger, "Failed to access payment store: {}", e); @@ -738,7 +738,7 @@ where status: Some(PaymentStatus::Failed), ..PaymentDetailsUpdate::new(payment_id) }; - match self.payment_store.update(update) { + match self.payment_store.update(update).await { Ok(_) => return Ok(()), Err(e) => { log_error!(self.logger, "Failed to access payment store: {}", e); @@ -781,7 +781,7 @@ where status: Some(PaymentStatus::Failed), ..PaymentDetailsUpdate::new(payment_id) }; - match self.payment_store.update(update) { + match self.payment_store.update(update).await { Ok(_) => return Ok(()), Err(e) => { log_error!(self.logger, "Failed to access payment store: {}", e); @@ -809,7 +809,7 @@ where hex_utils::to_string(&payment_hash.0), counterparty_skimmed_fee_msat, ); - self.fail_claimable_payment(payment_id, &payment_hash)?; + self.fail_claimable_payment(payment_id, &payment_hash).await?; return Ok(()); }; @@ -821,7 +821,7 @@ where counterparty_skimmed_fee_msat, max_total_opening_fee_msat, ); - self.fail_claimable_payment(payment_id, &payment_hash)?; + self.fail_claimable_payment(payment_id, &payment_hash).await?; return Ok(()); } @@ -832,14 +832,14 @@ where counterparty_skimmed_fee_msat: Some(Some(counterparty_skimmed_fee_msat)), ..PaymentDetailsUpdate::new(payment_id) }; - match self.payment_store.update(update) { + match self.payment_store.update(update).await { Ok(_) => (), Err(e) => { log_error!(self.logger, "Failed to access payment store: {}", e); return Err(ReplayEvent()); }, }; - } + }, _ => debug_assert!(false, "We only expect the counterparty to get away with withholding fees for BOLT11 payments."), } } @@ -923,7 +923,7 @@ where PaymentStatus::Pending, ); - match self.payment_store.insert(payment) { + match self.payment_store.insert(payment).await { Ok(false) => (), Ok(true) => { log_error!( @@ -964,7 +964,7 @@ where PaymentStatus::Pending, ); - match self.payment_store.insert(payment) { + match self.payment_store.insert(payment).await { Ok(false) => (), Ok(true) => { log_error!( @@ -1004,7 +1004,7 @@ where status: Some(PaymentStatus::Failed), ..PaymentDetailsUpdate::new(payment_id) }; - match self.payment_store.update(update) { + match self.payment_store.update(update).await { Ok(_) => return Ok(()), Err(e) => { log_error!(self.logger, "Failed to access payment store: {}", e); @@ -1072,7 +1072,7 @@ where }, }; - match self.payment_store.update(update) { + match self.payment_store.update(update).await { Ok(DataStoreUpdateResult::Updated) | Ok(DataStoreUpdateResult::Unchanged) => ( // No need to do anything if the idempotent update was applied, which might // be the result of a replayed event. @@ -1134,7 +1134,7 @@ where ..PaymentDetailsUpdate::new(payment_id) }; - match self.payment_store.update(update) { + match self.payment_store.update(update).await { Ok(_) => {}, Err(e) => { log_error!(self.logger, "Failed to access payment store: {}", e); @@ -1189,7 +1189,7 @@ where status: Some(PaymentStatus::Failed), ..PaymentDetailsUpdate::new(payment_id) }; - match self.payment_store.update(update) { + match self.payment_store.update(update).await { Ok(_) => {}, Err(e) => { log_error!(self.logger, "Failed to access payment store: {}", e); @@ -1524,36 +1524,38 @@ where }, }; - let network_graph = self.network_graph.read_only(); - let channels = - self.channel_manager.list_channels_with_counterparty(&counterparty_node_id); - if let Some(pending_channel) = - channels.into_iter().find(|c| c.channel_id == channel_id) - { - if !pending_channel.is_outbound - && self.peer_store.get_peer(&counterparty_node_id).is_none() - { - if let Some(address) = network_graph - .nodes() - .get(&NodeId::from_pubkey(&counterparty_node_id)) - .and_then(|node_info| node_info.announcement_info.as_ref()) - .and_then(|ann_info| ann_info.addresses().first()) - { - let peer = PeerInfo { - node_id: counterparty_node_id, - address: address.clone(), - }; - - self.peer_store.add_peer(peer).unwrap_or_else(|e| { - log_error!( - self.logger, - "Failed to add peer {} to peer store: {}", - counterparty_node_id, - e - ); - }); - } - } + let peer_to_store = { + let network_graph = self.network_graph.read_only(); + let channels = + self.channel_manager.list_channels_with_counterparty(&counterparty_node_id); + channels + .into_iter() + .find(|c| c.channel_id == channel_id) + .filter(|pending_channel| { + !pending_channel.is_outbound + && self.peer_store.get_peer(&counterparty_node_id).is_none() + }) + .and_then(|_| { + network_graph + .nodes() + .get(&NodeId::from_pubkey(&counterparty_node_id)) + .and_then(|node_info| node_info.announcement_info.as_ref()) + .and_then(|ann_info| ann_info.addresses().first()) + .map(|address| PeerInfo { + node_id: counterparty_node_id, + address: address.clone(), + }) + }) + }; + if let Some(peer) = peer_to_store { + self.peer_store.add_peer(peer).await.unwrap_or_else(|e| { + log_error!( + self.logger, + "Failed to add peer {} to peer store: {}", + counterparty_node_id, + e + ); + }); } }, LdkEvent::ChannelReady { diff --git a/src/io/in_memory_store.rs b/src/io/in_memory_store.rs new file mode 100644 index 0000000000..8b7d41c843 --- /dev/null +++ b/src/io/in_memory_store.rs @@ -0,0 +1,191 @@ +// This file is Copyright its original authors, visible in version control history. +// +// This file is licensed under the Apache License, Version 2.0 or the MIT license , at your option. You may not use this file except in +// accordance with one or both of these licenses. + +use std::collections::{hash_map, HashMap}; +use std::future::Future; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Mutex; + +use lightning::io; +use lightning::util::persist::{KVStore, PageToken, PaginatedKVStore, PaginatedListResponse}; + +const IN_MEMORY_PAGE_SIZE: usize = 50; + +pub struct InMemoryStore { + persisted_bytes: Mutex>>>, + creation_counter: AtomicU64, + creation_times: Mutex>>, +} + +impl InMemoryStore { + pub fn new() -> Self { + let persisted_bytes = Mutex::new(HashMap::new()); + let creation_counter = AtomicU64::new(1); + let creation_times = Mutex::new(HashMap::new()); + Self { persisted_bytes, creation_counter, creation_times } + } + + fn read_internal( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, + ) -> io::Result> { + let persisted_lock = self.persisted_bytes.lock().unwrap(); + let prefixed = format!("{primary_namespace}/{secondary_namespace}"); + + if let Some(outer_ref) = persisted_lock.get(&prefixed) { + if let Some(inner_ref) = outer_ref.get(key) { + let bytes = inner_ref.clone(); + Ok(bytes) + } else { + Err(io::Error::new(io::ErrorKind::NotFound, "Key not found")) + } + } else { + Err(io::Error::new(io::ErrorKind::NotFound, "Namespace not found")) + } + } + + fn write_internal( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, + ) -> io::Result<()> { + let mut persisted_lock = self.persisted_bytes.lock().unwrap(); + + let prefixed = format!("{primary_namespace}/{secondary_namespace}"); + let outer_e = persisted_lock.entry(prefixed.clone()).or_insert(HashMap::new()); + outer_e.insert(key.to_string(), buf); + + // Only assign creation time on first write (not on update) + let mut ct_lock = self.creation_times.lock().unwrap(); + let ct_ns = ct_lock.entry(prefixed).or_insert(HashMap::new()); + ct_ns + .entry(key.to_string()) + .or_insert_with(|| self.creation_counter.fetch_add(1, Ordering::Relaxed)); + + Ok(()) + } + + fn remove_internal( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, _lazy: bool, + ) -> io::Result<()> { + let mut persisted_lock = self.persisted_bytes.lock().unwrap(); + + let prefixed = format!("{primary_namespace}/{secondary_namespace}"); + if let Some(outer_ref) = persisted_lock.get_mut(&prefixed) { + outer_ref.remove(&key.to_string()); + } + + // Remove creation time entry + let mut ct_lock = self.creation_times.lock().unwrap(); + if let Some(ct_ns) = ct_lock.get_mut(&prefixed) { + ct_ns.remove(key); + } + + Ok(()) + } + + fn list_internal( + &self, primary_namespace: &str, secondary_namespace: &str, + ) -> io::Result> { + let mut persisted_lock = self.persisted_bytes.lock().unwrap(); + + let prefixed = format!("{primary_namespace}/{secondary_namespace}"); + match persisted_lock.entry(prefixed) { + hash_map::Entry::Occupied(e) => Ok(e.get().keys().cloned().collect()), + hash_map::Entry::Vacant(_) => Ok(Vec::new()), + } + } +} + +impl KVStore for InMemoryStore { + fn read( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, + ) -> impl Future, io::Error>> + 'static + Send { + let res = self.read_internal(&primary_namespace, &secondary_namespace, &key); + async move { res } + } + fn write( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, + ) -> impl Future> + 'static + Send { + let res = self.write_internal(&primary_namespace, &secondary_namespace, &key, buf); + async move { res } + } + fn remove( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, + ) -> impl Future> + 'static + Send { + let res = self.remove_internal(&primary_namespace, &secondary_namespace, &key, lazy); + async move { res } + } + fn list( + &self, primary_namespace: &str, secondary_namespace: &str, + ) -> impl Future, io::Error>> + 'static + Send { + let res = self.list_internal(primary_namespace, secondary_namespace); + async move { res } + } +} + +impl InMemoryStore { + fn list_paginated_internal( + &self, primary_namespace: &str, secondary_namespace: &str, page_token: Option, + ) -> io::Result { + let ct_lock = self.creation_times.lock().unwrap(); + let prefixed = format!("{primary_namespace}/{secondary_namespace}"); + + let ct_ns = match ct_lock.get(&prefixed) { + Some(m) => m, + None => { + return Ok(PaginatedListResponse { keys: Vec::new(), next_page_token: None }); + }, + }; + + // Build list of (key, sort_order) sorted by sort_order DESC (newest first). + let mut entries: Vec<(&String, &u64)> = ct_ns.iter().collect(); + entries.sort_by(|a, b| b.1.cmp(a.1)); + + // Apply page token filter + let start_idx = if let Some(ref token) = page_token { + let token_sort_order: u64 = token + .as_str() + .parse() + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "Invalid page token"))?; + + entries + .iter() + .position(|(_, sort_order)| **sort_order < token_sort_order) + .unwrap_or(entries.len()) + } else { + 0 + }; + + // Fetch one extra entry beyond page size to determine whether a next page exists. + let mut page: Vec<(&String, &u64)> = + entries[start_idx..].iter().take(IN_MEMORY_PAGE_SIZE + 1).cloned().collect(); + + let has_more = page.len() > IN_MEMORY_PAGE_SIZE; + page.truncate(IN_MEMORY_PAGE_SIZE); + + let next_page_token = if has_more { + let (_, last_sort_order) = page.last().unwrap(); + Some(PageToken::new(last_sort_order.to_string())) + } else { + None + }; + + let page: Vec = page.into_iter().map(|(k, _)| k.clone()).collect(); + + Ok(PaginatedListResponse { keys: page, next_page_token }) + } +} + +impl PaginatedKVStore for InMemoryStore { + fn list_paginated( + &self, primary_namespace: &str, secondary_namespace: &str, page_token: Option, + ) -> impl Future> + 'static + Send { + let res = self.list_paginated_internal(primary_namespace, secondary_namespace, page_token); + async move { res } + } +} + +unsafe impl Sync for InMemoryStore {} +unsafe impl Send for InMemoryStore {} diff --git a/src/io/postgres_store/mod.rs b/src/io/postgres_store/mod.rs index 7319d08985..f62a171b78 100644 --- a/src/io/postgres_store/mod.rs +++ b/src/io/postgres_store/mod.rs @@ -8,13 +8,11 @@ //! Objects related to [`PostgresStore`] live here. use std::collections::HashMap; use std::future::Future; -use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; use lightning::io; -use lightning::util::persist::{ - KVStore, KVStoreSync, PageToken, PaginatedKVStore, PaginatedKVStoreSync, PaginatedListResponse, -}; +use lightning::util::persist::{KVStore, PageToken, PaginatedKVStore, PaginatedListResponse}; use lightning_types::string::PrintableString; use native_tls::TlsConnector; use postgres_native_tls::MakeTlsConnector; @@ -24,6 +22,7 @@ use tokio_postgres::{Config, Error as PgError}; use self::pool::{make_config_connection, ClientConnection, PgTlsConnector, SmallPool}; use crate::io::utils::check_namespace_key_validity; use crate::logger::{log_debug, log_info, LdkLogger, Logger}; +use crate::runtime::StoreRuntime; mod migrations; mod pool; @@ -43,6 +42,18 @@ const PAGE_SIZE: usize = 50; // Keep this small while still allowing progress if one runtime worker blocks on sync store access. const INTERNAL_RUNTIME_WORKERS: usize = 2; +async fn run_on_internal_runtime( + runtime: Arc, future: impl Future> + Send + 'static, +) -> io::Result +where + T: Send + 'static, +{ + let task = runtime.spawn(future); + task.await.map_err(|e| { + io::Error::new(io::ErrorKind::Other, format!("PostgreSQL runtime task failed: {}", e)) + })? +} + fn sql_identifier(identifier: &str) -> io::Result { if identifier.is_empty() || identifier.contains('\0') { return Err(io::Error::new( @@ -91,10 +102,9 @@ macro_rules! query_with_retry { }}; } -/// A [`KVStoreSync`] implementation that writes to and reads from a [PostgreSQL] database. +/// A [`KVStore`] implementation that writes to and reads from a [PostgreSQL] database. /// -/// Maintains an internal runtime for the underlying tokio-postgres connection drivers and for -/// synchronous [`KVStoreSync`] and [`PaginatedKVStoreSync`] calls. +/// Maintains an internal runtime for the underlying tokio-postgres connection drivers. /// /// [PostgreSQL]: https://www.postgresql.org pub struct PostgresStore { @@ -104,14 +114,14 @@ pub struct PostgresStore { // operations aren't sensitive to the order of execution. next_write_version: AtomicU64, - // A store-internal runtime used for setup, connection driver tasks, and sync store access. - internal_runtime: Option, + // A store-internal runtime that drives PostgreSQL I/O independently from the node runtime. + internal_runtime: Option>, } // tokio::sync::Mutex (used for the DB client) contains UnsafeCell which opts out of // RefUnwindSafe. std::sync::Mutex (used by SqliteStore) doesn't have this issue because // it poisons on panic. This impl is needed for do_read_write_remove_list_persist which -// requires K: KVStoreSync + RefUnwindSafe. +// requires K: KVStore + RefUnwindSafe. #[cfg(test)] impl std::panic::RefUnwindSafe for PostgresStore {} @@ -148,30 +158,16 @@ impl PostgresStore { connection_string: String, db_name: Option, kv_table_name: Option, certificate_pem: Option, logger: Option>, ) -> io::Result { - let internal_runtime = tokio::runtime::Builder::new_multi_thread() - .enable_all() - .thread_name_fn(|| { - static ATOMIC_ID: AtomicUsize = AtomicUsize::new(0); - let id = ATOMIC_ID.fetch_add(1, Ordering::SeqCst); - format!("ldk-node-postgres-runtime-{}", id) - }) - .worker_threads(INTERNAL_RUNTIME_WORKERS) - .max_blocking_threads(INTERNAL_RUNTIME_WORKERS) - .build() - .map_err(|e| { - io::Error::new( - io::ErrorKind::Other, - format!("Failed to build PostgreSQL runtime: {e}"), - ) - })?; + let internal_runtime = Arc::new(StoreRuntime::new( + "ldk-node-postgres-runtime", + INTERNAL_RUNTIME_WORKERS, + "PostgreSQL", + )?); let tls = Self::build_tls_connector(certificate_pem)?; - let runtime_handle = internal_runtime.handle(); - let inner = tokio::task::block_in_place(|| { - runtime_handle.block_on(async { - PostgresStoreInner::new(connection_string, db_name, kv_table_name, tls, logger) - .await - }) - })?; + let inner = run_on_internal_runtime(Arc::clone(&internal_runtime), async move { + PostgresStoreInner::new(connection_string, db_name, kv_table_name, tls, logger).await + }) + .await?; let inner = Arc::new(inner); let next_write_version = AtomicU64::new(1); Ok(Self { inner, next_write_version, internal_runtime: Some(internal_runtime) }) @@ -217,12 +213,18 @@ impl PostgresStore { (inner_lock_ref, version) } + + fn internal_runtime(&self) -> Arc { + Arc::clone(self.internal_runtime.as_ref().expect("PostgreSQL runtime must be available")) + } } impl Drop for PostgresStore { fn drop(&mut self) { if let Some(internal_runtime) = self.internal_runtime.take() { - internal_runtime.shutdown_background(); + if let Ok(internal_runtime) = Arc::try_unwrap(internal_runtime) { + internal_runtime.shutdown_background(); + } } } } @@ -235,7 +237,13 @@ impl KVStore for PostgresStore { let secondary_namespace = secondary_namespace.to_string(); let key = key.to_string(); let inner = Arc::clone(&self.inner); - async move { inner.read_internal(&primary_namespace, &secondary_namespace, &key).await } + let runtime = self.internal_runtime(); + async move { + run_on_internal_runtime(runtime, async move { + inner.read_internal(&primary_namespace, &secondary_namespace, &key).await + }) + .await + } } fn write( @@ -247,18 +255,22 @@ impl KVStore for PostgresStore { let secondary_namespace = secondary_namespace.to_string(); let key = key.to_string(); let inner = Arc::clone(&self.inner); + let runtime = self.internal_runtime(); async move { - inner - .write_internal( - inner_lock_ref, - locking_key, - version, - &primary_namespace, - &secondary_namespace, - &key, - buf, - ) - .await + run_on_internal_runtime(runtime, async move { + inner + .write_internal( + inner_lock_ref, + locking_key, + version, + &primary_namespace, + &secondary_namespace, + &key, + buf, + ) + .await + }) + .await } } @@ -271,17 +283,21 @@ impl KVStore for PostgresStore { let secondary_namespace = secondary_namespace.to_string(); let key = key.to_string(); let inner = Arc::clone(&self.inner); + let runtime = self.internal_runtime(); async move { - inner - .remove_internal( - inner_lock_ref, - locking_key, - version, - &primary_namespace, - &secondary_namespace, - &key, - ) - .await + run_on_internal_runtime(runtime, async move { + inner + .remove_internal( + inner_lock_ref, + locking_key, + version, + &primary_namespace, + &secondary_namespace, + &key, + ) + .await + }) + .await } } @@ -291,58 +307,13 @@ impl KVStore for PostgresStore { let primary_namespace = primary_namespace.to_string(); let secondary_namespace = secondary_namespace.to_string(); let inner = Arc::clone(&self.inner); - async move { inner.list_internal(&primary_namespace, &secondary_namespace).await } - } -} - -impl PostgresStore { - fn internal_runtime(&self) -> io::Result<&tokio::runtime::Runtime> { - self.internal_runtime.as_ref().ok_or_else(|| { - debug_assert!(false, "Failed to access internal PostgreSQL runtime"); - io::Error::new(io::ErrorKind::Other, "Failed to access internal PostgreSQL runtime") - }) - } - - fn block_on(&self, fut: F) -> io::Result { - let internal_runtime = self.internal_runtime()?; - Ok(tokio::task::block_in_place(move || internal_runtime.block_on(fut))) - } -} - -impl KVStoreSync for PostgresStore { - fn read( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, - ) -> io::Result> { - self.block_on(KVStore::read(self, primary_namespace, secondary_namespace, key))? - } - - fn write( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, - ) -> io::Result<()> { - self.block_on(KVStore::write(self, primary_namespace, secondary_namespace, key, buf))? - } - - fn remove( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, - ) -> io::Result<()> { - self.block_on(KVStore::remove(self, primary_namespace, secondary_namespace, key, lazy))? - } - - fn list(&self, primary_namespace: &str, secondary_namespace: &str) -> io::Result> { - self.block_on(KVStore::list(self, primary_namespace, secondary_namespace))? - } -} - -impl PaginatedKVStoreSync for PostgresStore { - fn list_paginated( - &self, primary_namespace: &str, secondary_namespace: &str, page_token: Option, - ) -> io::Result { - self.block_on(PaginatedKVStore::list_paginated( - self, - primary_namespace, - secondary_namespace, - page_token, - ))? + let runtime = self.internal_runtime(); + async move { + run_on_internal_runtime(runtime, async move { + inner.list_internal(&primary_namespace, &secondary_namespace).await + }) + .await + } } } @@ -353,10 +324,14 @@ impl PaginatedKVStore for PostgresStore { let primary_namespace = primary_namespace.to_string(); let secondary_namespace = secondary_namespace.to_string(); let inner = Arc::clone(&self.inner); + let runtime = self.internal_runtime(); async move { - inner - .list_paginated_internal(&primary_namespace, &secondary_namespace, page_token) - .await + run_on_internal_runtime(runtime, async move { + inner + .list_paginated_internal(&primary_namespace, &secondary_namespace, page_token) + .await + }) + .await } } } @@ -901,7 +876,7 @@ mod tests { #[tokio::test(flavor = "multi_thread")] async fn read_write_remove_list_persist() { let store = create_test_store("test_rwrl").await; - do_read_write_remove_list_persist(&store); + do_read_write_remove_list_persist(&store).await; cleanup_store(&store).await; } @@ -931,17 +906,17 @@ mod tests { let sub = "test_sub"; // Write a value before disconnecting. - KVStoreSync::write(&store, ns, sub, "key_a", vec![1u8; 8]).unwrap(); + KVStore::write(&store, ns, sub, "key_a", vec![1u8; 8]).await.unwrap(); // Read should auto-reconnect and return the previously written value. kill_connection(&store).await; - let data = KVStoreSync::read(&store, ns, sub, "key_a").unwrap(); + let data = KVStore::read(&store, ns, sub, "key_a").await.unwrap(); assert_eq!(data, vec![1u8; 8]); // Write should auto-reconnect without a preceding read. kill_connection(&store).await; - KVStoreSync::write(&store, ns, sub, "key_b", vec![2u8; 8]).unwrap(); - let data = KVStoreSync::read(&store, ns, sub, "key_b").unwrap(); + KVStore::write(&store, ns, sub, "key_b", vec![2u8; 8]).await.unwrap(); + let data = KVStore::read(&store, ns, sub, "key_b").await.unwrap(); assert_eq!(data, vec![2u8; 8]); cleanup_store(&store).await; @@ -958,7 +933,9 @@ mod tests { for i in 0..num_entries { let key = format!("key_{:04}", i); let data = vec![i as u8; 32]; - KVStoreSync::write(&store, primary_namespace, secondary_namespace, &key, data).unwrap(); + KVStore::write(&store, primary_namespace, secondary_namespace, &key, data) + .await + .unwrap(); } // Paginate through all entries and collect them @@ -967,12 +944,13 @@ mod tests { let mut page_count = 0; loop { - let response = PaginatedKVStoreSync::list_paginated( + let response = PaginatedKVStore::list_paginated( &store, primary_namespace, secondary_namespace, page_token, ) + .await .unwrap(); all_keys.extend(response.keys.clone()); @@ -1010,32 +988,33 @@ mod tests { let primary_namespace = "test_ns"; let secondary_namespace = "test_sub"; - KVStoreSync::write(&store, primary_namespace, secondary_namespace, "first", vec![1u8; 8]) + KVStore::write(&store, primary_namespace, secondary_namespace, "first", vec![1u8; 8]) + .await .unwrap(); - KVStoreSync::write(&store, primary_namespace, secondary_namespace, "second", vec![2u8; 8]) + KVStore::write(&store, primary_namespace, secondary_namespace, "second", vec![2u8; 8]) + .await .unwrap(); - KVStoreSync::write(&store, primary_namespace, secondary_namespace, "third", vec![3u8; 8]) + KVStore::write(&store, primary_namespace, secondary_namespace, "third", vec![3u8; 8]) + .await .unwrap(); // Update the first entry - KVStoreSync::write(&store, primary_namespace, secondary_namespace, "first", vec![99u8; 8]) + KVStore::write(&store, primary_namespace, secondary_namespace, "first", vec![99u8; 8]) + .await .unwrap(); // Paginated listing should still show "first" with its original creation order - let response = PaginatedKVStoreSync::list_paginated( - &store, - primary_namespace, - secondary_namespace, - None, - ) - .unwrap(); + let response = + PaginatedKVStore::list_paginated(&store, primary_namespace, secondary_namespace, None) + .await + .unwrap(); // Newest first: third, second, first assert_eq!(response.keys, vec!["third", "second", "first"]); // Verify the updated value was persisted let data = - KVStoreSync::read(&store, primary_namespace, secondary_namespace, "first").unwrap(); + KVStore::read(&store, primary_namespace, secondary_namespace, "first").await.unwrap(); assert_eq!(data, vec![99u8; 8]); cleanup_store(&store).await; @@ -1047,7 +1026,7 @@ mod tests { // Paginating an empty or unknown namespace returns an empty result with no token. let response = - PaginatedKVStoreSync::list_paginated(&store, "nonexistent", "ns", None).unwrap(); + PaginatedKVStore::list_paginated(&store, "nonexistent", "ns", None).await.unwrap(); assert!(response.keys.is_empty()); assert!(response.next_page_token.is_none()); @@ -1058,22 +1037,23 @@ mod tests { async fn test_postgres_store_paginated_namespace_isolation() { let store = create_test_store("test_pg_paginated_isolation").await; - KVStoreSync::write(&store, "ns_a", "sub", "key_1", vec![1u8; 8]).unwrap(); - KVStoreSync::write(&store, "ns_a", "sub", "key_2", vec![2u8; 8]).unwrap(); - KVStoreSync::write(&store, "ns_b", "sub", "key_3", vec![3u8; 8]).unwrap(); - KVStoreSync::write(&store, "ns_a", "other", "key_4", vec![4u8; 8]).unwrap(); + KVStore::write(&store, "ns_a", "sub", "key_1", vec![1u8; 8]).await.unwrap(); + KVStore::write(&store, "ns_a", "sub", "key_2", vec![2u8; 8]).await.unwrap(); + KVStore::write(&store, "ns_b", "sub", "key_3", vec![3u8; 8]).await.unwrap(); + KVStore::write(&store, "ns_a", "other", "key_4", vec![4u8; 8]).await.unwrap(); // ns_a/sub should only contain key_1 and key_2 (newest first). - let response = PaginatedKVStoreSync::list_paginated(&store, "ns_a", "sub", None).unwrap(); + let response = PaginatedKVStore::list_paginated(&store, "ns_a", "sub", None).await.unwrap(); assert_eq!(response.keys, vec!["key_2", "key_1"]); assert!(response.next_page_token.is_none()); // ns_b/sub should only contain key_3. - let response = PaginatedKVStoreSync::list_paginated(&store, "ns_b", "sub", None).unwrap(); + let response = PaginatedKVStore::list_paginated(&store, "ns_b", "sub", None).await.unwrap(); assert_eq!(response.keys, vec!["key_3"]); // ns_a/other should only contain key_4. - let response = PaginatedKVStoreSync::list_paginated(&store, "ns_a", "other", None).unwrap(); + let response = + PaginatedKVStore::list_paginated(&store, "ns_a", "other", None).await.unwrap(); assert_eq!(response.keys, vec!["key_4"]); cleanup_store(&store).await; @@ -1086,13 +1066,13 @@ mod tests { let ns = "test_ns"; let sub = "test_sub"; - KVStoreSync::write(&store, ns, sub, "a", vec![1u8; 8]).unwrap(); - KVStoreSync::write(&store, ns, sub, "b", vec![2u8; 8]).unwrap(); - KVStoreSync::write(&store, ns, sub, "c", vec![3u8; 8]).unwrap(); + KVStore::write(&store, ns, sub, "a", vec![1u8; 8]).await.unwrap(); + KVStore::write(&store, ns, sub, "b", vec![2u8; 8]).await.unwrap(); + KVStore::write(&store, ns, sub, "c", vec![3u8; 8]).await.unwrap(); - KVStoreSync::remove(&store, ns, sub, "b", false).unwrap(); + KVStore::remove(&store, ns, sub, "b", false).await.unwrap(); - let response = PaginatedKVStoreSync::list_paginated(&store, ns, sub, None).unwrap(); + let response = PaginatedKVStore::list_paginated(&store, ns, sub, None).await.unwrap(); assert_eq!(response.keys, vec!["c", "a"]); assert!(response.next_page_token.is_none()); @@ -1109,24 +1089,24 @@ mod tests { // Write exactly PAGE_SIZE entries (50). for i in 0..PAGE_SIZE { let key = format!("key_{:04}", i); - KVStoreSync::write(&store, ns, sub, &key, vec![i as u8; 8]).unwrap(); + KVStore::write(&store, ns, sub, &key, vec![i as u8; 8]).await.unwrap(); } // Exactly PAGE_SIZE entries: all returned in one page with no next-page token. - let response = PaginatedKVStoreSync::list_paginated(&store, ns, sub, None).unwrap(); + let response = PaginatedKVStore::list_paginated(&store, ns, sub, None).await.unwrap(); assert_eq!(response.keys.len(), PAGE_SIZE); assert!(response.next_page_token.is_none()); // Add one more entry (PAGE_SIZE + 1 total). First page should now have a token. - KVStoreSync::write(&store, ns, sub, "key_extra", vec![0u8; 8]).unwrap(); - let response = PaginatedKVStoreSync::list_paginated(&store, ns, sub, None).unwrap(); + KVStore::write(&store, ns, sub, "key_extra", vec![0u8; 8]).await.unwrap(); + let response = PaginatedKVStore::list_paginated(&store, ns, sub, None).await.unwrap(); assert_eq!(response.keys.len(), PAGE_SIZE); assert!(response.next_page_token.is_some()); // Second page should have exactly 1 entry and no token. - let response = - PaginatedKVStoreSync::list_paginated(&store, ns, sub, response.next_page_token) - .unwrap(); + let response = PaginatedKVStore::list_paginated(&store, ns, sub, response.next_page_token) + .await + .unwrap(); assert_eq!(response.keys.len(), 1); assert!(response.next_page_token.is_none()); @@ -1143,10 +1123,10 @@ mod tests { // Write fewer entries than PAGE_SIZE. for i in 0..5 { let key = format!("key_{i}"); - KVStoreSync::write(&store, ns, sub, &key, vec![i as u8; 8]).unwrap(); + KVStore::write(&store, ns, sub, &key, vec![i as u8; 8]).await.unwrap(); } - let response = PaginatedKVStoreSync::list_paginated(&store, ns, sub, None).unwrap(); + let response = PaginatedKVStore::list_paginated(&store, ns, sub, None).await.unwrap(); assert_eq!(response.keys.len(), 5); // Fewer than PAGE_SIZE means no next page. assert!(response.next_page_token.is_none()); @@ -1165,22 +1145,12 @@ mod tests { { let store = create_test_store(table_name).await; - KVStoreSync::write( - &store, - primary_namespace, - secondary_namespace, - "key_a", - vec![1u8; 8], - ) - .unwrap(); - KVStoreSync::write( - &store, - primary_namespace, - secondary_namespace, - "key_b", - vec![2u8; 8], - ) - .unwrap(); + KVStore::write(&store, primary_namespace, secondary_namespace, "key_a", vec![1u8; 8]) + .await + .unwrap(); + KVStore::write(&store, primary_namespace, secondary_namespace, "key_b", vec![2u8; 8]) + .await + .unwrap(); // Don't clean up since we want to reopen } @@ -1189,22 +1159,18 @@ mod tests { { let store = create_test_store(table_name).await; - KVStoreSync::write( - &store, - primary_namespace, - secondary_namespace, - "key_c", - vec![3u8; 8], - ) - .unwrap(); + KVStore::write(&store, primary_namespace, secondary_namespace, "key_c", vec![3u8; 8]) + .await + .unwrap(); // Paginated listing should show newest first: key_c, key_b, key_a - let response = PaginatedKVStoreSync::list_paginated( + let response = PaginatedKVStore::list_paginated( &store, primary_namespace, secondary_namespace, None, ) + .await .unwrap(); assert_eq!(response.keys, vec!["key_c", "key_b", "key_a"]); diff --git a/src/io/sqlite_store/migrations.rs b/src/io/sqlite_store/migrations.rs index f596b1a42f..1b4de9aa07 100644 --- a/src/io/sqlite_store/migrations.rs +++ b/src/io/sqlite_store/migrations.rs @@ -169,14 +169,14 @@ fn migrate_v2_to_v3(connection: &mut Connection, kv_table_name: &str) -> io::Res mod tests { use std::fs; - use lightning::util::persist::{KVStoreSync, PaginatedKVStoreSync}; + use lightning::util::persist::{KVStore, PaginatedKVStore}; use rusqlite::{named_params, Connection}; use crate::io::sqlite_store::SqliteStore; use crate::io::test_utils::{do_read_write_remove_list_persist, random_storage_path}; - #[test] - fn rwrl_post_schema_1_migration() { + #[tokio::test] + async fn rwrl_post_schema_1_migration() { let old_schema_version = 1; let mut temp_path = random_storage_path(); @@ -253,15 +253,15 @@ mod tests { // Check we migrate the db just fine without losing our written data. let store = SqliteStore::new(temp_path, Some(db_file_name), Some(kv_table_name)).unwrap(); - let res = store.read(&test_namespace, "", &test_key).unwrap(); + let res = KVStore::read(&store, &test_namespace, "", &test_key).await.unwrap(); assert_eq!(res, test_data); // Check we can continue to use the store just fine. - do_read_write_remove_list_persist(&store); + do_read_write_remove_list_persist(&store).await; } - #[test] - fn rwrl_post_schema_2_migration() { + #[tokio::test] + async fn rwrl_post_schema_2_migration() { let old_schema_version = 2u16; let mut temp_path = random_storage_path(); @@ -325,24 +325,24 @@ mod tests { // Verify data survived for i in 0..3 { let key = format!("key_{}", i); - let data = store.read(test_ns, test_sub, &key).unwrap(); + let data = KVStore::read(&store, test_ns, test_sub, &key).await.unwrap(); assert_eq!(data, vec![i as u8; 8]); } // Verify paginated listing works and returns entries in ROWID-backfilled order (newest first) let response = - PaginatedKVStoreSync::list_paginated(&store, test_ns, test_sub, None).unwrap(); + PaginatedKVStore::list_paginated(&store, test_ns, test_sub, None).await.unwrap(); assert_eq!(response.keys.len(), 3); // ROWIDs were 1, 2, 3 so sort_order was backfilled as 1, 2, 3; newest first assert_eq!(response.keys, vec!["key_2", "key_1", "key_0"]); // Verify we can write new entries and they get proper ordering - KVStoreSync::write(&store, test_ns, test_sub, "key_new", vec![99u8; 8]).unwrap(); + KVStore::write(&store, test_ns, test_sub, "key_new", vec![99u8; 8]).await.unwrap(); let response = - PaginatedKVStoreSync::list_paginated(&store, test_ns, test_sub, None).unwrap(); + PaginatedKVStore::list_paginated(&store, test_ns, test_sub, None).await.unwrap(); assert_eq!(response.keys[0], "key_new"); // Check we can continue to use the store just fine. - do_read_write_remove_list_persist(&store); + do_read_write_remove_list_persist(&store).await; } } diff --git a/src/io/sqlite_store/mod.rs b/src/io/sqlite_store/mod.rs index 84af03adc5..076aeef9bd 100644 --- a/src/io/sqlite_store/mod.rs +++ b/src/io/sqlite_store/mod.rs @@ -14,9 +14,7 @@ use std::sync::atomic::{AtomicI64, AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; use lightning::io; -use lightning::util::persist::{ - KVStore, KVStoreSync, PageToken, PaginatedKVStore, PaginatedKVStoreSync, PaginatedListResponse, -}; +use lightning::util::persist::{KVStore, PageToken, PaginatedKVStore, PaginatedListResponse}; use lightning_types::string::PrintableString; use rusqlite::{named_params, Connection}; @@ -41,7 +39,7 @@ const SCHEMA_USER_VERSION: u16 = 3; // The number of entries returned per page in paginated list operations. const PAGE_SIZE: usize = 50; -/// A [`KVStoreSync`] implementation that writes to and reads from an [SQLite] database. +/// A [`KVStore`] implementation that writes to and reads from an [SQLite] database. /// /// [SQLite]: https://sqlite.org pub struct SqliteStore { @@ -185,57 +183,6 @@ impl KVStore for SqliteStore { } } -impl KVStoreSync for SqliteStore { - fn read( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, - ) -> io::Result> { - self.inner.read_internal(primary_namespace, secondary_namespace, key) - } - - fn write( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, - ) -> io::Result<()> { - let locking_key = self.build_locking_key(primary_namespace, secondary_namespace, key); - let (inner_lock_ref, version) = self.get_new_version_and_lock_ref(locking_key.clone()); - self.inner.write_internal( - inner_lock_ref, - locking_key, - version, - primary_namespace, - secondary_namespace, - key, - buf, - ) - } - - fn remove( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, _lazy: bool, - ) -> io::Result<()> { - let locking_key = self.build_locking_key(primary_namespace, secondary_namespace, key); - let (inner_lock_ref, version) = self.get_new_version_and_lock_ref(locking_key.clone()); - self.inner.remove_internal( - inner_lock_ref, - locking_key, - version, - primary_namespace, - secondary_namespace, - key, - ) - } - - fn list(&self, primary_namespace: &str, secondary_namespace: &str) -> io::Result> { - self.inner.list_internal(primary_namespace, secondary_namespace) - } -} - -impl PaginatedKVStoreSync for SqliteStore { - fn list_paginated( - &self, primary_namespace: &str, secondary_namespace: &str, page_token: Option, - ) -> io::Result { - self.inner.list_paginated_internal(primary_namespace, secondary_namespace, page_token) - } -} - impl PaginatedKVStore for SqliteStore { fn list_paginated( &self, primary_namespace: &str, secondary_namespace: &str, page_token: Option, @@ -700,8 +647,8 @@ mod tests { } } - #[test] - fn read_write_remove_list_persist() { + #[tokio::test] + async fn read_write_remove_list_persist() { let mut temp_path = random_storage_path(); temp_path.push("read_write_remove_list_persist"); let store = SqliteStore::new( @@ -710,11 +657,11 @@ mod tests { Some("test_table".to_string()), ) .unwrap(); - do_read_write_remove_list_persist(&store); + do_read_write_remove_list_persist(&store).await; } - #[test] - fn test_sqlite_store() { + #[tokio::test(flavor = "multi_thread")] + async fn test_sqlite_store() { let mut temp_path = random_storage_path(); temp_path.push("test_sqlite_store"); let store_0 = SqliteStore::new( @@ -732,8 +679,8 @@ mod tests { do_test_store(&store_0, &store_1) } - #[test] - fn test_sqlite_store_paginated_listing() { + #[tokio::test] + async fn test_sqlite_store_paginated_listing() { let mut temp_path = random_storage_path(); temp_path.push("test_sqlite_store_paginated_listing"); let store = SqliteStore::new( @@ -750,7 +697,9 @@ mod tests { for i in 0..num_entries { let key = format!("key_{:04}", i); let data = vec![i as u8; 32]; - KVStoreSync::write(&store, primary_namespace, secondary_namespace, &key, data).unwrap(); + KVStore::write(&store, primary_namespace, secondary_namespace, &key, data) + .await + .unwrap(); } // Paginate through all entries and collect them @@ -759,12 +708,13 @@ mod tests { let mut page_count = 0; loop { - let response = PaginatedKVStoreSync::list_paginated( + let response = PaginatedKVStore::list_paginated( &store, primary_namespace, secondary_namespace, page_token, ) + .await .unwrap(); all_keys.extend(response.keys.clone()); @@ -795,8 +745,8 @@ mod tests { assert_eq!(all_keys[num_entries - 1], "key_0000"); } - #[test] - fn test_sqlite_store_paginated_update_preserves_order() { + #[tokio::test] + async fn test_sqlite_store_paginated_update_preserves_order() { let mut temp_path = random_storage_path(); temp_path.push("test_sqlite_store_paginated_update"); let store = SqliteStore::new( @@ -809,37 +759,38 @@ mod tests { let primary_namespace = "test_ns"; let secondary_namespace = "test_sub"; - KVStoreSync::write(&store, primary_namespace, secondary_namespace, "first", vec![1u8; 8]) + KVStore::write(&store, primary_namespace, secondary_namespace, "first", vec![1u8; 8]) + .await .unwrap(); - KVStoreSync::write(&store, primary_namespace, secondary_namespace, "second", vec![2u8; 8]) + KVStore::write(&store, primary_namespace, secondary_namespace, "second", vec![2u8; 8]) + .await .unwrap(); - KVStoreSync::write(&store, primary_namespace, secondary_namespace, "third", vec![3u8; 8]) + KVStore::write(&store, primary_namespace, secondary_namespace, "third", vec![3u8; 8]) + .await .unwrap(); // Update the first entry - KVStoreSync::write(&store, primary_namespace, secondary_namespace, "first", vec![99u8; 8]) + KVStore::write(&store, primary_namespace, secondary_namespace, "first", vec![99u8; 8]) + .await .unwrap(); // Paginated listing should still show "first" with its original creation order - let response = PaginatedKVStoreSync::list_paginated( - &store, - primary_namespace, - secondary_namespace, - None, - ) - .unwrap(); + let response = + PaginatedKVStore::list_paginated(&store, primary_namespace, secondary_namespace, None) + .await + .unwrap(); // Newest first: third, second, first assert_eq!(response.keys, vec!["third", "second", "first"]); // Verify the updated value was persisted let data = - KVStoreSync::read(&store, primary_namespace, secondary_namespace, "first").unwrap(); + KVStore::read(&store, primary_namespace, secondary_namespace, "first").await.unwrap(); assert_eq!(data, vec![99u8; 8]); } - #[test] - fn test_sqlite_store_paginated_empty_namespace() { + #[tokio::test] + async fn test_sqlite_store_paginated_empty_namespace() { let mut temp_path = random_storage_path(); temp_path.push("test_sqlite_store_paginated_empty"); let store = SqliteStore::new( @@ -851,13 +802,13 @@ mod tests { // Paginating an empty or unknown namespace returns an empty result with no token. let response = - PaginatedKVStoreSync::list_paginated(&store, "nonexistent", "ns", None).unwrap(); + PaginatedKVStore::list_paginated(&store, "nonexistent", "ns", None).await.unwrap(); assert!(response.keys.is_empty()); assert!(response.next_page_token.is_none()); } - #[test] - fn test_sqlite_store_paginated_namespace_isolation() { + #[tokio::test] + async fn test_sqlite_store_paginated_namespace_isolation() { let mut temp_path = random_storage_path(); temp_path.push("test_sqlite_store_paginated_isolation"); let store = SqliteStore::new( @@ -867,27 +818,28 @@ mod tests { ) .unwrap(); - KVStoreSync::write(&store, "ns_a", "sub", "key_1", vec![1u8; 8]).unwrap(); - KVStoreSync::write(&store, "ns_a", "sub", "key_2", vec![2u8; 8]).unwrap(); - KVStoreSync::write(&store, "ns_b", "sub", "key_3", vec![3u8; 8]).unwrap(); - KVStoreSync::write(&store, "ns_a", "other", "key_4", vec![4u8; 8]).unwrap(); + KVStore::write(&store, "ns_a", "sub", "key_1", vec![1u8; 8]).await.unwrap(); + KVStore::write(&store, "ns_a", "sub", "key_2", vec![2u8; 8]).await.unwrap(); + KVStore::write(&store, "ns_b", "sub", "key_3", vec![3u8; 8]).await.unwrap(); + KVStore::write(&store, "ns_a", "other", "key_4", vec![4u8; 8]).await.unwrap(); // ns_a/sub should only contain key_1 and key_2 (newest first). - let response = PaginatedKVStoreSync::list_paginated(&store, "ns_a", "sub", None).unwrap(); + let response = PaginatedKVStore::list_paginated(&store, "ns_a", "sub", None).await.unwrap(); assert_eq!(response.keys, vec!["key_2", "key_1"]); assert!(response.next_page_token.is_none()); // ns_b/sub should only contain key_3. - let response = PaginatedKVStoreSync::list_paginated(&store, "ns_b", "sub", None).unwrap(); + let response = PaginatedKVStore::list_paginated(&store, "ns_b", "sub", None).await.unwrap(); assert_eq!(response.keys, vec!["key_3"]); // ns_a/other should only contain key_4. - let response = PaginatedKVStoreSync::list_paginated(&store, "ns_a", "other", None).unwrap(); + let response = + PaginatedKVStore::list_paginated(&store, "ns_a", "other", None).await.unwrap(); assert_eq!(response.keys, vec!["key_4"]); } - #[test] - fn test_sqlite_store_paginated_removal() { + #[tokio::test] + async fn test_sqlite_store_paginated_removal() { let mut temp_path = random_storage_path(); temp_path.push("test_sqlite_store_paginated_removal"); let store = SqliteStore::new( @@ -900,19 +852,19 @@ mod tests { let ns = "test_ns"; let sub = "test_sub"; - KVStoreSync::write(&store, ns, sub, "a", vec![1u8; 8]).unwrap(); - KVStoreSync::write(&store, ns, sub, "b", vec![2u8; 8]).unwrap(); - KVStoreSync::write(&store, ns, sub, "c", vec![3u8; 8]).unwrap(); + KVStore::write(&store, ns, sub, "a", vec![1u8; 8]).await.unwrap(); + KVStore::write(&store, ns, sub, "b", vec![2u8; 8]).await.unwrap(); + KVStore::write(&store, ns, sub, "c", vec![3u8; 8]).await.unwrap(); - KVStoreSync::remove(&store, ns, sub, "b", false).unwrap(); + KVStore::remove(&store, ns, sub, "b", false).await.unwrap(); - let response = PaginatedKVStoreSync::list_paginated(&store, ns, sub, None).unwrap(); + let response = PaginatedKVStore::list_paginated(&store, ns, sub, None).await.unwrap(); assert_eq!(response.keys, vec!["c", "a"]); assert!(response.next_page_token.is_none()); } - #[test] - fn test_sqlite_store_paginated_exact_page_boundary() { + #[tokio::test] + async fn test_sqlite_store_paginated_exact_page_boundary() { let mut temp_path = random_storage_path(); temp_path.push("test_sqlite_store_paginated_boundary"); let store = SqliteStore::new( @@ -928,30 +880,30 @@ mod tests { // Write exactly PAGE_SIZE entries (50). for i in 0..PAGE_SIZE { let key = format!("key_{:04}", i); - KVStoreSync::write(&store, ns, sub, &key, vec![i as u8; 8]).unwrap(); + KVStore::write(&store, ns, sub, &key, vec![i as u8; 8]).await.unwrap(); } // Exactly PAGE_SIZE entries: all returned in one page with no next-page token. - let response = PaginatedKVStoreSync::list_paginated(&store, ns, sub, None).unwrap(); + let response = PaginatedKVStore::list_paginated(&store, ns, sub, None).await.unwrap(); assert_eq!(response.keys.len(), PAGE_SIZE); assert!(response.next_page_token.is_none()); // Add one more entry (PAGE_SIZE + 1 total). First page should now have a token. - KVStoreSync::write(&store, ns, sub, "key_extra", vec![0u8; 8]).unwrap(); - let response = PaginatedKVStoreSync::list_paginated(&store, ns, sub, None).unwrap(); + KVStore::write(&store, ns, sub, "key_extra", vec![0u8; 8]).await.unwrap(); + let response = PaginatedKVStore::list_paginated(&store, ns, sub, None).await.unwrap(); assert_eq!(response.keys.len(), PAGE_SIZE); assert!(response.next_page_token.is_some()); // Second page should have exactly 1 entry and no token. - let response = - PaginatedKVStoreSync::list_paginated(&store, ns, sub, response.next_page_token) - .unwrap(); + let response = PaginatedKVStore::list_paginated(&store, ns, sub, response.next_page_token) + .await + .unwrap(); assert_eq!(response.keys.len(), 1); assert!(response.next_page_token.is_none()); } - #[test] - fn test_sqlite_store_paginated_fewer_than_page_size() { + #[tokio::test] + async fn test_sqlite_store_paginated_fewer_than_page_size() { let mut temp_path = random_storage_path(); temp_path.push("test_sqlite_store_paginated_few"); let store = SqliteStore::new( @@ -967,10 +919,10 @@ mod tests { // Write fewer entries than PAGE_SIZE. for i in 0..5 { let key = format!("key_{}", i); - KVStoreSync::write(&store, ns, sub, &key, vec![i as u8; 8]).unwrap(); + KVStore::write(&store, ns, sub, &key, vec![i as u8; 8]).await.unwrap(); } - let response = PaginatedKVStoreSync::list_paginated(&store, ns, sub, None).unwrap(); + let response = PaginatedKVStore::list_paginated(&store, ns, sub, None).await.unwrap(); assert_eq!(response.keys.len(), 5); // Fewer than PAGE_SIZE means no next page. assert!(response.next_page_token.is_none()); @@ -978,8 +930,8 @@ mod tests { assert_eq!(response.keys, vec!["key_4", "key_3", "key_2", "key_1", "key_0"]); } - #[test] - fn test_sqlite_store_write_version_persists_across_restart() { + #[tokio::test] + async fn test_sqlite_store_write_version_persists_across_restart() { let mut temp_path = random_storage_path(); temp_path.push("test_sqlite_store_write_version_restart"); @@ -994,22 +946,12 @@ mod tests { ) .unwrap(); - KVStoreSync::write( - &store, - primary_namespace, - secondary_namespace, - "key_a", - vec![1u8; 8], - ) - .unwrap(); - KVStoreSync::write( - &store, - primary_namespace, - secondary_namespace, - "key_b", - vec![2u8; 8], - ) - .unwrap(); + KVStore::write(&store, primary_namespace, secondary_namespace, "key_a", vec![1u8; 8]) + .await + .unwrap(); + KVStore::write(&store, primary_namespace, secondary_namespace, "key_b", vec![2u8; 8]) + .await + .unwrap(); // Don't drop/cleanup since we want to reopen std::mem::forget(store); @@ -1024,22 +966,18 @@ mod tests { ) .unwrap(); - KVStoreSync::write( - &store, - primary_namespace, - secondary_namespace, - "key_c", - vec![3u8; 8], - ) - .unwrap(); + KVStore::write(&store, primary_namespace, secondary_namespace, "key_c", vec![3u8; 8]) + .await + .unwrap(); // Paginated listing should show newest first: key_c, key_b, key_a - let response = PaginatedKVStoreSync::list_paginated( + let response = PaginatedKVStore::list_paginated( &store, primary_namespace, secondary_namespace, None, ) + .await .unwrap(); assert_eq!(response.keys, vec!["key_c", "key_b", "key_a"]); diff --git a/src/io/test_utils.rs b/src/io/test_utils.rs index eed8c3e2da..aadb4b79a8 100644 --- a/src/io/test_utils.rs +++ b/src/io/test_utils.rs @@ -5,13 +5,13 @@ // http://opensource.org/licenses/MIT>, at your option. You may not use this file except in // accordance with one or both of these licenses. -use std::collections::{hash_map, HashMap}; use std::future::Future; use std::panic::RefUnwindSafe; use std::path::PathBuf; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::Mutex; +use std::sync::Arc; +use lightning::chain::channelmonitor::{ChannelMonitor, ChannelMonitorUpdate}; +use lightning::chain::{chainmonitor, BlockLocator, ChannelMonitorUpdateStatus}; use lightning::events::ClosureReason; use lightning::io; use lightning::ln::functional_test_utils::{ @@ -21,232 +21,145 @@ use lightning::ln::functional_test_utils::{ TestChanMonCfg, }; use lightning::util::persist::{ - KVStore, KVStoreSync, MonitorUpdatingPersister, PageToken, PaginatedKVStore, - PaginatedKVStoreSync, PaginatedListResponse, KVSTORE_NAMESPACE_KEY_MAX_LEN, + KVStore, MonitorName, ARCHIVED_CHANNEL_MONITOR_PERSISTENCE_PRIMARY_NAMESPACE, + ARCHIVED_CHANNEL_MONITOR_PERSISTENCE_SECONDARY_NAMESPACE, + CHANNEL_MONITOR_PERSISTENCE_PRIMARY_NAMESPACE, CHANNEL_MONITOR_PERSISTENCE_SECONDARY_NAMESPACE, + KVSTORE_NAMESPACE_KEY_MAX_LEN, }; +use lightning::util::ser::{ReadableArgs, Writeable}; +use lightning::util::test_channel_signer::TestChannelSigner; use lightning::util::test_utils; use rand::distr::Alphanumeric; use rand::{rng, Rng}; -type TestMonitorUpdatePersister<'a, K> = MonitorUpdatingPersister< - &'a K, - &'a test_utils::TestLogger, - &'a test_utils::TestKeysInterface, - &'a test_utils::TestKeysInterface, - &'a test_utils::TestBroadcaster, - &'a test_utils::TestFeeEstimator, ->; +#[path = "in_memory_store.rs"] +mod in_memory_store; -const EXPECTED_UPDATES_PER_PAYMENT: u64 = 5; - -const IN_MEMORY_PAGE_SIZE: usize = 50; +use crate::logger::Logger; +use crate::runtime::Runtime; -pub struct InMemoryStore { - persisted_bytes: Mutex>>>, - creation_counter: AtomicU64, - creation_times: Mutex>>, +pub(crate) struct TestMonitorUpdatePersister<'a, K> { + store: &'a K, + runtime: Runtime, + entropy_source: &'a test_utils::TestKeysInterface, + signer_provider: &'a test_utils::TestKeysInterface, } -impl InMemoryStore { - pub fn new() -> Self { - let persisted_bytes = Mutex::new(HashMap::new()); - let creation_counter = AtomicU64::new(1); - let creation_times = Mutex::new(HashMap::new()); - Self { persisted_bytes, creation_counter, creation_times } - } - - fn read_internal( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, - ) -> io::Result> { - let persisted_lock = self.persisted_bytes.lock().unwrap(); - let prefixed = format!("{primary_namespace}/{secondary_namespace}"); - - if let Some(outer_ref) = persisted_lock.get(&prefixed) { - if let Some(inner_ref) = outer_ref.get(key) { - let bytes = inner_ref.clone(); - Ok(bytes) - } else { - Err(io::Error::new(io::ErrorKind::NotFound, "Key not found")) +impl TestMonitorUpdatePersister<'_, K> { + pub(crate) fn read_all_channel_monitors_with_updates( + &self, + ) -> Result)>, io::Error> { + self.runtime.block_on(async { + let stored_keys = KVStore::list( + self.store, + CHANNEL_MONITOR_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MONITOR_PERSISTENCE_SECONDARY_NAMESPACE, + ) + .await?; + + let mut res = Vec::with_capacity(stored_keys.len()); + for stored_key in stored_keys { + let data = KVStore::read( + self.store, + CHANNEL_MONITOR_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MONITOR_PERSISTENCE_SECONDARY_NAMESPACE, + &stored_key, + ) + .await?; + match )>>::read( + &mut io::Cursor::new(data), + (self.entropy_source, self.signer_provider), + ) { + Ok(Some((best_block, channel_monitor))) => { + res.push((best_block, channel_monitor)); + }, + Ok(None) => {}, + Err(_) => { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Failed to read ChannelMonitor", + )); + }, + } } - } else { - Err(io::Error::new(io::ErrorKind::NotFound, "Namespace not found")) - } - } - - fn write_internal( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, - ) -> io::Result<()> { - let mut persisted_lock = self.persisted_bytes.lock().unwrap(); - - let prefixed = format!("{primary_namespace}/{secondary_namespace}"); - let outer_e = persisted_lock.entry(prefixed.clone()).or_insert(HashMap::new()); - outer_e.insert(key.to_string(), buf); - - // Only assign creation time on first write (not on update) - let mut ct_lock = self.creation_times.lock().unwrap(); - let ct_ns = ct_lock.entry(prefixed).or_insert(HashMap::new()); - ct_ns - .entry(key.to_string()) - .or_insert_with(|| self.creation_counter.fetch_add(1, Ordering::Relaxed)); - - Ok(()) - } - - fn remove_internal( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, _lazy: bool, - ) -> io::Result<()> { - let mut persisted_lock = self.persisted_bytes.lock().unwrap(); - - let prefixed = format!("{primary_namespace}/{secondary_namespace}"); - if let Some(outer_ref) = persisted_lock.get_mut(&prefixed) { - outer_ref.remove(&key.to_string()); - } - - // Remove creation time entry - let mut ct_lock = self.creation_times.lock().unwrap(); - if let Some(ct_ns) = ct_lock.get_mut(&prefixed) { - ct_ns.remove(key); - } - - Ok(()) + Ok(res) + }) } - fn list_internal( - &self, primary_namespace: &str, secondary_namespace: &str, - ) -> io::Result> { - let mut persisted_lock = self.persisted_bytes.lock().unwrap(); - - let prefixed = format!("{primary_namespace}/{secondary_namespace}"); - match persisted_lock.entry(prefixed) { - hash_map::Entry::Occupied(e) => Ok(e.get().keys().cloned().collect()), - hash_map::Entry::Vacant(_) => Ok(Vec::new()), + fn write_monitor( + &self, monitor_name: MonitorName, monitor: &ChannelMonitor, + ) -> ChannelMonitorUpdateStatus { + let write_res = self.runtime.block_on(KVStore::write( + self.store, + CHANNEL_MONITOR_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MONITOR_PERSISTENCE_SECONDARY_NAMESPACE, + &monitor_name.to_string(), + monitor.encode(), + )); + match write_res { + Ok(()) => ChannelMonitorUpdateStatus::Completed, + Err(_) => ChannelMonitorUpdateStatus::UnrecoverableError, } } } -impl KVStore for InMemoryStore { - fn read( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, - ) -> impl Future, io::Error>> + 'static + Send { - let res = self.read_internal(&primary_namespace, &secondary_namespace, &key); - async move { res } - } - fn write( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, - ) -> impl Future> + 'static + Send { - let res = self.write_internal(&primary_namespace, &secondary_namespace, &key, buf); - async move { res } - } - fn remove( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, - ) -> impl Future> + 'static + Send { - let res = self.remove_internal(&primary_namespace, &secondary_namespace, &key, lazy); - async move { res } - } - fn list( - &self, primary_namespace: &str, secondary_namespace: &str, - ) -> impl Future, io::Error>> + 'static + Send { - let res = self.list_internal(primary_namespace, secondary_namespace); - async move { res } - } -} - -impl KVStoreSync for InMemoryStore { - fn read( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, - ) -> io::Result> { - self.read_internal(primary_namespace, secondary_namespace, key) +impl chainmonitor::Persist + for TestMonitorUpdatePersister<'_, K> +{ + fn persist_new_channel( + &self, monitor_name: MonitorName, monitor: &ChannelMonitor, + ) -> ChannelMonitorUpdateStatus { + self.write_monitor(monitor_name, monitor) } - fn write( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, - ) -> io::Result<()> { - self.write_internal(primary_namespace, secondary_namespace, key, buf) + fn update_persisted_channel( + &self, monitor_name: MonitorName, _monitor_update: Option<&ChannelMonitorUpdate>, + monitor: &ChannelMonitor, + ) -> ChannelMonitorUpdateStatus { + self.write_monitor(monitor_name, monitor) } - fn remove( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, - ) -> io::Result<()> { - self.remove_internal(primary_namespace, secondary_namespace, key, lazy) - } - - fn list(&self, primary_namespace: &str, secondary_namespace: &str) -> io::Result> { - self.list_internal(primary_namespace, secondary_namespace) - } -} - -impl InMemoryStore { - fn list_paginated_internal( - &self, primary_namespace: &str, secondary_namespace: &str, page_token: Option, - ) -> io::Result { - let ct_lock = self.creation_times.lock().unwrap(); - let prefixed = format!("{primary_namespace}/{secondary_namespace}"); - - let ct_ns = match ct_lock.get(&prefixed) { - Some(m) => m, - None => { - return Ok(PaginatedListResponse { keys: Vec::new(), next_page_token: None }); - }, - }; - - // Build list of (key, sort_order) sorted by sort_order DESC (newest first). - let mut entries: Vec<(&String, &u64)> = ct_ns.iter().collect(); - entries.sort_by(|a, b| b.1.cmp(a.1)); - - // Apply page token filter - let start_idx = if let Some(ref token) = page_token { - let token_sort_order: u64 = token - .as_str() - .parse() - .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "Invalid page token"))?; - - entries - .iter() - .position(|(_, sort_order)| **sort_order < token_sort_order) - .unwrap_or(entries.len()) - } else { - 0 - }; - - // Fetch one extra entry beyond page size to determine whether a next page exists. - let mut page: Vec<(&String, &u64)> = - entries[start_idx..].iter().take(IN_MEMORY_PAGE_SIZE + 1).cloned().collect(); - - let has_more = page.len() > IN_MEMORY_PAGE_SIZE; - page.truncate(IN_MEMORY_PAGE_SIZE); - - let next_page_token = if has_more { - let (_, last_sort_order) = page.last().unwrap(); - Some(PageToken::new(last_sort_order.to_string())) - } else { - None - }; - - let page: Vec = page.into_iter().map(|(k, _)| k.clone()).collect(); - - Ok(PaginatedListResponse { keys: page, next_page_token }) - } -} - -impl PaginatedKVStoreSync for InMemoryStore { - fn list_paginated( - &self, primary_namespace: &str, secondary_namespace: &str, page_token: Option, - ) -> io::Result { - self.list_paginated_internal(primary_namespace, secondary_namespace, page_token) + fn archive_persisted_channel(&self, monitor_name: MonitorName) { + let key = monitor_name.to_string(); + self.runtime.block_on(async { + let monitor = match KVStore::read( + self.store, + CHANNEL_MONITOR_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MONITOR_PERSISTENCE_SECONDARY_NAMESPACE, + &key, + ) + .await + { + Ok(monitor) => monitor, + Err(_) => return, + }; + + if KVStore::write( + self.store, + ARCHIVED_CHANNEL_MONITOR_PERSISTENCE_PRIMARY_NAMESPACE, + ARCHIVED_CHANNEL_MONITOR_PERSISTENCE_SECONDARY_NAMESPACE, + &key, + monitor, + ) + .await + .is_ok() + { + let _ = KVStore::remove( + self.store, + CHANNEL_MONITOR_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MONITOR_PERSISTENCE_SECONDARY_NAMESPACE, + &key, + true, + ) + .await; + } + }); } } -impl PaginatedKVStore for InMemoryStore { - fn list_paginated( - &self, primary_namespace: &str, secondary_namespace: &str, page_token: Option, - ) -> impl Future> + 'static + Send { - let res = self.list_paginated_internal(primary_namespace, secondary_namespace, page_token); - async move { res } - } -} +const EXPECTED_UPDATES_PER_PAYMENT: u64 = 5; -unsafe impl Sync for InMemoryStore {} -unsafe impl Send for InMemoryStore {} +pub(crate) use in_memory_store::InMemoryStore; pub(crate) fn random_storage_path() -> PathBuf { let mut temp_path = std::env::temp_dir(); @@ -256,7 +169,32 @@ pub(crate) fn random_storage_path() -> PathBuf { temp_path } -pub(crate) fn do_read_write_remove_list_persist(kv_store: &K) { +async fn catch_future_unwind(future: F) -> std::thread::Result { + let mut future = std::pin::pin!(future); + std::future::poll_fn(|cx| { + match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| future.as_mut().poll(cx))) { + Ok(std::task::Poll::Ready(output)) => std::task::Poll::Ready(Ok(output)), + Ok(std::task::Poll::Pending) => std::task::Poll::Pending, + Err(panic) => std::task::Poll::Ready(Err(panic)), + } + }) + .await +} + +async fn assert_invalid_write_fails( + kv_store: &K, primary_namespace: &str, secondary_namespace: &str, key: &str, data: Vec, +) { + let res = std::panic::catch_unwind(|| { + KVStore::write(kv_store, primary_namespace, secondary_namespace, key, data) + }); + if let Ok(fut) = res { + if let Ok(write_res) = catch_future_unwind(fut).await { + assert!(write_res.is_err()); + } + } +} + +pub(crate) async fn do_read_write_remove_list_persist(kv_store: &K) { let data = vec![42u8; 32]; let primary_namespace = "testspace"; @@ -264,63 +202,63 @@ pub(crate) fn do_read_write_remove_list_persist( let key = "testkey"; // Test the basic KVStore operations. - kv_store.write(primary_namespace, secondary_namespace, key, data.clone()).unwrap(); + KVStore::write(kv_store, primary_namespace, secondary_namespace, key, data.clone()) + .await + .unwrap(); // Test empty primary/secondary namespaces are allowed, but not empty primary namespace and non-empty // secondary primary_namespace, and not empty key. - kv_store.write("", "", key, data.clone()).unwrap(); - let res = - std::panic::catch_unwind(|| kv_store.write("", secondary_namespace, key, data.clone())); - assert!(res.is_err()); - let res = std::panic::catch_unwind(|| { - kv_store.write(primary_namespace, secondary_namespace, "", data.clone()) - }); - assert!(res.is_err()); + KVStore::write(kv_store, "", "", key, data.clone()).await.unwrap(); + assert_invalid_write_fails(kv_store, "", secondary_namespace, key, data.clone()).await; + assert_invalid_write_fails(kv_store, primary_namespace, secondary_namespace, "", data.clone()) + .await; - let listed_keys = kv_store.list(primary_namespace, secondary_namespace).unwrap(); + let listed_keys = + KVStore::list(kv_store, primary_namespace, secondary_namespace).await.unwrap(); assert_eq!(listed_keys.len(), 1); assert_eq!(listed_keys[0], key); - let read_data = kv_store.read(primary_namespace, secondary_namespace, key).unwrap(); + let read_data = + KVStore::read(kv_store, primary_namespace, secondary_namespace, key).await.unwrap(); assert_eq!(data, &*read_data); - kv_store.remove(primary_namespace, secondary_namespace, key, false).unwrap(); + KVStore::remove(kv_store, primary_namespace, secondary_namespace, key, false).await.unwrap(); - let listed_keys = kv_store.list(primary_namespace, secondary_namespace).unwrap(); + let listed_keys = + KVStore::list(kv_store, primary_namespace, secondary_namespace).await.unwrap(); assert_eq!(listed_keys.len(), 0); // Ensure we have no issue operating with primary_namespace/secondary_namespace/key being KVSTORE_NAMESPACE_KEY_MAX_LEN let max_chars: String = std::iter::repeat('A').take(KVSTORE_NAMESPACE_KEY_MAX_LEN).collect(); - kv_store.write(&max_chars, &max_chars, &max_chars, data.clone()).unwrap(); + KVStore::write(kv_store, &max_chars, &max_chars, &max_chars, data.clone()).await.unwrap(); - let listed_keys = kv_store.list(&max_chars, &max_chars).unwrap(); + let listed_keys = KVStore::list(kv_store, &max_chars, &max_chars).await.unwrap(); assert_eq!(listed_keys.len(), 1); assert_eq!(listed_keys[0], max_chars); - let read_data = kv_store.read(&max_chars, &max_chars, &max_chars).unwrap(); + let read_data = KVStore::read(kv_store, &max_chars, &max_chars, &max_chars).await.unwrap(); assert_eq!(data, &*read_data); - kv_store.remove(&max_chars, &max_chars, &max_chars, false).unwrap(); + KVStore::remove(kv_store, &max_chars, &max_chars, &max_chars, false).await.unwrap(); - let listed_keys = kv_store.list(&max_chars, &max_chars).unwrap(); + let listed_keys = KVStore::list(kv_store, &max_chars, &max_chars).await.unwrap(); assert_eq!(listed_keys.len(), 0); } -pub(crate) fn create_persister<'a, K: KVStoreSync + Sync>( - store: &'a K, chanmon_cfg: &'a TestChanMonCfg, max_pending_updates: u64, +pub(crate) fn create_persister<'a, K: KVStore + Sync>( + store: &'a K, chanmon_cfg: &'a TestChanMonCfg, _max_pending_updates: u64, ) -> TestMonitorUpdatePersister<'a, K> { - MonitorUpdatingPersister::new( + let runtime = + Runtime::new(Arc::new(Logger::new_log_facade())).expect("Failed to setup runtime"); + TestMonitorUpdatePersister { store, - &chanmon_cfg.logger, - max_pending_updates, - &chanmon_cfg.keys_manager, - &chanmon_cfg.keys_manager, - &chanmon_cfg.tx_broadcaster, - &chanmon_cfg.fee_estimator, - ) + runtime, + entropy_source: &chanmon_cfg.keys_manager, + signer_provider: &chanmon_cfg.keys_manager, + } } -pub(crate) fn create_chain_monitor<'a, K: KVStoreSync + Sync>( +pub(crate) fn create_chain_monitor<'a, K: KVStore + Sync>( chanmon_cfg: &'a TestChanMonCfg, persister: &'a TestMonitorUpdatePersister<'a, K>, ) -> test_utils::TestChainMonitor<'a> { test_utils::TestChainMonitor::new( @@ -335,7 +273,7 @@ pub(crate) fn create_chain_monitor<'a, K: KVStoreSync + Sync>( // Integration-test the given KVStore implementation. Test relaying a few payments and check that // the persisted data is updated the appropriate number of times. -pub(crate) fn do_test_store(store_0: &K, store_1: &K) { +pub(crate) fn do_test_store(store_0: &K, store_1: &K) { // This value is used later to limit how many iterations we perform. let persister_0_max_pending_updates = 7; // Intentionally set this to a smaller value to test a different alignment. diff --git a/src/io/utils.rs b/src/io/utils.rs index 89d4afc5c3..4657688f51 100644 --- a/src/io/utils.rs +++ b/src/io/utils.rs @@ -11,7 +11,7 @@ use std::ops::Deref; #[cfg(unix)] use std::os::unix::fs::OpenOptionsExt; use std::path::{Path, PathBuf}; -use std::sync::{Arc, RwLock}; +use std::sync::Arc; use bdk_chain::indexer::keychain_txout::ChangeSet as BdkIndexerChangeSet; use bdk_chain::local_chain::ChangeSet as BdkLocalChainChangeSet; @@ -26,7 +26,7 @@ use lightning::routing::scoring::{ ChannelLiquidities, ProbabilisticScorer, ProbabilisticScoringDecayParameters, }; use lightning::util::persist::{ - migrate_kv_store_data, KVStore, KVStoreSync, KVSTORE_NAMESPACE_KEY_ALPHABET, + migrate_kv_store_data_async, KVStore, KVSTORE_NAMESPACE_KEY_ALPHABET, KVSTORE_NAMESPACE_KEY_MAX_LEN, NETWORK_GRAPH_PERSISTENCE_KEY, NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, OUTPUT_SWEEPER_PERSISTENCE_KEY, OUTPUT_SWEEPER_PERSISTENCE_PRIMARY_NAMESPACE, @@ -49,7 +49,7 @@ use crate::logger::{log_error, LdkLogger, Logger}; use crate::peer_store::PeerStore; use crate::types::{Broadcaster, DynStore, KeysManager, Sweeper}; use crate::wallet::ser::{ChangeSetDeserWrapper, ChangeSetSerWrapper}; -use crate::{BuildError, Error, EventQueue, NodeMetrics}; +use crate::{BuildError, Error, EventQueue, NodeMetrics, PersistedNodeMetrics}; pub const EXTERNAL_PATHFINDING_SCORES_CACHE_KEY: &str = "external_pathfinding_scores_cache"; @@ -336,26 +336,27 @@ where } /// Take a write lock on `node_metrics`, apply `update`, and persist the result to `kv_store`. -/// -/// The write lock is held across the KV-store write, preserving the invariant that readers only -/// observe the mutation once it has been durably persisted (or the persist has failed). -pub(crate) fn update_and_persist_node_metrics( - node_metrics: &RwLock, kv_store: &DynStore, logger: L, +pub(crate) async fn update_and_persist_node_metrics( + node_metrics: &PersistedNodeMetrics, kv_store: &DynStore, logger: L, update: impl FnOnce(&mut NodeMetrics), ) -> Result<(), Error> where L::Target: LdkLogger, { - let mut locked_node_metrics = node_metrics.write().expect("lock"); - update(&mut *locked_node_metrics); - let data = locked_node_metrics.encode(); - KVStoreSync::write( + let _guard = node_metrics.lock_mutation().await; + let data = { + let mut locked_node_metrics = node_metrics.write().expect("lock"); + update(&mut *locked_node_metrics); + locked_node_metrics.encode() + }; + KVStore::write( &*kv_store, NODE_METRICS_PRIMARY_NAMESPACE, NODE_METRICS_SECONDARY_NAMESPACE, NODE_METRICS_KEY, data, ) + .await .map_err(|e| { log_error!( logger, @@ -469,14 +470,15 @@ macro_rules! impl_read_write_change_set_type { $secondary_namespace:expr, $key:expr ) => { - pub(crate) fn $read_name( + pub(crate) async fn $read_name( kv_store: &DynStore, logger: L, ) -> Result, std::io::Error> where L::Target: LdkLogger, { let reader = - match KVStoreSync::read(&*kv_store, $primary_namespace, $secondary_namespace, $key) + match KVStore::read(&*kv_store, $primary_namespace, $secondary_namespace, $key) + .await { Ok(bytes) => bytes, Err(e) => { @@ -510,14 +512,15 @@ macro_rules! impl_read_write_change_set_type { } } - pub(crate) fn $write_name( + pub(crate) async fn $write_name( value: &$change_set_type, kv_store: &DynStore, logger: L, ) -> Result<(), std::io::Error> where L::Target: LdkLogger, { let data = ChangeSetSerWrapper(value).encode(); - KVStoreSync::write(&*kv_store, $primary_namespace, $secondary_namespace, $key, data) + KVStore::write(&*kv_store, $primary_namespace, $secondary_namespace, $key, data) + .await .map_err(|e| { log_error!( logger, @@ -588,36 +591,39 @@ impl_read_write_change_set_type!( ); // Reads the full BdkWalletChangeSet or returns default fields -pub(crate) fn read_bdk_wallet_change_set( +pub(crate) async fn read_bdk_wallet_change_set( kv_store: &DynStore, logger: &Logger, ) -> Result, std::io::Error> { let mut change_set = BdkWalletChangeSet::default(); // We require a descriptor and return `None` to signal creation of a new wallet otherwise. - if let Some(descriptor) = read_bdk_wallet_descriptor(kv_store, logger)? { + if let Some(descriptor) = read_bdk_wallet_descriptor(kv_store, logger).await? { change_set.descriptor = Some(descriptor); } else { return Ok(None); } // We require a change_descriptor and return `None` to signal creation of a new wallet otherwise. - if let Some(change_descriptor) = read_bdk_wallet_change_descriptor(kv_store, logger)? { + if let Some(change_descriptor) = read_bdk_wallet_change_descriptor(kv_store, logger).await? { change_set.change_descriptor = Some(change_descriptor); } else { return Ok(None); } // We require a network and return `None` to signal creation of a new wallet otherwise. - if let Some(network) = read_bdk_wallet_network(kv_store, logger)? { + if let Some(network) = read_bdk_wallet_network(kv_store, logger).await? { change_set.network = Some(network); } else { return Ok(None); } - read_bdk_wallet_local_chain(&*kv_store, logger)? + read_bdk_wallet_local_chain(&*kv_store, logger) + .await? .map(|local_chain| change_set.local_chain = local_chain); - read_bdk_wallet_tx_graph(&*kv_store, logger)?.map(|tx_graph| change_set.tx_graph = tx_graph); - read_bdk_wallet_indexer(&*kv_store, logger)?.map(|indexer| change_set.indexer = indexer); + read_bdk_wallet_tx_graph(&*kv_store, logger) + .await? + .map(|tx_graph| change_set.tx_graph = tx_graph); + read_bdk_wallet_indexer(&*kv_store, logger).await?.map(|indexer| change_set.indexer = indexer); Ok(Some(change_set)) } @@ -626,7 +632,7 @@ pub(crate) fn read_bdk_wallet_change_set( /// If the directory contains v1 data (files at the top level), the data is migrated to v2 format /// in a temporary directory, the original is renamed to `fs_store_v1_backup`, and the migrated /// directory is moved into place. -pub(crate) fn open_or_migrate_fs_store( +pub(crate) async fn open_or_migrate_fs_store( storage_dir_path: PathBuf, ) -> Result { let parent_dir = storage_dir_path.parent().ok_or(BuildError::StoragePathAccessFailed)?; @@ -641,14 +647,15 @@ pub(crate) fn open_or_migrate_fs_store( Ok(store) => Ok(store), Err(FilesystemStoreV2Error::V1DataDetected(_)) => { // The directory contains v1 data, migrate to v2. - let mut v1_store = FilesystemStore::new(storage_dir_path.clone()); + let v1_store = FilesystemStore::new(storage_dir_path.clone()); let v2_dir = fs_store_sibling_path(&storage_dir_path, "fs_store_v2_migrating"); fs::create_dir_all(v2_dir.clone()).map_err(|_| BuildError::StoragePathAccessFailed)?; - let mut v2_store = FilesystemStoreV2::new(v2_dir.clone()) + let v2_store = FilesystemStoreV2::new(v2_dir.clone()) .map_err(|_| BuildError::KVStoreSetupFailed)?; - migrate_kv_store_data(&mut v1_store, &mut v2_store) + migrate_kv_store_data_async(&v1_store, &v2_store) + .await .map_err(|_| BuildError::KVStoreSetupFailed)?; // Swap directories: rename v1 out of the way, move v2 into place. @@ -712,7 +719,7 @@ mod tests { use std::fs; use std::path::{Path, PathBuf}; - use lightning::util::persist::{migrate_kv_store_data, KVStoreSync}; + use lightning::util::persist::{migrate_kv_store_data_async, KVStore}; use lightning_persister::fs_store::v1::FilesystemStore; use lightning_persister::fs_store::v2::FilesystemStoreV2; @@ -733,22 +740,23 @@ mod tests { assert_eq!(expected_seed_bytes, read_seed_bytes); } - #[test] - fn fs_store_migration_recovers_before_v1_backup_rename() { + #[tokio::test] + async fn fs_store_migration_recovers_before_v1_backup_rename() { let fs_store_path = fs_store_path(); - let mut v1_store = write_v1_test_data(&fs_store_path); + let v1_store = write_v1_test_data(&fs_store_path).await; let v2_migrating_path = sibling_path(&fs_store_path, "fs_store_v2_migrating"); - let mut v2_store = FilesystemStoreV2::new(v2_migrating_path.clone()).unwrap(); - migrate_kv_store_data(&mut v1_store, &mut v2_store).unwrap(); + let v2_store = FilesystemStoreV2::new(v2_migrating_path.clone()).unwrap(); + migrate_kv_store_data_async(&v1_store, &v2_store).await.unwrap(); - let migrated_store = open_or_migrate_fs_store(fs_store_path.clone()).unwrap(); + let migrated_store = open_or_migrate_fs_store(fs_store_path.clone()).await.unwrap(); assert_eq!( - KVStoreSync::read( + KVStore::read( &migrated_store, TEST_PRIMARY_NAMESPACE, TEST_SECONDARY_NAMESPACE, TEST_KEY ) + .await .unwrap(), TEST_VALUE ); @@ -756,25 +764,26 @@ mod tests { assert!(!v2_migrating_path.exists()); } - #[test] - fn fs_store_migration_recovers_after_v1_backup_rename() { + #[tokio::test] + async fn fs_store_migration_recovers_after_v1_backup_rename() { let fs_store_path = fs_store_path(); - let mut v1_store = write_v1_test_data(&fs_store_path); + let v1_store = write_v1_test_data(&fs_store_path).await; let v2_migrating_path = sibling_path(&fs_store_path, "fs_store_v2_migrating"); - let mut v2_store = FilesystemStoreV2::new(v2_migrating_path.clone()).unwrap(); - migrate_kv_store_data(&mut v1_store, &mut v2_store).unwrap(); + let v2_store = FilesystemStoreV2::new(v2_migrating_path.clone()).unwrap(); + migrate_kv_store_data_async(&v1_store, &v2_store).await.unwrap(); let backup_path = sibling_path(&fs_store_path, "fs_store_v1_backup"); fs::rename(&fs_store_path, backup_path).unwrap(); - let migrated_store = open_or_migrate_fs_store(fs_store_path.clone()).unwrap(); + let migrated_store = open_or_migrate_fs_store(fs_store_path.clone()).await.unwrap(); assert_eq!( - KVStoreSync::read( + KVStore::read( &migrated_store, TEST_PRIMARY_NAMESPACE, TEST_SECONDARY_NAMESPACE, TEST_KEY ) + .await .unwrap(), TEST_VALUE ); @@ -782,26 +791,27 @@ mod tests { assert!(!v2_migrating_path.exists()); } - #[test] - fn fs_store_migration_recovers_after_v2_rename() { + #[tokio::test] + async fn fs_store_migration_recovers_after_v2_rename() { let fs_store_path = fs_store_path(); - let mut v1_store = write_v1_test_data(&fs_store_path); + let v1_store = write_v1_test_data(&fs_store_path).await; let v2_migrating_path = sibling_path(&fs_store_path, "fs_store_v2_migrating"); - let mut v2_store = FilesystemStoreV2::new(v2_migrating_path.clone()).unwrap(); - migrate_kv_store_data(&mut v1_store, &mut v2_store).unwrap(); + let v2_store = FilesystemStoreV2::new(v2_migrating_path.clone()).unwrap(); + migrate_kv_store_data_async(&v1_store, &v2_store).await.unwrap(); let backup_path = sibling_path(&fs_store_path, "fs_store_v1_backup"); fs::rename(&fs_store_path, &backup_path).unwrap(); fs::rename(&v2_migrating_path, &fs_store_path).unwrap(); - let migrated_store = open_or_migrate_fs_store(fs_store_path.clone()).unwrap(); + let migrated_store = open_or_migrate_fs_store(fs_store_path.clone()).await.unwrap(); assert_eq!( - KVStoreSync::read( + KVStore::read( &migrated_store, TEST_PRIMARY_NAMESPACE, TEST_SECONDARY_NAMESPACE, TEST_KEY ) + .await .unwrap(), TEST_VALUE ); @@ -810,22 +820,23 @@ mod tests { assert!(!v2_migrating_path.exists()); } - #[test] - fn fs_store_migration_recovers_backup_without_migrating_dir() { + #[tokio::test] + async fn fs_store_migration_recovers_backup_without_migrating_dir() { let fs_store_path = fs_store_path(); - write_v1_test_data(&fs_store_path); + write_v1_test_data(&fs_store_path).await; let backup_path = sibling_path(&fs_store_path, "fs_store_v1_backup"); fs::rename(&fs_store_path, backup_path).unwrap(); - let migrated_store = open_or_migrate_fs_store(fs_store_path.clone()).unwrap(); + let migrated_store = open_or_migrate_fs_store(fs_store_path.clone()).await.unwrap(); assert_eq!( - KVStoreSync::read( + KVStore::read( &migrated_store, TEST_PRIMARY_NAMESPACE, TEST_SECONDARY_NAMESPACE, TEST_KEY ) + .await .unwrap(), TEST_VALUE ); @@ -833,28 +844,30 @@ mod tests { assert!(!sibling_path(&fs_store_path, "fs_store_v1_backup").exists()); } - #[test] - fn fs_store_migration_recovers_unexpected_migrating_dir_without_backup() { + #[tokio::test] + async fn fs_store_migration_recovers_unexpected_migrating_dir_without_backup() { let fs_store_path = fs_store_path(); let v2_migrating_path = sibling_path(&fs_store_path, "fs_store_v2_migrating"); let v2_store = FilesystemStoreV2::new(v2_migrating_path.clone()).unwrap(); - KVStoreSync::write( + KVStore::write( &v2_store, TEST_PRIMARY_NAMESPACE, TEST_SECONDARY_NAMESPACE, TEST_KEY, TEST_VALUE.to_vec(), ) + .await .unwrap(); - let migrated_store = open_or_migrate_fs_store(fs_store_path.clone()).unwrap(); + let migrated_store = open_or_migrate_fs_store(fs_store_path.clone()).await.unwrap(); assert_eq!( - KVStoreSync::read( + KVStore::read( &migrated_store, TEST_PRIMARY_NAMESPACE, TEST_SECONDARY_NAMESPACE, TEST_KEY ) + .await .unwrap(), TEST_VALUE ); @@ -874,15 +887,16 @@ mod tests { sibling_path } - fn write_v1_test_data(fs_store_path: &Path) -> FilesystemStore { + async fn write_v1_test_data(fs_store_path: &Path) -> FilesystemStore { let v1_store = FilesystemStore::new(fs_store_path.to_path_buf()); - KVStoreSync::write( + KVStore::write( &v1_store, TEST_PRIMARY_NAMESPACE, TEST_SECONDARY_NAMESPACE, TEST_KEY, TEST_VALUE.to_vec(), ) + .await .unwrap(); v1_store } diff --git a/src/io/vss_store.rs b/src/io/vss_store.rs index 97883b5d53..4b8cca754b 100644 --- a/src/io/vss_store.rs +++ b/src/io/vss_store.rs @@ -13,7 +13,7 @@ use std::fmt; use std::future::Future; #[cfg(test)] use std::panic::RefUnwindSafe; -use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; use std::time::Duration; @@ -21,10 +21,10 @@ use bitcoin::bip32::{ChildNumber, Xpriv}; use bitcoin::hashes::{sha256, Hash, HashEngine, Hmac, HmacEngine}; use bitcoin::key::Secp256k1; use bitcoin::Network; -use lightning::impl_writeable_tlv_based_enum; +use lightning::impl_ser_tlv_based_enum; use lightning::io::{self, Error, ErrorKind}; use lightning::sign::{EntropySource as LdkEntropySource, RandomBytes}; -use lightning::util::persist::{KVStore, KVStoreSync}; +use lightning::util::persist::KVStore; use lightning::util::ser::{Readable, Writeable}; use prost::Message; use vss_client::client::VssClient; @@ -45,6 +45,7 @@ use vss_client::util::storable_builder::{EntropySource, StorableBuilder}; use crate::entropy::NodeEntropy; use crate::io::utils::check_namespace_key_validity; use crate::lnurl_auth::LNURL_AUTH_HARDENED_CHILD_INDEX; +use crate::runtime::StoreRuntime; type CustomRetryPolicy = FilteredRetryPolicy< JitteredRetryPolicy< @@ -64,7 +65,7 @@ enum VssSchemaVersion { V1, } -impl_writeable_tlv_based_enum!(VssSchemaVersion, +impl_ser_tlv_based_enum!(VssSchemaVersion, (0, V0) => {}, (1, V1) => {}, ); @@ -77,7 +78,19 @@ const VSS_SCHEMA_VERSION_KEY: &str = "vss_schema_version"; // would hit a blocking case const INTERNAL_RUNTIME_WORKERS: usize = 2; -/// A [`KVStore`]/[`KVStoreSync`] implementation that writes to and reads from a [VSS] backend. +async fn run_on_internal_runtime( + runtime: Arc, future: impl Future> + Send + 'static, +) -> io::Result +where + T: Send + 'static, +{ + let task = runtime.spawn(future); + task.await.map_err(|e| { + io::Error::new(io::ErrorKind::Other, format!("VSS runtime task failed: {}", e)) + })? +} + +/// A [`KVStore`] implementation that writes to and reads from a [VSS] backend. /// /// [VSS]: https://github.com/lightningdevkit/vss-server/blob/main/README.md pub struct VssStore { @@ -85,13 +98,8 @@ pub struct VssStore { // Version counter to ensure that writes are applied in the correct order. It is assumed that read and list // operations aren't sensitive to the order of execution. next_version: AtomicU64, - // A VSS-internal runtime we use to avoid any deadlocks we could hit when waiting on a spawned - // blocking task to finish while the blocked thread had acquired the reactor. In particular, - // this works around a previously-hit case where a concurrent call to - // `PeerManager::process_pending_events` -> `ChannelManager::get_and_clear_pending_msg_events` - // would deadlock when trying to acquire sync `Mutex` locks that are held by the thread - // currently being blocked waiting on the VSS operation to finish. - internal_runtime: Option, + // A VSS-internal runtime that drives VSS I/O independently from the node runtime. + internal_runtime: Option>, } impl VssStore { @@ -100,56 +108,49 @@ impl VssStore { header_provider: Arc, ) -> io::Result { let next_version = AtomicU64::new(1); - let internal_runtime = tokio::runtime::Builder::new_multi_thread() - .enable_all() - .thread_name_fn(|| { - static ATOMIC_ID: AtomicUsize = AtomicUsize::new(0); - let id = ATOMIC_ID.fetch_add(1, Ordering::SeqCst); - format!("ldk-node-vss-runtime-{}", id) - }) - .worker_threads(INTERNAL_RUNTIME_WORKERS) - .max_blocking_threads(INTERNAL_RUNTIME_WORKERS) - .build() - .map_err(|e| { - io::Error::new(io::ErrorKind::Other, format!("Failed to build VSS runtime: {}", e)) - })?; + let internal_runtime = + Arc::new(StoreRuntime::new("ldk-node-vss-runtime", INTERNAL_RUNTIME_WORKERS, "VSS")?); let (data_encryption_key, obfuscation_master_key) = derive_data_encryption_and_obfuscation_keys(&vss_seed); let key_obfuscator = KeyObfuscator::new(obfuscation_master_key); + let setup_key_obfuscator = KeyObfuscator::new(obfuscation_master_key); let mut entropy_seed = [0u8; 32]; getrandom::fill(&mut entropy_seed).expect("Failed to generate random bytes"); let entropy_source = RandomBytes::new(entropy_seed); + let setup_entropy_source = RandomBytes::new(entropy_seed); - let sync_retry_policy = retry_policy(); - let blocking_client = VssClient::new_with_headers( + let setup_retry_policy = retry_policy(); + let setup_client = VssClient::new_with_headers( base_url.clone(), - sync_retry_policy, - header_provider.clone(), + setup_retry_policy, + Arc::clone(&header_provider), ); - let runtime_handle = internal_runtime.handle(); - let schema_version = tokio::task::block_in_place(|| { + let async_retry_policy = retry_policy(); + let async_client = + VssClient::new_with_headers(base_url, async_retry_policy, header_provider); + + let setup_store_id = store_id.clone(); + let runtime_handle = internal_runtime.handle().clone(); + let schema_version = std::thread::spawn(move || { runtime_handle.block_on(async { determine_and_write_schema_version( - &blocking_client, - &store_id, + &setup_client, + &setup_store_id, data_encryption_key, - &key_obfuscator, - &entropy_source, + &setup_key_obfuscator, + &setup_entropy_source, ) .await }) - })?; - - let async_retry_policy = retry_policy(); - let async_client = - VssClient::new_with_headers(base_url, async_retry_policy, header_provider); + }) + .join() + .map_err(|_| io::Error::new(io::ErrorKind::Other, "VSS schema setup task panicked"))??; let inner = Arc::new(VssStoreInner::new( schema_version, - blocking_client, async_client, store_id, data_encryption_key, @@ -159,6 +160,10 @@ impl VssStore { Ok(Self { inner, next_version, internal_runtime: Some(internal_runtime) }) } + + fn internal_runtime(&self) -> Arc { + Arc::clone(self.internal_runtime.as_ref().expect("VSS runtime must be available")) + } /// Returns a [`VssStoreBuilder`] allowing to build a [`VssStore`]. pub fn builder( node_entropy: NodeEntropy, vss_url: String, store_id: String, network: Network, @@ -193,111 +198,6 @@ impl VssStore { } } -impl KVStoreSync for VssStore { - fn read( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, - ) -> io::Result> { - let internal_runtime = self.internal_runtime.as_ref().ok_or_else(|| { - debug_assert!(false, "Failed to access internal runtime"); - let msg = format!("Failed to access internal runtime"); - Error::new(ErrorKind::Other, msg) - })?; - let primary_namespace = primary_namespace.to_string(); - let secondary_namespace = secondary_namespace.to_string(); - let key = key.to_string(); - let inner = Arc::clone(&self.inner); - let fut = async move { - inner - .read_internal(&inner.blocking_client, primary_namespace, secondary_namespace, key) - .await - }; - tokio::task::block_in_place(move || internal_runtime.block_on(fut)) - } - - fn write( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, - ) -> io::Result<()> { - let internal_runtime = self.internal_runtime.as_ref().ok_or_else(|| { - debug_assert!(false, "Failed to access internal runtime"); - let msg = format!("Failed to access internal runtime"); - Error::new(ErrorKind::Other, msg) - })?; - let primary_namespace = primary_namespace.to_string(); - let secondary_namespace = secondary_namespace.to_string(); - let key = key.to_string(); - let inner = Arc::clone(&self.inner); - let locking_key = self.build_locking_key(&primary_namespace, &secondary_namespace, &key); - let (inner_lock_ref, version) = self.get_new_version_and_lock_ref(locking_key.clone()); - let fut = async move { - inner - .write_internal( - &inner.blocking_client, - inner_lock_ref, - locking_key, - version, - primary_namespace, - secondary_namespace, - key, - buf, - ) - .await - }; - tokio::task::block_in_place(move || internal_runtime.block_on(fut)) - } - - fn remove( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, - ) -> io::Result<()> { - let internal_runtime = self.internal_runtime.as_ref().ok_or_else(|| { - debug_assert!(false, "Failed to access internal runtime"); - let msg = format!("Failed to access internal runtime"); - Error::new(ErrorKind::Other, msg) - })?; - let primary_namespace = primary_namespace.to_string(); - let secondary_namespace = secondary_namespace.to_string(); - let key = key.to_string(); - let inner = Arc::clone(&self.inner); - let locking_key = self.build_locking_key(&primary_namespace, &secondary_namespace, &key); - let (inner_lock_ref, version) = self.get_new_version_and_lock_ref(locking_key.clone()); - let fut = async move { - inner - .remove_internal( - &inner.blocking_client, - inner_lock_ref, - locking_key, - version, - primary_namespace, - secondary_namespace, - key, - ) - .await - }; - if lazy { - internal_runtime.spawn(async { fut.await }); - Ok(()) - } else { - tokio::task::block_in_place(move || internal_runtime.block_on(fut)) - } - } - - fn list(&self, primary_namespace: &str, secondary_namespace: &str) -> io::Result> { - let internal_runtime = self.internal_runtime.as_ref().ok_or_else(|| { - debug_assert!(false, "Failed to access internal runtime"); - let msg = format!("Failed to access internal runtime"); - Error::new(ErrorKind::Other, msg) - })?; - let primary_namespace = primary_namespace.to_string(); - let secondary_namespace = secondary_namespace.to_string(); - let inner = Arc::clone(&self.inner); - let fut = async move { - inner - .list_internal(&inner.blocking_client, primary_namespace, secondary_namespace) - .await - }; - tokio::task::block_in_place(move || internal_runtime.block_on(fut)) - } -} - impl KVStore for VssStore { fn read( &self, primary_namespace: &str, secondary_namespace: &str, key: &str, @@ -306,10 +206,14 @@ impl KVStore for VssStore { let secondary_namespace = secondary_namespace.to_string(); let key = key.to_string(); let inner = Arc::clone(&self.inner); + let runtime = self.internal_runtime(); async move { - inner - .read_internal(&inner.async_client, primary_namespace, secondary_namespace, key) - .await + run_on_internal_runtime(runtime, async move { + inner + .read_internal(&inner.async_client, primary_namespace, secondary_namespace, key) + .await + }) + .await } } fn write( @@ -321,19 +225,23 @@ impl KVStore for VssStore { let secondary_namespace = secondary_namespace.to_string(); let key = key.to_string(); let inner = Arc::clone(&self.inner); + let runtime = self.internal_runtime(); async move { - inner - .write_internal( - &inner.async_client, - inner_lock_ref, - locking_key, - version, - primary_namespace, - secondary_namespace, - key, - buf, - ) - .await + run_on_internal_runtime(runtime, async move { + inner + .write_internal( + &inner.async_client, + inner_lock_ref, + locking_key, + version, + primary_namespace, + secondary_namespace, + key, + buf, + ) + .await + }) + .await } } fn remove( @@ -345,6 +253,7 @@ impl KVStore for VssStore { let secondary_namespace = secondary_namespace.to_string(); let key = key.to_string(); let inner = Arc::clone(&self.inner); + let runtime = self.internal_runtime(); let fut = async move { inner .remove_internal( @@ -360,10 +269,12 @@ impl KVStore for VssStore { }; async move { if lazy { - tokio::task::spawn(async move { fut.await }); + runtime.spawn(async move { + let _ = fut.await; + }); Ok(()) } else { - fut.await + run_on_internal_runtime(runtime, fut).await } } } @@ -373,22 +284,30 @@ impl KVStore for VssStore { let primary_namespace = primary_namespace.to_string(); let secondary_namespace = secondary_namespace.to_string(); let inner = Arc::clone(&self.inner); + let runtime = self.internal_runtime(); async move { - inner.list_internal(&inner.async_client, primary_namespace, secondary_namespace).await + run_on_internal_runtime(runtime, async move { + inner + .list_internal(&inner.async_client, primary_namespace, secondary_namespace) + .await + }) + .await } } } impl Drop for VssStore { fn drop(&mut self) { - let internal_runtime = self.internal_runtime.take(); - tokio::task::block_in_place(move || drop(internal_runtime)); + if let Some(runtime) = self.internal_runtime.take() { + if let Ok(runtime) = Arc::try_unwrap(runtime) { + runtime.shutdown_background(); + } + } } } struct VssStoreInner { schema_version: VssSchemaVersion, - blocking_client: VssClient, // A secondary client that will only be used for async persistence via `KVStore`, to ensure TCP // connections aren't shared between our outer and the internal runtime. async_client: VssClient, @@ -403,14 +322,13 @@ struct VssStoreInner { impl VssStoreInner { pub(crate) fn new( - schema_version: VssSchemaVersion, blocking_client: VssClient, - async_client: VssClient, store_id: String, - data_encryption_key: [u8; 32], key_obfuscator: KeyObfuscator, entropy_source: RandomBytes, + schema_version: VssSchemaVersion, async_client: VssClient, + store_id: String, data_encryption_key: [u8; 32], key_obfuscator: KeyObfuscator, + entropy_source: RandomBytes, ) -> Self { let locks = Mutex::new(HashMap::new()); Self { schema_version, - blocking_client, async_client, store_id, data_encryption_key, @@ -1026,8 +944,8 @@ mod tests { use super::*; use crate::io::test_utils::do_read_write_remove_list_persist; - #[test] - fn vss_read_write_remove_list_persist() { + #[tokio::test] + async fn vss_read_write_remove_list_persist() { let vss_base_url = std::env::var("TEST_VSS_BASE_URL").unwrap(); let mut rng = rng(); let rand_store_id: String = (0..7).map(|_| rng.sample(Alphanumeric) as char).collect(); @@ -1038,7 +956,7 @@ mod tests { VssStoreBuilder::new(entropy, vss_base_url, rand_store_id, Network::Testnet) .build_with_sigs_auth(HashMap::new()) .unwrap(); - do_read_write_remove_list_persist(&vss_store); + do_read_write_remove_list_persist(&vss_store).await; } #[tokio::test(flavor = "multi_thread", worker_threads = 1)] @@ -1054,7 +972,7 @@ mod tests { .build_with_sigs_auth(HashMap::new()) .unwrap(); - do_read_write_remove_list_persist(&vss_store); + do_read_write_remove_list_persist(&vss_store).await; drop(vss_store) } } diff --git a/src/lib.rs b/src/lib.rs index 614be098b0..4f79c5adf2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -147,7 +147,7 @@ use graph::NetworkGraph; use io::utils::update_and_persist_node_metrics; pub use lightning; use lightning::chain::BlockLocator; -use lightning::impl_writeable_tlv_based; +use lightning::impl_ser_tlv_based; use lightning::ln::chan_utils::FUNDING_TRANSACTION_WITNESS_WEIGHT; use lightning::ln::channel_state::ChannelDetails as LdkChannelDetails; pub use lightning::ln::channel_state::ChannelShutdownState; @@ -156,7 +156,7 @@ use lightning::ln::msgs::{BaseMessageHandler, SocketAddress}; use lightning::ln::peer_handler::CustomMessageHandler; use lightning::routing::gossip::NodeAlias; use lightning::sign::EntropySource; -use lightning::util::persist::KVStoreSync; +use lightning::util::persist::KVStore; use lightning::util::wallet_utils::{Input, Wallet as LdkWallet}; use lightning_background_processor::process_events_async; pub use lightning_invoice; @@ -180,7 +180,7 @@ use types::{ HRNResolver, KeysManager, OnionMessenger, PaymentStore, PeerManager, Router, Scorer, Sweeper, Wallet, }; -pub use types::{ChannelDetails, CustomTlvRecord, PeerDetails, SyncAndAsyncKVStore, UserChannelId}; +pub use types::{ChannelDetails, CustomTlvRecord, PeerDetails, UserChannelId}; pub use vss_client; use crate::scoring::setup_background_pathfinding_scores_sync; @@ -243,7 +243,7 @@ pub struct Node { payment_store: Arc, lnurl_auth: Arc, is_running: Arc>, - node_metrics: Arc>, + node_metrics: Arc, om_mailbox: Option>, async_payments_role: Option, hrn_resolver: HRNResolver, @@ -556,6 +556,7 @@ impl Node { Arc::clone(&bcast_logger), |m| m.latest_node_announcement_broadcast_timestamp = unix_time_secs_opt, ) + .await .unwrap_or_else(|e| { log_error!(bcast_logger, "Persistence failed: {}", e); }); @@ -925,6 +926,7 @@ impl Node { #[cfg(not(feature = "uniffi"))] pub fn bolt12_payment(&self) -> Bolt12Payment { Bolt12Payment::new( + Arc::clone(&self.runtime), Arc::clone(&self.channel_manager), Arc::clone(&self.keys_manager), Arc::clone(&self.payment_store), @@ -941,6 +943,7 @@ impl Node { #[cfg(feature = "uniffi")] pub fn bolt12_payment(&self) -> Arc { Arc::new(Bolt12Payment::new( + Arc::clone(&self.runtime), Arc::clone(&self.channel_manager), Arc::clone(&self.keys_manager), Arc::clone(&self.payment_store), @@ -955,6 +958,7 @@ impl Node { #[cfg(not(feature = "uniffi"))] pub fn spontaneous_payment(&self) -> SpontaneousPayment { SpontaneousPayment::new( + Arc::clone(&self.runtime), Arc::clone(&self.channel_manager), Arc::clone(&self.keys_manager), Arc::clone(&self.payment_store), @@ -968,6 +972,7 @@ impl Node { #[cfg(feature = "uniffi")] pub fn spontaneous_payment(&self) -> Arc { Arc::new(SpontaneousPayment::new( + Arc::clone(&self.runtime), Arc::clone(&self.channel_manager), Arc::clone(&self.keys_manager), Arc::clone(&self.payment_store), @@ -1121,7 +1126,7 @@ impl Node { log_info!(self.logger, "Connected to peer {}@{}. ", peer_info.node_id, peer_info.address); if persist { - self.peer_store.add_peer(peer_info)?; + self.runtime.block_on(self.peer_store.add_peer(peer_info))?; } Ok(()) @@ -1138,7 +1143,7 @@ impl Node { log_info!(self.logger, "Disconnecting peer {}..", counterparty_node_id); - match self.peer_store.remove_peer(&counterparty_node_id) { + match self.runtime.block_on(self.peer_store.remove_peer(&counterparty_node_id)) { Ok(()) => {}, Err(e) => { log_error!(self.logger, "Failed to remove peer {}: {}", counterparty_node_id, e) @@ -1255,7 +1260,7 @@ impl Node { zero_reserve_string, peer_info.node_id ); - self.peer_store.add_peer(peer_info)?; + self.runtime.block_on(self.peer_store.add_peer(peer_info))?; Ok(UserChannelId(user_channel_id)) }, Err(e) => { @@ -1861,7 +1866,7 @@ impl Node { // Check if this was the last open channel, if so, forget the peer. if open_channels.len() == 1 { - self.peer_store.remove_peer(&counterparty_node_id)?; + self.runtime.block_on(self.peer_store.remove_peer(&counterparty_node_id))?; } } @@ -1899,7 +1904,7 @@ impl Node { /// Remove the payment with the given id from the store. pub fn remove_payment(&self, payment_id: &PaymentId) -> Result<(), Error> { - self.payment_store.remove(&payment_id) + self.runtime.block_on(self.payment_store.remove(&payment_id)) } /// Retrieves an overview of all known balances. @@ -2057,20 +2062,21 @@ impl Node { /// Exports the current state of the scorer. The result can be shared with and merged by light nodes that only have /// a limited view of the network. pub fn export_pathfinding_scores(&self) -> Result, Error> { - KVStoreSync::read( - &*self.kv_store, - lightning::util::persist::SCORER_PERSISTENCE_PRIMARY_NAMESPACE, - lightning::util::persist::SCORER_PERSISTENCE_SECONDARY_NAMESPACE, - lightning::util::persist::SCORER_PERSISTENCE_KEY, - ) - .map_err(|e| { - log_error!( - self.logger, - "Failed to access store while exporting pathfinding scores: {}", - e - ); - Error::PersistenceFailed - }) + self.runtime + .block_on(KVStore::read( + &*self.kv_store, + lightning::util::persist::SCORER_PERSISTENCE_PRIMARY_NAMESPACE, + lightning::util::persist::SCORER_PERSISTENCE_SECONDARY_NAMESPACE, + lightning::util::persist::SCORER_PERSISTENCE_KEY, + )) + .map_err(|e| { + log_error!( + self.logger, + "Failed to access store while exporting pathfinding scores: {}", + e + ); + Error::PersistenceFailed + }) } /// Return the features used in node announcement. @@ -2181,7 +2187,34 @@ impl Default for NodeMetrics { } } -impl_writeable_tlv_based!(NodeMetrics, { +pub(crate) struct PersistedNodeMetrics { + metrics: RwLock, + mutation_lock: tokio::sync::Mutex<()>, +} + +impl PersistedNodeMetrics { + pub(crate) fn new(metrics: NodeMetrics) -> Self { + Self { metrics: RwLock::new(metrics), mutation_lock: tokio::sync::Mutex::new(()) } + } + + pub(crate) fn read( + &self, + ) -> std::sync::LockResult> { + self.metrics.read() + } + + pub(crate) fn write( + &self, + ) -> std::sync::LockResult> { + self.metrics.write() + } + + pub(crate) async fn lock_mutation(&self) -> tokio::sync::MutexGuard<'_, ()> { + self.mutation_lock.lock().await + } +} + +impl_ser_tlv_based!(NodeMetrics, { (0, latest_lightning_wallet_sync_timestamp, option), (1, latest_pathfinding_scores_sync_timestamp, option), (2, latest_onchain_wallet_sync_timestamp, option), @@ -2251,7 +2284,7 @@ mod tests { latest_pathfinding_scores_sync_timestamp: Option, latest_node_announcement_broadcast_timestamp: Option, } - impl_writeable_tlv_based!(OldNodeMetrics, { + impl_ser_tlv_based!(OldNodeMetrics, { (0, latest_lightning_wallet_sync_timestamp, option), (1, latest_pathfinding_scores_sync_timestamp, option), (2, latest_onchain_wallet_sync_timestamp, option), diff --git a/src/payment/asynchronous/static_invoice_store.rs b/src/payment/asynchronous/static_invoice_store.rs index 6fb406334c..85d9479234 100644 --- a/src/payment/asynchronous/static_invoice_store.rs +++ b/src/payment/asynchronous/static_invoice_store.rs @@ -13,9 +13,9 @@ use std::time::Duration; use bitcoin::hashes::sha256::Hash as Sha256; use bitcoin::hashes::Hash; use lightning::blinded_path::message::BlindedMessagePath; -use lightning::impl_writeable_tlv_based; +use lightning::impl_ser_tlv_based; use lightning::offers::static_invoice::StaticInvoice; -use lightning::util::persist::KVStoreSync; +use lightning::util::persist::KVStore; use lightning::util::ser::{Readable, Writeable}; use crate::hex_utils; @@ -28,7 +28,7 @@ struct PersistedStaticInvoice { request_path: BlindedMessagePath, } -impl_writeable_tlv_based!(PersistedStaticInvoice, { +impl_ser_tlv_based!(PersistedStaticInvoice, { (0, invoice, required), (2, request_path, required) }); @@ -78,12 +78,13 @@ impl StaticInvoiceStore { let (secondary_namespace, key) = Self::get_storage_location(invoice_slot, recipient_id); - KVStoreSync::read( + KVStore::read( &*self.kv_store, STATIC_INVOICE_STORE_PRIMARY_NAMESPACE, &secondary_namespace, &key, ) + .await .and_then(|data| { PersistedStaticInvoice::read(&mut &*data) .map(|persisted_invoice| { @@ -124,13 +125,14 @@ impl StaticInvoiceStore { // Static invoices will be persisted at "static_invoices//". // // Example: static_invoices/039058c6f2c0cb492c533b0a4d14ef77cc0f78abccced5287d84a1a2011cfb81/00001 - KVStoreSync::write( + KVStore::write( &*self.kv_store, STATIC_INVOICE_STORE_PRIMARY_NAMESPACE, &secondary_namespace, &key, buf, ) + .await } fn get_storage_location(invoice_slot: u16, recipient_id: &[u8]) -> (String, String) { diff --git a/src/payment/bolt11.rs b/src/payment/bolt11.rs index e81aa51f7b..97dd7a3493 100644 --- a/src/payment/bolt11.rs +++ b/src/payment/bolt11.rs @@ -13,7 +13,7 @@ use std::sync::{Arc, RwLock}; use bitcoin::hashes::sha256::Hash as Sha256; use bitcoin::hashes::Hash; -use lightning::impl_writeable_tlv_based; +use lightning::impl_ser_tlv_based; use lightning::ln::channelmanager::{ Bolt11InvoiceParameters, OptionalBolt11PaymentParams, PaymentId, }; @@ -55,7 +55,7 @@ pub(crate) struct PaymentMetadata { pub(crate) lsps2_parameters: Option, } -impl_writeable_tlv_based!(PaymentMetadata, { +impl_ser_tlv_based!(PaymentMetadata, { (0, lsps2_parameters, option), }); @@ -158,7 +158,7 @@ impl Bolt11Payment { PaymentDirection::Inbound, PaymentStatus::Pending, ); - self.payment_store.insert(payment)?; + self.runtime.block_on(self.payment_store.insert(payment))?; Ok(invoice) } @@ -239,10 +239,10 @@ impl Bolt11Payment { PaymentDirection::Inbound, PaymentStatus::Pending, ); - self.payment_store.insert(payment)?; + self.runtime.block_on(self.payment_store.insert(payment))?; // Persist LSP peer to make sure we reconnect on restart. - self.peer_store.add_peer(peer_info)?; + self.runtime.block_on(self.peer_store.add_peer(peer_info))?; Ok(invoice) } @@ -341,7 +341,7 @@ impl Bolt11Payment { PaymentStatus::Pending, ); - self.payment_store.insert(payment)?; + self.runtime.block_on(self.payment_store.insert(payment))?; Ok(payment_id) }, @@ -371,7 +371,7 @@ impl Bolt11Payment { PaymentStatus::Failed, ); - self.payment_store.insert(payment)?; + self.runtime.block_on(self.payment_store.insert(payment))?; Err(Error::PaymentSendingFailed) }, } @@ -457,7 +457,7 @@ impl Bolt11Payment { PaymentDirection::Outbound, PaymentStatus::Pending, ); - self.payment_store.insert(payment)?; + self.runtime.block_on(self.payment_store.insert(payment))?; Ok(payment_id) }, @@ -488,7 +488,7 @@ impl Bolt11Payment { PaymentStatus::Failed, ); - self.payment_store.insert(payment)?; + self.runtime.block_on(self.payment_store.insert(payment))?; Err(Error::PaymentSendingFailed) }, } @@ -582,7 +582,7 @@ impl Bolt11Payment { ..PaymentDetailsUpdate::new(payment_id) }; - match self.payment_store.update(update) { + match self.runtime.block_on(self.payment_store.update(update)) { Ok(DataStoreUpdateResult::Updated) | Ok(DataStoreUpdateResult::Unchanged) => (), Ok(DataStoreUpdateResult::NotFound) => { log_error!( diff --git a/src/payment/bolt12.rs b/src/payment/bolt12.rs index 2e5a5fb451..d79aca6c24 100644 --- a/src/payment/bolt12.rs +++ b/src/payment/bolt12.rs @@ -29,6 +29,7 @@ use crate::error::Error; use crate::ffi::{maybe_deref, maybe_wrap}; use crate::logger::{log_error, log_info, LdkLogger, Logger}; use crate::payment::store::{PaymentDetails, PaymentDirection, PaymentKind, PaymentStatus}; +use crate::runtime::Runtime; use crate::types::{ChannelManager, KeysManager, PaymentStore}; #[cfg(not(feature = "uniffi"))] @@ -59,6 +60,7 @@ type HumanReadableName = Arc; /// [`Node::bolt12_payment`]: crate::Node::bolt12_payment #[cfg_attr(feature = "uniffi", derive(uniffi::Object))] pub struct Bolt12Payment { + runtime: Arc, channel_manager: Arc, keys_manager: Arc, payment_store: Arc, @@ -70,11 +72,13 @@ pub struct Bolt12Payment { impl Bolt12Payment { pub(crate) fn new( - channel_manager: Arc, keys_manager: Arc, - payment_store: Arc, config: Arc, is_running: Arc>, - logger: Arc, async_payments_role: Option, + runtime: Arc, channel_manager: Arc, + keys_manager: Arc, payment_store: Arc, config: Arc, + is_running: Arc>, logger: Arc, + async_payments_role: Option, ) -> Self { Self { + runtime, channel_manager, keys_manager, payment_store, @@ -163,7 +167,7 @@ impl Bolt12Payment { PaymentDirection::Outbound, PaymentStatus::Pending, ); - self.payment_store.insert(payment)?; + self.runtime.block_on(self.payment_store.insert(payment))?; Ok(payment_id) }, @@ -188,7 +192,7 @@ impl Bolt12Payment { PaymentDirection::Outbound, PaymentStatus::Failed, ); - self.payment_store.insert(payment)?; + self.runtime.block_on(self.payment_store.insert(payment))?; Err(Error::PaymentSendingFailed) }, } @@ -325,7 +329,7 @@ impl Bolt12Payment { PaymentDirection::Outbound, PaymentStatus::Pending, ); - self.payment_store.insert(payment)?; + self.runtime.block_on(self.payment_store.insert(payment))?; Ok(payment_id) }, @@ -350,7 +354,7 @@ impl Bolt12Payment { PaymentDirection::Outbound, PaymentStatus::Failed, ); - self.payment_store.insert(payment)?; + self.runtime.block_on(self.payment_store.insert(payment))?; Err(Error::InvoiceRequestCreationFailed) }, } @@ -457,7 +461,7 @@ impl Bolt12Payment { PaymentStatus::Pending, ); - self.payment_store.insert(payment)?; + self.runtime.block_on(self.payment_store.insert(payment))?; Ok(maybe_wrap(invoice)) } @@ -526,7 +530,7 @@ impl Bolt12Payment { PaymentStatus::Pending, ); - self.payment_store.insert(payment)?; + self.runtime.block_on(self.payment_store.insert(payment))?; Ok(maybe_wrap(refund)) } diff --git a/src/payment/pending_payment_store.rs b/src/payment/pending_payment_store.rs index eb72f89ec9..37a3b09347 100644 --- a/src/payment/pending_payment_store.rs +++ b/src/payment/pending_payment_store.rs @@ -6,7 +6,7 @@ // accordance with one or both of these licenses. use bitcoin::Txid; -use lightning::impl_writeable_tlv_based; +use lightning::impl_ser_tlv_based; use lightning::ln::channelmanager::PaymentId; use crate::data_store::{StorableObject, StorableObjectUpdate}; @@ -33,7 +33,7 @@ impl PendingPaymentDetails { } } -impl_writeable_tlv_based!(PendingPaymentDetails, { +impl_ser_tlv_based!(PendingPaymentDetails, { (0, details, required), (2, conflicting_txids, optional_vec), }); diff --git a/src/payment/spontaneous.rs b/src/payment/spontaneous.rs index 1c819582e4..45dab644d4 100644 --- a/src/payment/spontaneous.rs +++ b/src/payment/spontaneous.rs @@ -22,6 +22,7 @@ use crate::config::{Config, LDK_PAYMENT_RETRY_TIMEOUT}; use crate::error::Error; use crate::logger::{log_error, log_info, LdkLogger, Logger}; use crate::payment::store::{PaymentDetails, PaymentDirection, PaymentKind, PaymentStatus}; +use crate::runtime::Runtime; use crate::types::{ChannelManager, CustomTlvRecord, KeysManager, PaymentStore}; // The default `final_cltv_expiry_delta` we apply when not set. @@ -34,6 +35,7 @@ const LDK_DEFAULT_FINAL_CLTV_EXPIRY_DELTA: u32 = 144; /// [`Node::spontaneous_payment`]: crate::Node::spontaneous_payment #[cfg_attr(feature = "uniffi", derive(uniffi::Object))] pub struct SpontaneousPayment { + runtime: Arc, channel_manager: Arc, keys_manager: Arc, payment_store: Arc, @@ -44,11 +46,11 @@ pub struct SpontaneousPayment { impl SpontaneousPayment { pub(crate) fn new( - channel_manager: Arc, keys_manager: Arc, - payment_store: Arc, config: Arc, is_running: Arc>, - logger: Arc, + runtime: Arc, channel_manager: Arc, + keys_manager: Arc, payment_store: Arc, config: Arc, + is_running: Arc>, logger: Arc, ) -> Self { - Self { channel_manager, keys_manager, payment_store, config, is_running, logger } + Self { runtime, channel_manager, keys_manager, payment_store, config, is_running, logger } } fn send_inner( @@ -130,7 +132,7 @@ impl SpontaneousPayment { PaymentDirection::Outbound, PaymentStatus::Pending, ); - self.payment_store.insert(payment)?; + self.runtime.block_on(self.payment_store.insert(payment))?; Ok(payment_id) }, @@ -153,7 +155,7 @@ impl SpontaneousPayment { PaymentStatus::Failed, ); - self.payment_store.insert(payment)?; + self.runtime.block_on(self.payment_store.insert(payment))?; Err(Error::PaymentSendingFailed) }, } diff --git a/src/payment/store.rs b/src/payment/store.rs index f80ab6f8a5..db4ba06ed8 100644 --- a/src/payment/store.rs +++ b/src/payment/store.rs @@ -13,8 +13,8 @@ use lightning::ln::msgs::DecodeError; use lightning::offers::offer::OfferId; use lightning::util::ser::{Readable, Writeable}; use lightning::{ - _init_and_read_len_prefixed_tlv_fields, impl_writeable_tlv_based, - impl_writeable_tlv_based_enum, write_tlv_fields, + _init_and_read_len_prefixed_tlv_fields, impl_ser_tlv_based, impl_ser_tlv_based_enum, + write_tlv_fields, }; use lightning_types::payment::{PaymentHash, PaymentPreimage, PaymentSecret}; use lightning_types::string::UntrustedString; @@ -307,7 +307,7 @@ pub enum PaymentDirection { Outbound, } -impl_writeable_tlv_based_enum!(PaymentDirection, +impl_ser_tlv_based_enum!(PaymentDirection, (0, Inbound) => {}, (1, Outbound) => {} ); @@ -324,7 +324,7 @@ pub enum PaymentStatus { Failed, } -impl_writeable_tlv_based_enum!(PaymentStatus, +impl_ser_tlv_based_enum!(PaymentStatus, (0, Pending) => {}, (2, Succeeded) => {}, (4, Failed) => {} @@ -420,7 +420,7 @@ pub enum PaymentKind { }, } -impl_writeable_tlv_based_enum!(PaymentKind, +impl_ser_tlv_based_enum!(PaymentKind, (0, Onchain) => { (0, txid, required), (2, status, required), @@ -479,7 +479,7 @@ pub enum ConfirmationStatus { Unconfirmed, } -impl_writeable_tlv_based_enum!(ConfirmationStatus, +impl_ser_tlv_based_enum!(ConfirmationStatus, (0, Confirmed) => { (0, block_hash, required), (2, height, required), @@ -504,7 +504,7 @@ pub struct LSPS2Parameters { pub max_proportional_opening_fee_ppm_msat: Option, } -impl_writeable_tlv_based!(LSPS2Parameters, { +impl_ser_tlv_based!(LSPS2Parameters, { (0, max_total_opening_fee_msat, option), (2, max_proportional_opening_fee_ppm_msat, option), }); @@ -604,7 +604,7 @@ mod tests { pub status: PaymentStatus, } - impl_writeable_tlv_based!(OldPaymentDetails, { + impl_ser_tlv_based!(OldPaymentDetails, { (0, hash, required), (2, preimage, required), (4, secret, required), @@ -706,7 +706,7 @@ mod tests { lsp_fee_limits: LSPS2Parameters, } - impl_writeable_tlv_based!(LegacyBolt11JitKind, { + impl_ser_tlv_based!(LegacyBolt11JitKind, { (0, hash, required), (1, counterparty_skimmed_fee_msat, option), (2, preimage, option), diff --git a/src/peer_store.rs b/src/peer_store.rs index 307fb69296..19993d3f7d 100644 --- a/src/peer_store.rs +++ b/src/peer_store.rs @@ -10,8 +10,8 @@ use std::ops::Deref; use std::sync::{Arc, RwLock}; use bitcoin::secp256k1::PublicKey; -use lightning::impl_writeable_tlv_based; -use lightning::util::persist::KVStoreSync; +use lightning::impl_ser_tlv_based; +use lightning::util::persist::KVStore; use lightning::util::ser::{Readable, ReadableArgs, Writeable, Writer}; use crate::io::{ @@ -27,6 +27,7 @@ where L::Target: LdkLogger, { peers: RwLock>, + mutation_lock: tokio::sync::Mutex<()>, kv_store: Arc, logger: L, } @@ -37,25 +38,31 @@ where { pub(crate) fn new(kv_store: Arc, logger: L) -> Self { let peers = RwLock::new(HashMap::new()); - Self { peers, kv_store, logger } + let mutation_lock = tokio::sync::Mutex::new(()); + Self { peers, mutation_lock, kv_store, logger } } - pub(crate) fn add_peer(&self, peer_info: PeerInfo) -> Result<(), Error> { - let mut locked_peers = self.peers.write().expect("lock"); - - if locked_peers.contains_key(&peer_info.node_id) { - return Ok(()); - } - - locked_peers.insert(peer_info.node_id, peer_info); - self.persist_peers(&*locked_peers) + pub(crate) async fn add_peer(&self, peer_info: PeerInfo) -> Result<(), Error> { + let _guard = self.mutation_lock.lock().await; + let data = { + let mut locked_peers = self.peers.write().expect("lock"); + if locked_peers.contains_key(&peer_info.node_id) { + return Ok(()); + } + locked_peers.insert(peer_info.node_id, peer_info); + PeerStoreSerWrapper(&locked_peers).encode() + }; + self.persist_peers(data).await } - pub(crate) fn remove_peer(&self, node_id: &PublicKey) -> Result<(), Error> { - let mut locked_peers = self.peers.write().expect("lock"); - - locked_peers.remove(node_id); - self.persist_peers(&*locked_peers) + pub(crate) async fn remove_peer(&self, node_id: &PublicKey) -> Result<(), Error> { + let _guard = self.mutation_lock.lock().await; + let data = { + let mut locked_peers = self.peers.write().expect("lock"); + locked_peers.remove(node_id); + PeerStoreSerWrapper(&locked_peers).encode() + }; + self.persist_peers(data).await } pub(crate) fn list_peers(&self) -> Vec { @@ -66,15 +73,15 @@ where self.peers.read().expect("lock").get(node_id).cloned() } - fn persist_peers(&self, locked_peers: &HashMap) -> Result<(), Error> { - let data = PeerStoreSerWrapper(&*locked_peers).encode(); - KVStoreSync::write( + async fn persist_peers(&self, data: Vec) -> Result<(), Error> { + KVStore::write( &*self.kv_store, PEER_INFO_PERSISTENCE_PRIMARY_NAMESPACE, PEER_INFO_PERSISTENCE_SECONDARY_NAMESPACE, PEER_INFO_PERSISTENCE_KEY, data, ) + .await .map_err(|e| { log_error!( self.logger, @@ -101,7 +108,8 @@ where let (kv_store, logger) = args; let read_peers: PeerStoreDeserWrapper = Readable::read(reader)?; let peers: RwLock> = RwLock::new(read_peers.0); - Ok(Self { peers, kv_store, logger }) + let mutation_lock = tokio::sync::Mutex::new(()); + Ok(Self { peers, mutation_lock, kv_store, logger }) } } @@ -142,7 +150,7 @@ pub(crate) struct PeerInfo { pub address: SocketAddress, } -impl_writeable_tlv_based!(PeerInfo, { +impl_ser_tlv_based!(PeerInfo, { (0, node_id, required), (2, address, required), }); @@ -158,8 +166,8 @@ mod tests { use crate::io::test_utils::InMemoryStore; use crate::types::DynStoreWrapper; - #[test] - fn peer_info_persistence() { + #[tokio::test] + async fn peer_info_persistence() { let store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); let logger = Arc::new(TestLogger::new()); let peer_store = PeerStore::new(Arc::clone(&store), Arc::clone(&logger)); @@ -170,22 +178,24 @@ mod tests { .unwrap(); let address = SocketAddress::from_str("127.0.0.1:9738").unwrap(); let expected_peer_info = PeerInfo { node_id, address }; - assert!(KVStoreSync::read( + assert!(KVStore::read( &*store, PEER_INFO_PERSISTENCE_PRIMARY_NAMESPACE, PEER_INFO_PERSISTENCE_SECONDARY_NAMESPACE, PEER_INFO_PERSISTENCE_KEY, ) + .await .is_err()); - peer_store.add_peer(expected_peer_info.clone()).unwrap(); + peer_store.add_peer(expected_peer_info.clone()).await.unwrap(); // Check we can read back what we persisted. - let persisted_bytes = KVStoreSync::read( + let persisted_bytes = KVStore::read( &*store, PEER_INFO_PERSISTENCE_PRIMARY_NAMESPACE, PEER_INFO_PERSISTENCE_SECONDARY_NAMESPACE, PEER_INFO_PERSISTENCE_KEY, ) + .await .unwrap(); let deser_peer_store = PeerStore::read(&mut &persisted_bytes[..], (Arc::clone(&store), logger)).unwrap(); diff --git a/src/runtime.rs b/src/runtime.rs index 1d8eb32b0a..f2fa0c982d 100644 --- a/src/runtime.rs +++ b/src/runtime.rs @@ -6,6 +6,8 @@ // accordance with one or both of these licenses. use std::future::Future; +use std::io; +use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; use std::time::Duration; @@ -30,7 +32,11 @@ impl Runtime { let mode = match tokio::runtime::Handle::try_current() { Ok(handle) => RuntimeMode::Handle(handle), Err(_) => { - let rt = tokio::runtime::Builder::new_multi_thread().enable_all().build()?; + let mut runtime_builder = tokio::runtime::Builder::new_multi_thread(); + runtime_builder.enable_all(); + #[cfg(tokio_unstable)] + runtime_builder.enable_eager_driver_handoff(); + let rt = runtime_builder.build()?; RuntimeMode::Owned(rt) }, }; @@ -223,6 +229,60 @@ enum RuntimeMode { Handle(tokio::runtime::Handle), } +pub(crate) struct StoreRuntime { + runtime: Option, +} + +impl StoreRuntime { + pub(crate) fn new( + thread_name_prefix: &'static str, worker_threads: usize, runtime_name: &'static str, + ) -> io::Result { + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .thread_name_fn(move || { + static ATOMIC_ID: AtomicUsize = AtomicUsize::new(0); + let id = ATOMIC_ID.fetch_add(1, Ordering::SeqCst); + format!("{}-{}", thread_name_prefix, id) + }) + .worker_threads(worker_threads) + .max_blocking_threads(worker_threads) + .build() + .map_err(|e| { + io::Error::new( + io::ErrorKind::Other, + format!("Failed to build {runtime_name} runtime: {e}"), + ) + })?; + Ok(Self { runtime: Some(runtime) }) + } + + pub(crate) fn handle(&self) -> &tokio::runtime::Handle { + self.runtime.as_ref().expect("store runtime must be available").handle() + } + + pub(crate) fn spawn(&self, future: F) -> JoinHandle + where + F: Future + Send + 'static, + F::Output: Send + 'static, + { + self.handle().spawn(future) + } + + pub(crate) fn shutdown_background(mut self) { + if let Some(runtime) = self.runtime.take() { + runtime.shutdown_background(); + } + } +} + +impl Drop for StoreRuntime { + fn drop(&mut self) { + if let Some(runtime) = self.runtime.take() { + runtime.shutdown_background(); + } + } +} + pub(crate) struct RuntimeSpawner { runtime: Arc, } diff --git a/src/scoring.rs b/src/scoring.rs index 8abc4eab6a..401d9c3f1f 100644 --- a/src/scoring.rs +++ b/src/scoring.rs @@ -1,4 +1,4 @@ -use std::sync::{Arc, Mutex, RwLock}; +use std::sync::{Arc, Mutex}; use std::time::SystemTime; use lightning::routing::scoring::ChannelLiquidities; @@ -13,12 +13,12 @@ use crate::io::utils::write_external_pathfinding_scores_to_cache; use crate::logger::LdkLogger; use crate::runtime::Runtime; use crate::types::DynStore; -use crate::{update_and_persist_node_metrics, Logger, NodeMetrics, Scorer}; +use crate::{update_and_persist_node_metrics, Logger, PersistedNodeMetrics, Scorer}; /// Start a background task that periodically downloads scores via an external url and merges them into the local /// pathfinding scores. pub fn setup_background_pathfinding_scores_sync( - url: String, scorer: Arc>, node_metrics: Arc>, + url: String, scorer: Arc>, node_metrics: Arc, kv_store: Arc, logger: Arc, runtime: Arc, mut stop_receiver: tokio::sync::watch::Receiver<()>, ) { @@ -51,7 +51,7 @@ pub fn setup_background_pathfinding_scores_sync( } async fn sync_external_scores( - logger: &Logger, scorer: &Mutex, node_metrics: &RwLock, + logger: &Logger, scorer: &Mutex, node_metrics: &PersistedNodeMetrics, kv_store: Arc, url: &String, ) -> () { let request = bitreq::get(url) @@ -86,9 +86,10 @@ async fn sync_external_scores( .duration_since(SystemTime::UNIX_EPOCH) .expect("system time must be after Unix epoch"); scorer.lock().expect("lock").merge(liquidities, duration_since_epoch); - update_and_persist_node_metrics(&node_metrics, &*kv_store, logger, |m| { + update_and_persist_node_metrics(node_metrics, &*kv_store, logger, |m| { m.latest_pathfinding_scores_sync_timestamp = Some(duration_since_epoch.as_secs()); }) + .await .unwrap_or_else(|e| { log_error!(logger, "Persisting node metrics failed: {}", e); }); diff --git a/src/types.rs b/src/types.rs index 06e65fbd0a..a8ce812b8e 100644 --- a/src/types.rs +++ b/src/types.rs @@ -19,7 +19,7 @@ use bitcoin_payment_instructions::hrn_resolution::{ }; use bitcoin_payment_instructions::onion_message_resolver::LDKOnionMessageDNSSECHrnResolver; use lightning::chain::chainmonitor; -use lightning::impl_writeable_tlv_based; +use lightning::impl_ser_tlv_based; use lightning::ln::channel_state::{ChannelDetails as LdkChannelDetails, ChannelShutdownState}; use lightning::ln::msgs::{RoutingMessageHandler, SocketAddress}; use lightning::ln::peer_handler::IgnoringMessageHandler; @@ -29,7 +29,7 @@ use lightning::routing::gossip; use lightning::routing::router::DefaultRouter; use lightning::routing::scoring::{CombinedScorer, ProbabilisticScoringFeeParameters}; use lightning::sign::InMemorySigner; -use lightning::util::persist::{KVStore, KVStoreSync, MonitorUpdatingPersisterAsync}; +use lightning::util::persist::{KVStore, MonitorUpdatingPersisterAsync}; use lightning::util::ser::{Readable, Writeable, Writer}; use lightning::util::sweep::OutputSweeper; use lightning_block_sync::gossip::GossipVerifier; @@ -46,17 +46,6 @@ use crate::message_handler::NodeCustomMessageHandler; use crate::payment::{PaymentDetails, PendingPaymentDetails}; use crate::runtime::RuntimeSpawner; -/// A supertrait that requires that a type implements both [`KVStore`] and [`KVStoreSync`] at the -/// same time. -pub trait SyncAndAsyncKVStore: KVStore + KVStoreSync {} - -impl SyncAndAsyncKVStore for T -where - T: KVStore, - T: KVStoreSync, -{ -} - pub(crate) trait DynStoreTrait: Send + Sync { fn read_async( &self, primary_namespace: &str, secondary_namespace: &str, key: &str, @@ -70,19 +59,6 @@ pub(crate) trait DynStoreTrait: Send + Sync { fn list_async( &self, primary_namespace: &str, secondary_namespace: &str, ) -> Pin, bitcoin::io::Error>> + Send + 'static>>; - - fn read( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, - ) -> Result, bitcoin::io::Error>; - fn write( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, - ) -> Result<(), bitcoin::io::Error>; - fn remove( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, - ) -> Result<(), bitcoin::io::Error>; - fn list( - &self, primary_namespace: &str, secondary_namespace: &str, - ) -> Result, bitcoin::io::Error>; } impl<'a> KVStore for dyn DynStoreTrait + 'a { @@ -111,32 +87,6 @@ impl<'a> KVStore for dyn DynStoreTrait + 'a { } } -impl<'a> KVStoreSync for dyn DynStoreTrait + 'a { - fn read( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, - ) -> Result, bitcoin::io::Error> { - DynStoreTrait::read(self, primary_namespace, secondary_namespace, key) - } - - fn write( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, - ) -> Result<(), bitcoin::io::Error> { - DynStoreTrait::write(self, primary_namespace, secondary_namespace, key, buf) - } - - fn remove( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, - ) -> Result<(), bitcoin::io::Error> { - DynStoreTrait::remove(self, primary_namespace, secondary_namespace, key, lazy) - } - - fn list( - &self, primary_namespace: &str, secondary_namespace: &str, - ) -> Result, bitcoin::io::Error> { - DynStoreTrait::list(self, primary_namespace, secondary_namespace) - } -} - pub(crate) type DynStore = dyn DynStoreTrait; // Newtype wrapper that implements `KVStore` for `Arc`. This is needed because `KVStore` @@ -172,9 +122,9 @@ impl KVStore for DynStoreRef { } } -pub(crate) struct DynStoreWrapper(pub(crate) T); +pub(crate) struct DynStoreWrapper(pub(crate) T); -impl DynStoreTrait for DynStoreWrapper { +impl DynStoreTrait for DynStoreWrapper { fn read_async( &self, primary_namespace: &str, secondary_namespace: &str, key: &str, ) -> Pin, bitcoin::io::Error>> + Send + 'static>> { @@ -198,30 +148,6 @@ impl DynStoreTrait for DynStoreWrapper ) -> Pin, bitcoin::io::Error>> + Send + 'static>> { Box::pin(KVStore::list(&self.0, primary_namespace, secondary_namespace)) } - - fn read( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, - ) -> Result, bitcoin::io::Error> { - KVStoreSync::read(&self.0, primary_namespace, secondary_namespace, key) - } - - fn write( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, - ) -> Result<(), bitcoin::io::Error> { - KVStoreSync::write(&self.0, primary_namespace, secondary_namespace, key, buf) - } - - fn remove( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, - ) -> Result<(), bitcoin::io::Error> { - KVStoreSync::remove(&self.0, primary_namespace, secondary_namespace, key, lazy) - } - - fn list( - &self, primary_namespace: &str, secondary_namespace: &str, - ) -> Result, bitcoin::io::Error> { - KVStoreSync::list(&self.0, primary_namespace, secondary_namespace) - } } pub(crate) type AsyncPersister = MonitorUpdatingPersisterAsync< @@ -690,7 +616,7 @@ pub struct CustomTlvRecord { pub value: Vec, } -impl_writeable_tlv_based!(CustomTlvRecord, { +impl_ser_tlv_based!(CustomTlvRecord, { (0, type_num, required), (2, value, required), }); diff --git a/src/wallet/mod.rs b/src/wallet/mod.rs index 13b1f384f5..76f2aa9ce6 100644 --- a/src/wallet/mod.rs +++ b/src/wallet/mod.rs @@ -58,6 +58,7 @@ use crate::payment::store::ConfirmationStatus; use crate::payment::{ PaymentDetails, PaymentDirection, PaymentKind, PaymentStatus, PendingPaymentDetails, }; +use crate::runtime::Runtime; use crate::types::{Broadcaster, PaymentStore, PendingPaymentStore}; use crate::{ChainSource, Error}; @@ -85,6 +86,7 @@ pub(crate) struct Wallet { fee_estimator: Arc, chain_source: Arc, payment_store: Arc, + runtime: Arc, config: Arc, logger: Arc, pending_payment_store: Arc, @@ -95,8 +97,8 @@ impl Wallet { wallet: bdk_wallet::PersistedWallet, wallet_persister: KVStoreWalletPersister, broadcaster: Arc, fee_estimator: Arc, chain_source: Arc, - payment_store: Arc, config: Arc, logger: Arc, - pending_payment_store: Arc, + payment_store: Arc, runtime: Arc, config: Arc, + logger: Arc, pending_payment_store: Arc, ) -> Self { let inner = Mutex::new(wallet); let persister = Mutex::new(wallet_persister); @@ -107,6 +109,7 @@ impl Wallet { fee_estimator, chain_source, payment_store, + runtime, config, logger, pending_payment_store, @@ -158,10 +161,12 @@ impl Wallet { })?; let mut locked_persister = self.persister.lock().expect("lock"); - locked_wallet.persist(&mut locked_persister).map_err(|e| { - log_error!(self.logger, "Failed to persist wallet: {}", e); - Error::PersistenceFailed - })?; + self.runtime.block_on(locked_wallet.persist_async(&mut locked_persister)).map_err( + |e| { + log_error!(self.logger, "Failed to persist wallet: {}", e); + Error::PersistenceFailed + }, + )?; Ok(()) }, @@ -211,7 +216,7 @@ impl Wallet { })?; let mut locked_persister = self.persister.lock().expect("lock"); - locked_wallet.persist(&mut locked_persister).map_err(|e| { + self.runtime.block_on(locked_wallet.persist_async(&mut locked_persister)).map_err(|e| { log_error!(self.logger, "Failed to persist wallet: {}", e); Error::PersistenceFailed })?; @@ -278,13 +283,15 @@ impl Wallet { confirmation_status, ); - self.payment_store.insert_or_update(payment.clone())?; + self.runtime.block_on(self.payment_store.insert_or_update(payment.clone()))?; if payment_status == PaymentStatus::Pending { let pending_payment = self.create_pending_payment_from_tx(payment, Vec::new()); - self.pending_payment_store.insert_or_update(pending_payment)?; + self.runtime.block_on( + self.pending_payment_store.insert_or_update(pending_payment), + )?; } }, WalletEvent::ChainTipChanged { new_tip, .. } => { @@ -310,8 +317,11 @@ impl Wallet { let payment_id = payment.details.id; if new_tip.height >= height + ANTI_REORG_DELAY - 1 { payment.details.status = PaymentStatus::Succeeded; - self.payment_store.insert_or_update(payment.details)?; - self.pending_payment_store.remove(&payment_id)?; + self.runtime.block_on( + self.payment_store.insert_or_update(payment.details), + )?; + self.runtime + .block_on(self.pending_payment_store.remove(&payment_id))?; } }, PaymentKind::Onchain { @@ -367,8 +377,9 @@ impl Wallet { ); let pending_payment = self.create_pending_payment_from_tx(payment.clone(), Vec::new()); - self.payment_store.insert_or_update(payment)?; - self.pending_payment_store.insert_or_update(pending_payment)?; + self.runtime.block_on(self.payment_store.insert_or_update(payment))?; + self.runtime + .block_on(self.pending_payment_store.insert_or_update(pending_payment))?; }, WalletEvent::TxReplaced { txid, conflicts, .. } => { let Some(payment_id) = self.find_payment_by_txid(txid) else { @@ -398,7 +409,9 @@ impl Wallet { let pending_payment_details = self.create_pending_payment_from_tx(payment, conflict_txids.clone()); - self.pending_payment_store.insert_or_update(pending_payment_details)?; + self.runtime.block_on( + self.pending_payment_store.insert_or_update(pending_payment_details), + )?; }, WalletEvent::TxDropped { txid, tx } => { let payment_id = self @@ -414,8 +427,9 @@ impl Wallet { ); let pending_payment = self.create_pending_payment_from_tx(payment.clone(), Vec::new()); - self.payment_store.insert_or_update(payment)?; - self.pending_payment_store.insert_or_update(pending_payment)?; + self.runtime.block_on(self.payment_store.insert_or_update(payment))?; + self.runtime + .block_on(self.pending_payment_store.insert_or_update(pending_payment))?; }, _ => { continue; @@ -462,7 +476,7 @@ impl Wallet { } let mut locked_persister = self.persister.lock().expect("lock"); - locked_wallet.persist(&mut locked_persister).map_err(|e| { + self.runtime.block_on(locked_wallet.persist_async(&mut locked_persister)).map_err(|e| { log_error!(self.logger, "Failed to persist wallet: {}", e); Error::PersistenceFailed })?; @@ -480,7 +494,7 @@ impl Wallet { let mut locked_persister = self.persister.lock().expect("lock"); let address_info = locked_wallet.reveal_next_address(KeychainKind::External); - locked_wallet.persist(&mut locked_persister).map_err(|e| { + self.runtime.block_on(locked_wallet.persist_async(&mut locked_persister)).map_err(|e| { log_error!(self.logger, "Failed to persist wallet: {}", e); Error::PersistenceFailed })?; @@ -492,7 +506,7 @@ impl Wallet { let mut locked_persister = self.persister.lock().expect("lock"); let address_info = locked_wallet.next_unused_address(KeychainKind::Internal); - locked_wallet.persist(&mut locked_persister).map_err(|e| { + self.runtime.block_on(locked_wallet.persist_async(&mut locked_persister)).map_err(|e| { log_error!(self.logger, "Failed to persist wallet: {}", e); Error::PersistenceFailed })?; @@ -504,7 +518,7 @@ impl Wallet { let mut locked_persister = self.persister.lock().expect("lock"); locked_wallet.cancel_tx(tx); - locked_wallet.persist(&mut locked_persister).map_err(|e| { + self.runtime.block_on(locked_wallet.persist_async(&mut locked_persister)).map_err(|e| { log_error!(self.logger, "Failed to persist wallet: {}", e); Error::PersistenceFailed })?; @@ -842,10 +856,12 @@ impl Wallet { } let mut locked_persister = self.persister.lock().expect("lock"); - locked_wallet.persist(&mut locked_persister).map_err(|e| { - log_error!(self.logger, "Failed to persist wallet: {}", e); - Error::PersistenceFailed - })?; + self.runtime.block_on(locked_wallet.persist_async(&mut locked_persister)).map_err( + |e| { + log_error!(self.logger, "Failed to persist wallet: {}", e); + Error::PersistenceFailed + }, + )?; psbt.extract_tx().map_err(|e| { log_error!(self.logger, "Failed to extract transaction: {}", e); @@ -960,10 +976,12 @@ impl Wallet { .find(|txout| must_pay_to.iter().all(|output| output != txout)); if change_output.is_some() { - locked_wallet.persist(&mut locked_persister).map_err(|e| { - log_error!(self.logger, "Failed to persist wallet: {}", e); - () - })?; + self.runtime.block_on(locked_wallet.persist_async(&mut locked_persister)).map_err( + |e| { + log_error!(self.logger, "Failed to persist wallet: {}", e); + () + }, + )?; } Ok(CoinSelection { confirmed_utxos, change_output }) @@ -1068,7 +1086,7 @@ impl Wallet { let mut locked_persister = self.persister.lock().expect("lock"); let address_info = locked_wallet.next_unused_address(KeychainKind::Internal); - locked_wallet.persist(&mut locked_persister).map_err(|e| { + self.runtime.block_on(locked_wallet.persist_async(&mut locked_persister)).map_err(|e| { log_error!(self.logger, "Failed to persist wallet: {}", e); () })?; @@ -1387,7 +1405,7 @@ impl Wallet { } let mut locked_persister = self.persister.lock().expect("lock"); - locked_wallet.persist(&mut locked_persister).map_err(|e| { + self.runtime.block_on(locked_wallet.persist_async(&mut locked_persister)).map_err(|e| { log_error!(self.logger, "Failed to persist wallet after fee bump of {}: {}", txid, e); Error::PersistenceFailed })?; @@ -1416,8 +1434,9 @@ impl Wallet { let pending_payment_store = self.create_pending_payment_from_tx(new_payment.clone(), Vec::new()); - self.pending_payment_store.insert_or_update(pending_payment_store)?; - self.payment_store.insert_or_update(new_payment)?; + self.runtime + .block_on(self.pending_payment_store.insert_or_update(pending_payment_store))?; + self.runtime.block_on(self.payment_store.insert_or_update(new_payment))?; log_info!(self.logger, "RBF successful: replaced {} with {}", txid, new_txid); @@ -1488,7 +1507,7 @@ impl Listen for Wallet { }; let mut locked_persister = self.persister.lock().expect("lock"); - match locked_wallet.persist(&mut locked_persister) { + match self.runtime.block_on(locked_wallet.persist_async(&mut locked_persister)) { Ok(_) => (), Err(e) => { log_error!(self.logger, "Failed to persist on-chain wallet: {}", e); diff --git a/src/wallet/persist.rs b/src/wallet/persist.rs index 10be1fac08..364dc4b475 100644 --- a/src/wallet/persist.rs +++ b/src/wallet/persist.rs @@ -5,10 +5,12 @@ // http://opensource.org/licenses/MIT>, at your option. You may not use this file except in // accordance with one or both of these licenses. +use std::future::Future; +use std::pin::Pin; use std::sync::Arc; use bdk_chain::Merge; -use bdk_wallet::{ChangeSet, WalletPersister}; +use bdk_wallet::{AsyncWalletPersister, ChangeSet}; use crate::io::utils::{ read_bdk_wallet_change_set, write_bdk_wallet_change_descriptor, write_bdk_wallet_descriptor, @@ -17,6 +19,7 @@ use crate::io::utils::{ }; use crate::logger::{log_error, LdkLogger, Logger}; use crate::types::DynStore; + pub(crate) struct KVStoreWalletPersister { latest_change_set: Option, kv_store: Arc, @@ -27,18 +30,14 @@ impl KVStoreWalletPersister { pub(crate) fn new(kv_store: Arc, logger: Arc) -> Self { Self { latest_change_set: None, kv_store, logger } } -} - -impl WalletPersister for KVStoreWalletPersister { - type Error = std::io::Error; - fn initialize(persister: &mut Self) -> Result { + async fn initialize_inner(&mut self) -> Result { // Return immediately if we have already been initialized. - if let Some(latest_change_set) = persister.latest_change_set.as_ref() { + if let Some(latest_change_set) = self.latest_change_set.as_ref() { return Ok(latest_change_set.clone()); } - let change_set_opt = read_bdk_wallet_change_set(&*persister.kv_store, &*persister.logger)?; + let change_set_opt = read_bdk_wallet_change_set(&*self.kv_store, &*self.logger).await?; let change_set = match change_set_opt { Some(persisted_change_set) => persisted_change_set, @@ -49,18 +48,21 @@ impl WalletPersister for KVStoreWalletPersister { ChangeSet::default() }, }; - persister.latest_change_set = Some(change_set.clone()); + self.latest_change_set = Some(change_set.clone()); Ok(change_set) } - fn persist(persister: &mut Self, change_set: &ChangeSet) -> Result<(), Self::Error> { + async fn persist_inner(&mut self, change_set: &ChangeSet) -> Result<(), std::io::Error> { if change_set.is_empty() { return Ok(()); } + let kv_store = Arc::clone(&self.kv_store); + let logger = Arc::clone(&self.logger); + // We're allowed to fail here if we're not initialized, BDK docs state: "This method can fail if the // persister is not initialized." - let latest_change_set = persister.latest_change_set.as_mut().ok_or_else(|| { + let latest_change_set = self.latest_change_set.as_mut().ok_or_else(|| { std::io::Error::new( std::io::ErrorKind::Other, "Wallet must be initialized before calling persist", @@ -75,7 +77,7 @@ impl WalletPersister for KVStoreWalletPersister { { debug_assert!(false, "Wallet descriptor must never change"); log_error!( - persister.logger, + logger, "Wallet change set doesn't match persisted descriptor. This should never happen." ); return Err(std::io::Error::new( @@ -84,7 +86,7 @@ impl WalletPersister for KVStoreWalletPersister { )); } else { latest_change_set.descriptor = Some(descriptor.clone()); - write_bdk_wallet_descriptor(&descriptor, &*persister.kv_store, &*persister.logger)?; + write_bdk_wallet_descriptor(&descriptor, &*kv_store, Arc::clone(&logger)).await?; } } @@ -94,7 +96,7 @@ impl WalletPersister for KVStoreWalletPersister { { debug_assert!(false, "Wallet change_descriptor must never change"); log_error!( - persister.logger, + logger, "Wallet change set doesn't match persisted change_descriptor. This should never happen." ); return Err(std::io::Error::new( @@ -105,9 +107,10 @@ impl WalletPersister for KVStoreWalletPersister { latest_change_set.change_descriptor = Some(change_descriptor.clone()); write_bdk_wallet_change_descriptor( &change_descriptor, - &*persister.kv_store, - &*persister.logger, - )?; + &*kv_store, + Arc::clone(&logger), + ) + .await?; } } @@ -115,7 +118,7 @@ impl WalletPersister for KVStoreWalletPersister { if latest_change_set.network.is_some() && latest_change_set.network != Some(network) { debug_assert!(false, "Wallet network must never change"); log_error!( - persister.logger, + logger, "Wallet change set doesn't match persisted network. This should never happen." ); return Err(std::io::Error::new( @@ -124,7 +127,7 @@ impl WalletPersister for KVStoreWalletPersister { )); } else { latest_change_set.network = Some(network); - write_bdk_wallet_network(&network, &*persister.kv_store, &*persister.logger)?; + write_bdk_wallet_network(&network, &*kv_store, Arc::clone(&logger)).await?; } } @@ -144,31 +147,48 @@ impl WalletPersister for KVStoreWalletPersister { // particular order. if !change_set.indexer.is_empty() { latest_change_set.indexer.merge(change_set.indexer.clone()); - write_bdk_wallet_indexer( - &latest_change_set.indexer, - &*persister.kv_store, - Arc::clone(&persister.logger), - )?; + write_bdk_wallet_indexer(&latest_change_set.indexer, &*kv_store, Arc::clone(&logger)) + .await?; } if !change_set.tx_graph.is_empty() { latest_change_set.tx_graph.merge(change_set.tx_graph.clone()); - write_bdk_wallet_tx_graph( - &latest_change_set.tx_graph, - &*persister.kv_store, - Arc::clone(&persister.logger), - )?; + write_bdk_wallet_tx_graph(&latest_change_set.tx_graph, &*kv_store, Arc::clone(&logger)) + .await?; } if !change_set.local_chain.is_empty() { latest_change_set.local_chain.merge(change_set.local_chain.clone()); write_bdk_wallet_local_chain( &latest_change_set.local_chain, - &*persister.kv_store, - Arc::clone(&persister.logger), - )?; + &*kv_store, + Arc::clone(&logger), + ) + .await?; } Ok(()) } } + +impl AsyncWalletPersister for KVStoreWalletPersister { + type Error = std::io::Error; + + fn initialize<'a>( + persister: &'a mut Self, + ) -> Pin> + Send + 'a>> + where + Self: 'a, + { + Box::pin(persister.initialize_inner()) + } + + fn persist<'a>( + persister: &'a mut Self, change_set: &'a ChangeSet, + ) -> Pin> + Send + 'a>> + where + Self: 'a, + { + Box::pin(persister.persist_inner(change_set)) + } +} diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 30d9a4387e..d7775e67b3 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -24,7 +24,7 @@ use std::future::Future; use std::path::PathBuf; use std::str::FromStr; use std::sync::atomic::{AtomicU16, Ordering}; -use std::sync::{Arc, RwLock}; +use std::sync::Arc; use std::time::Duration; use bitcoin::hashes::hex::FromHex; @@ -50,8 +50,7 @@ use ldk_node::{ use lightning::io; use lightning::ln::msgs::SocketAddress; use lightning::routing::gossip::NodeAlias; -use lightning::util::persist::{KVStore, KVStoreSync}; -use lightning::util::test_utils::TestStore; +use lightning::util::persist::KVStore; use lightning_invoice::{Bolt11InvoiceDescription, Description}; use lightning_persister::fs_store::v1::FilesystemStore; use lightning_types::payment::{PaymentHash, PaymentPreimage}; @@ -60,6 +59,10 @@ use rand::distr::Alphanumeric; use rand::{rng, Rng}; use serde_json::{json, Value}; +#[path = "../../src/io/in_memory_store.rs"] +mod in_memory_store; +use in_memory_store::InMemoryStore; + /// Shared timeout (in seconds) for waiting on LDK events and external node operations. pub(crate) const INTEROP_TIMEOUT_SECS: u64 = 60; @@ -1660,16 +1663,9 @@ impl KVStore for TestSyncStore { let secondary_namespace = secondary_namespace.to_string(); let key = key.to_string(); let inner = Arc::clone(&self.inner); - let fut = tokio::task::spawn_blocking(move || { - inner.read_internal(&primary_namespace, &secondary_namespace, &key) - }); - async move { - fut.await.unwrap_or_else(|e| { - let msg = format!("Failed to IO operation due join error: {}", e); - Err(io::Error::new(io::ErrorKind::Other, msg)) - }) - } + async move { inner.read_internal_async(&primary_namespace, &secondary_namespace, &key).await } } + fn write( &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, ) -> impl Future> + 'static + Send { @@ -1677,16 +1673,11 @@ impl KVStore for TestSyncStore { let secondary_namespace = secondary_namespace.to_string(); let key = key.to_string(); let inner = Arc::clone(&self.inner); - let fut = tokio::task::spawn_blocking(move || { - inner.write_internal(&primary_namespace, &secondary_namespace, &key, buf) - }); async move { - fut.await.unwrap_or_else(|e| { - let msg = format!("Failed to IO operation due join error: {}", e); - Err(io::Error::new(io::ErrorKind::Other, msg)) - }) + inner.write_internal_async(&primary_namespace, &secondary_namespace, &key, buf).await } } + fn remove( &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, ) -> impl Future> + 'static + Send { @@ -1694,70 +1685,31 @@ impl KVStore for TestSyncStore { let secondary_namespace = secondary_namespace.to_string(); let key = key.to_string(); let inner = Arc::clone(&self.inner); - let fut = tokio::task::spawn_blocking(move || { - inner.remove_internal(&primary_namespace, &secondary_namespace, &key, lazy) - }); async move { - fut.await.unwrap_or_else(|e| { - let msg = format!("Failed to IO operation due join error: {}", e); - Err(io::Error::new(io::ErrorKind::Other, msg)) - }) + inner.remove_internal_async(&primary_namespace, &secondary_namespace, &key, lazy).await } } + fn list( &self, primary_namespace: &str, secondary_namespace: &str, ) -> impl Future, io::Error>> + 'static + Send { let primary_namespace = primary_namespace.to_string(); let secondary_namespace = secondary_namespace.to_string(); let inner = Arc::clone(&self.inner); - let fut = tokio::task::spawn_blocking(move || { - inner.list_internal(&primary_namespace, &secondary_namespace) - }); - async move { - fut.await.unwrap_or_else(|e| { - let msg = format!("Failed to IO operation due join error: {}", e); - Err(io::Error::new(io::ErrorKind::Other, msg)) - }) - } - } -} - -impl KVStoreSync for TestSyncStore { - fn read( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, - ) -> lightning::io::Result> { - self.inner.read_internal(primary_namespace, secondary_namespace, key) - } - - fn write( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, - ) -> lightning::io::Result<()> { - self.inner.write_internal(primary_namespace, secondary_namespace, key, buf) - } - - fn remove( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, - ) -> lightning::io::Result<()> { - self.inner.remove_internal(primary_namespace, secondary_namespace, key, lazy) - } - - fn list( - &self, primary_namespace: &str, secondary_namespace: &str, - ) -> lightning::io::Result> { - self.inner.list_internal(primary_namespace, secondary_namespace) + async move { inner.list_internal_async(&primary_namespace, &secondary_namespace).await } } } struct TestSyncStoreInner { - serializer: RwLock<()>, - test_store: TestStore, + serializer: tokio::sync::RwLock<()>, + test_store: InMemoryStore, fs_store: FilesystemStore, sqlite_store: SqliteStore, } impl TestSyncStoreInner { fn new(dest_dir: PathBuf) -> Self { - let serializer = RwLock::new(()); + let serializer = tokio::sync::RwLock::new(()); let mut fs_dir = dest_dir.clone(); fs_dir.push("fs_store"); let fs_store = FilesystemStore::new(fs_dir); @@ -1769,17 +1721,18 @@ impl TestSyncStoreInner { Some("test_sync_table".to_string()), ) .unwrap(); - let test_store = TestStore::new(false); + let test_store = InMemoryStore::new(); Self { serializer, fs_store, sqlite_store, test_store } } - fn do_list( + async fn do_list_async( &self, primary_namespace: &str, secondary_namespace: &str, ) -> lightning::io::Result> { - let fs_res = KVStoreSync::list(&self.fs_store, primary_namespace, secondary_namespace); + let fs_res = KVStore::list(&self.fs_store, primary_namespace, secondary_namespace).await; let sqlite_res = - KVStoreSync::list(&self.sqlite_store, primary_namespace, secondary_namespace); - let test_res = KVStoreSync::list(&self.test_store, primary_namespace, secondary_namespace); + KVStore::list(&self.sqlite_store, primary_namespace, secondary_namespace).await; + let test_res = + KVStore::list(&self.test_store, primary_namespace, secondary_namespace).await; match fs_res { Ok(mut list) => { @@ -1803,16 +1756,24 @@ impl TestSyncStoreInner { } } - fn read_internal( + async fn list_internal_async( + &self, primary_namespace: &str, secondary_namespace: &str, + ) -> lightning::io::Result> { + let _guard = self.serializer.read().await; + self.do_list_async(primary_namespace, secondary_namespace).await + } + + async fn read_internal_async( &self, primary_namespace: &str, secondary_namespace: &str, key: &str, ) -> lightning::io::Result> { - let _guard = self.serializer.read().unwrap(); + let _guard = self.serializer.read().await; - let fs_res = KVStoreSync::read(&self.fs_store, primary_namespace, secondary_namespace, key); + let fs_res = + KVStore::read(&self.fs_store, primary_namespace, secondary_namespace, key).await; let sqlite_res = - KVStoreSync::read(&self.sqlite_store, primary_namespace, secondary_namespace, key); + KVStore::read(&self.sqlite_store, primary_namespace, secondary_namespace, key).await; let test_res = - KVStoreSync::read(&self.test_store, primary_namespace, secondary_namespace, key); + KVStore::read(&self.test_store, primary_namespace, secondary_namespace, key).await; match fs_res { Ok(read) => { @@ -1830,34 +1791,38 @@ impl TestSyncStoreInner { } } - fn write_internal( + async fn write_internal_async( &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, ) -> lightning::io::Result<()> { - let _guard = self.serializer.write().unwrap(); - let fs_res = KVStoreSync::write( + let _guard = self.serializer.write().await; + let fs_res = KVStore::write( &self.fs_store, primary_namespace, secondary_namespace, key, buf.clone(), - ); - let sqlite_res = KVStoreSync::write( + ) + .await; + let sqlite_res = KVStore::write( &self.sqlite_store, primary_namespace, secondary_namespace, key, buf.clone(), - ); - let test_res = KVStoreSync::write( + ) + .await; + let test_res = KVStore::write( &self.test_store, primary_namespace, secondary_namespace, key, buf.clone(), - ); + ) + .await; assert!(self - .do_list(primary_namespace, secondary_namespace) + .do_list_async(primary_namespace, secondary_namespace) + .await .unwrap() .contains(&key.to_string())); @@ -1875,29 +1840,23 @@ impl TestSyncStoreInner { } } - fn remove_internal( + async fn remove_internal_async( &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, ) -> lightning::io::Result<()> { - let _guard = self.serializer.write().unwrap(); + let _guard = self.serializer.write().await; let fs_res = - KVStoreSync::remove(&self.fs_store, primary_namespace, secondary_namespace, key, lazy); - let sqlite_res = KVStoreSync::remove( - &self.sqlite_store, - primary_namespace, - secondary_namespace, - key, - lazy, - ); - let test_res = KVStoreSync::remove( - &self.test_store, - primary_namespace, - secondary_namespace, - key, - lazy, - ); + KVStore::remove(&self.fs_store, primary_namespace, secondary_namespace, key, lazy) + .await; + let sqlite_res = + KVStore::remove(&self.sqlite_store, primary_namespace, secondary_namespace, key, lazy) + .await; + let test_res = + KVStore::remove(&self.test_store, primary_namespace, secondary_namespace, key, lazy) + .await; assert!(!self - .do_list(primary_namespace, secondary_namespace) + .do_list_async(primary_namespace, secondary_namespace) + .await .unwrap() .contains(&key.to_string())); @@ -1914,11 +1873,4 @@ impl TestSyncStoreInner { }, } } - - fn list_internal( - &self, primary_namespace: &str, secondary_namespace: &str, - ) -> lightning::io::Result> { - let _guard = self.serializer.read().unwrap(); - self.do_list(primary_namespace, secondary_namespace) - } }