From c8cbaf2ce96917ce676d3d1f2e131abb5f81d67f Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Wed, 3 Jun 2026 10:30:22 +0200 Subject: [PATCH 01/21] DROPME: Bump LDK for async store migration Use a temporary rust-lightning fork revision that exposes async migratable KV-store support. This lets ldk-node migrate filesystem stores without reimplementing LDK's migration logic locally. Co-Authored-By: HAL 9000 --- Cargo.toml | 14 ++++++++++++++ src/data_store.rs | 6 +++--- src/event.rs | 6 +++--- src/io/vss_store.rs | 4 ++-- src/lib.rs | 6 +++--- .../asynchronous/static_invoice_store.rs | 4 ++-- src/payment/bolt11.rs | 4 ++-- src/payment/pending_payment_store.rs | 4 ++-- src/payment/store.rs | 18 +++++++++--------- src/peer_store.rs | 4 ++-- src/types.rs | 4 ++-- 11 files changed, 44 insertions(+), 30 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 2a60c8cedc..72db969b5a 100755 --- a/Cargo.toml +++ b/Cargo.toml @@ -196,3 +196,17 @@ harness = false #lightning-liquidity = { path = "../rust-lightning/lightning-liquidity" } #lightning-macros = { path = "../rust-lightning/lightning-macros" } #lightning-dns-resolver = { path = "../rust-lightning/lightning-dns-resolver" } + +[patch."https://github.com/lightningdevkit/rust-lightning"] +lightning = { git = "https://github.com/tnull/rust-lightning", rev = "2611766e6a5913d6e33afdc1932485575ee8ea4e" } +lightning-types = { git = "https://github.com/tnull/rust-lightning", rev = "2611766e6a5913d6e33afdc1932485575ee8ea4e" } +lightning-invoice = { git = "https://github.com/tnull/rust-lightning", rev = "2611766e6a5913d6e33afdc1932485575ee8ea4e" } +lightning-net-tokio = { git = "https://github.com/tnull/rust-lightning", rev = "2611766e6a5913d6e33afdc1932485575ee8ea4e" } +lightning-persister = { git = "https://github.com/tnull/rust-lightning", rev = "2611766e6a5913d6e33afdc1932485575ee8ea4e" } +lightning-background-processor = { git = "https://github.com/tnull/rust-lightning", rev = "2611766e6a5913d6e33afdc1932485575ee8ea4e" } +lightning-rapid-gossip-sync = { git = "https://github.com/tnull/rust-lightning", rev = "2611766e6a5913d6e33afdc1932485575ee8ea4e" } +lightning-block-sync = { git = "https://github.com/tnull/rust-lightning", rev = "2611766e6a5913d6e33afdc1932485575ee8ea4e" } +lightning-transaction-sync = { git = "https://github.com/tnull/rust-lightning", rev = "2611766e6a5913d6e33afdc1932485575ee8ea4e" } +lightning-liquidity = { git = "https://github.com/tnull/rust-lightning", rev = "2611766e6a5913d6e33afdc1932485575ee8ea4e" } +lightning-macros = { git = "https://github.com/tnull/rust-lightning", rev = "2611766e6a5913d6e33afdc1932485575ee8ea4e" } +lightning-dns-resolver = { git = "https://github.com/tnull/rust-lightning", rev = "2611766e6a5913d6e33afdc1932485575ee8ea4e" } diff --git a/src/data_store.rs b/src/data_store.rs index f80ec08915..2fad814221 100644 --- a/src/data_store.rs +++ b/src/data_store.rs @@ -175,7 +175,7 @@ where #[cfg(test)] mod tests { - use lightning::impl_writeable_tlv_based; + use lightning::impl_ser_tlv_based; use lightning::util::test_utils::TestLogger; use super::*; @@ -193,7 +193,7 @@ mod tests { hex_utils::to_string(&self.id) } } - impl_writeable_tlv_based!(TestObjectId, { (0, id, required) }); + impl_ser_tlv_based!(TestObjectId, { (0, id, required) }); struct TestObjectUpdate { id: TestObjectId, @@ -233,7 +233,7 @@ mod tests { } } - impl_writeable_tlv_based!(TestObject, { + impl_ser_tlv_based!(TestObject, { (0, id, required), (2, data, required), }); diff --git a/src/event.rs b/src/event.rs index 7d23be99a6..5ac64ce97d 100644 --- a/src/event.rs +++ b/src/event.rs @@ -29,7 +29,7 @@ use lightning::util::config::{ChannelConfigOverrides, ChannelConfigUpdate}; use lightning::util::errors::APIError; use lightning::util::persist::KVStore; use lightning::util::ser::{Readable, ReadableArgs, Writeable, Writer}; -use lightning::{impl_writeable_tlv_based, impl_writeable_tlv_based_enum}; +use lightning::{impl_ser_tlv_based, impl_ser_tlv_based_enum}; use lightning_liquidity::lsps2::utils::compute_opening_fee; use lightning_types::payment::{PaymentHash, PaymentPreimage}; @@ -78,7 +78,7 @@ pub struct HTLCLocator { pub node_id: Option, } -impl_writeable_tlv_based!(HTLCLocator, { +impl_ser_tlv_based!(HTLCLocator, { (1, channel_id, required), (3, user_channel_id, option), (5, node_id, option), @@ -294,7 +294,7 @@ pub enum Event { }, } -impl_writeable_tlv_based_enum!(Event, +impl_ser_tlv_based_enum!(Event, (0, PaymentSuccessful) => { (0, payment_hash, required), (1, fee_paid_msat, option), diff --git a/src/io/vss_store.rs b/src/io/vss_store.rs index 97883b5d53..007f7b9b48 100644 --- a/src/io/vss_store.rs +++ b/src/io/vss_store.rs @@ -21,7 +21,7 @@ use bitcoin::bip32::{ChildNumber, Xpriv}; use bitcoin::hashes::{sha256, Hash, HashEngine, Hmac, HmacEngine}; use bitcoin::key::Secp256k1; use bitcoin::Network; -use lightning::impl_writeable_tlv_based_enum; +use lightning::impl_ser_tlv_based_enum; use lightning::io::{self, Error, ErrorKind}; use lightning::sign::{EntropySource as LdkEntropySource, RandomBytes}; use lightning::util::persist::{KVStore, KVStoreSync}; @@ -64,7 +64,7 @@ enum VssSchemaVersion { V1, } -impl_writeable_tlv_based_enum!(VssSchemaVersion, +impl_ser_tlv_based_enum!(VssSchemaVersion, (0, V0) => {}, (1, V1) => {}, ); diff --git a/src/lib.rs b/src/lib.rs index 614be098b0..51edd76173 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -147,7 +147,7 @@ use graph::NetworkGraph; use io::utils::update_and_persist_node_metrics; pub use lightning; use lightning::chain::BlockLocator; -use lightning::impl_writeable_tlv_based; +use lightning::impl_ser_tlv_based; use lightning::ln::chan_utils::FUNDING_TRANSACTION_WITNESS_WEIGHT; use lightning::ln::channel_state::ChannelDetails as LdkChannelDetails; pub use lightning::ln::channel_state::ChannelShutdownState; @@ -2181,7 +2181,7 @@ impl Default for NodeMetrics { } } -impl_writeable_tlv_based!(NodeMetrics, { +impl_ser_tlv_based!(NodeMetrics, { (0, latest_lightning_wallet_sync_timestamp, option), (1, latest_pathfinding_scores_sync_timestamp, option), (2, latest_onchain_wallet_sync_timestamp, option), @@ -2251,7 +2251,7 @@ mod tests { latest_pathfinding_scores_sync_timestamp: Option, latest_node_announcement_broadcast_timestamp: Option, } - impl_writeable_tlv_based!(OldNodeMetrics, { + impl_ser_tlv_based!(OldNodeMetrics, { (0, latest_lightning_wallet_sync_timestamp, option), (1, latest_pathfinding_scores_sync_timestamp, option), (2, latest_onchain_wallet_sync_timestamp, option), diff --git a/src/payment/asynchronous/static_invoice_store.rs b/src/payment/asynchronous/static_invoice_store.rs index 6fb406334c..90c9513fcd 100644 --- a/src/payment/asynchronous/static_invoice_store.rs +++ b/src/payment/asynchronous/static_invoice_store.rs @@ -13,7 +13,7 @@ use std::time::Duration; use bitcoin::hashes::sha256::Hash as Sha256; use bitcoin::hashes::Hash; use lightning::blinded_path::message::BlindedMessagePath; -use lightning::impl_writeable_tlv_based; +use lightning::impl_ser_tlv_based; use lightning::offers::static_invoice::StaticInvoice; use lightning::util::persist::KVStoreSync; use lightning::util::ser::{Readable, Writeable}; @@ -28,7 +28,7 @@ struct PersistedStaticInvoice { request_path: BlindedMessagePath, } -impl_writeable_tlv_based!(PersistedStaticInvoice, { +impl_ser_tlv_based!(PersistedStaticInvoice, { (0, invoice, required), (2, request_path, required) }); diff --git a/src/payment/bolt11.rs b/src/payment/bolt11.rs index e81aa51f7b..4fba38cc83 100644 --- a/src/payment/bolt11.rs +++ b/src/payment/bolt11.rs @@ -13,7 +13,7 @@ use std::sync::{Arc, RwLock}; use bitcoin::hashes::sha256::Hash as Sha256; use bitcoin::hashes::Hash; -use lightning::impl_writeable_tlv_based; +use lightning::impl_ser_tlv_based; use lightning::ln::channelmanager::{ Bolt11InvoiceParameters, OptionalBolt11PaymentParams, PaymentId, }; @@ -55,7 +55,7 @@ pub(crate) struct PaymentMetadata { pub(crate) lsps2_parameters: Option, } -impl_writeable_tlv_based!(PaymentMetadata, { +impl_ser_tlv_based!(PaymentMetadata, { (0, lsps2_parameters, option), }); diff --git a/src/payment/pending_payment_store.rs b/src/payment/pending_payment_store.rs index eb72f89ec9..37a3b09347 100644 --- a/src/payment/pending_payment_store.rs +++ b/src/payment/pending_payment_store.rs @@ -6,7 +6,7 @@ // accordance with one or both of these licenses. use bitcoin::Txid; -use lightning::impl_writeable_tlv_based; +use lightning::impl_ser_tlv_based; use lightning::ln::channelmanager::PaymentId; use crate::data_store::{StorableObject, StorableObjectUpdate}; @@ -33,7 +33,7 @@ impl PendingPaymentDetails { } } -impl_writeable_tlv_based!(PendingPaymentDetails, { +impl_ser_tlv_based!(PendingPaymentDetails, { (0, details, required), (2, conflicting_txids, optional_vec), }); diff --git a/src/payment/store.rs b/src/payment/store.rs index f80ab6f8a5..db4ba06ed8 100644 --- a/src/payment/store.rs +++ b/src/payment/store.rs @@ -13,8 +13,8 @@ use lightning::ln::msgs::DecodeError; use lightning::offers::offer::OfferId; use lightning::util::ser::{Readable, Writeable}; use lightning::{ - _init_and_read_len_prefixed_tlv_fields, impl_writeable_tlv_based, - impl_writeable_tlv_based_enum, write_tlv_fields, + _init_and_read_len_prefixed_tlv_fields, impl_ser_tlv_based, impl_ser_tlv_based_enum, + write_tlv_fields, }; use lightning_types::payment::{PaymentHash, PaymentPreimage, PaymentSecret}; use lightning_types::string::UntrustedString; @@ -307,7 +307,7 @@ pub enum PaymentDirection { Outbound, } -impl_writeable_tlv_based_enum!(PaymentDirection, +impl_ser_tlv_based_enum!(PaymentDirection, (0, Inbound) => {}, (1, Outbound) => {} ); @@ -324,7 +324,7 @@ pub enum PaymentStatus { Failed, } -impl_writeable_tlv_based_enum!(PaymentStatus, +impl_ser_tlv_based_enum!(PaymentStatus, (0, Pending) => {}, (2, Succeeded) => {}, (4, Failed) => {} @@ -420,7 +420,7 @@ pub enum PaymentKind { }, } -impl_writeable_tlv_based_enum!(PaymentKind, +impl_ser_tlv_based_enum!(PaymentKind, (0, Onchain) => { (0, txid, required), (2, status, required), @@ -479,7 +479,7 @@ pub enum ConfirmationStatus { Unconfirmed, } -impl_writeable_tlv_based_enum!(ConfirmationStatus, +impl_ser_tlv_based_enum!(ConfirmationStatus, (0, Confirmed) => { (0, block_hash, required), (2, height, required), @@ -504,7 +504,7 @@ pub struct LSPS2Parameters { pub max_proportional_opening_fee_ppm_msat: Option, } -impl_writeable_tlv_based!(LSPS2Parameters, { +impl_ser_tlv_based!(LSPS2Parameters, { (0, max_total_opening_fee_msat, option), (2, max_proportional_opening_fee_ppm_msat, option), }); @@ -604,7 +604,7 @@ mod tests { pub status: PaymentStatus, } - impl_writeable_tlv_based!(OldPaymentDetails, { + impl_ser_tlv_based!(OldPaymentDetails, { (0, hash, required), (2, preimage, required), (4, secret, required), @@ -706,7 +706,7 @@ mod tests { lsp_fee_limits: LSPS2Parameters, } - impl_writeable_tlv_based!(LegacyBolt11JitKind, { + impl_ser_tlv_based!(LegacyBolt11JitKind, { (0, hash, required), (1, counterparty_skimmed_fee_msat, option), (2, preimage, option), diff --git a/src/peer_store.rs b/src/peer_store.rs index 307fb69296..38d4abc4e1 100644 --- a/src/peer_store.rs +++ b/src/peer_store.rs @@ -10,7 +10,7 @@ use std::ops::Deref; use std::sync::{Arc, RwLock}; use bitcoin::secp256k1::PublicKey; -use lightning::impl_writeable_tlv_based; +use lightning::impl_ser_tlv_based; use lightning::util::persist::KVStoreSync; use lightning::util::ser::{Readable, ReadableArgs, Writeable, Writer}; @@ -142,7 +142,7 @@ pub(crate) struct PeerInfo { pub address: SocketAddress, } -impl_writeable_tlv_based!(PeerInfo, { +impl_ser_tlv_based!(PeerInfo, { (0, node_id, required), (2, address, required), }); diff --git a/src/types.rs b/src/types.rs index 06e65fbd0a..8ff5b29660 100644 --- a/src/types.rs +++ b/src/types.rs @@ -19,7 +19,7 @@ use bitcoin_payment_instructions::hrn_resolution::{ }; use bitcoin_payment_instructions::onion_message_resolver::LDKOnionMessageDNSSECHrnResolver; use lightning::chain::chainmonitor; -use lightning::impl_writeable_tlv_based; +use lightning::impl_ser_tlv_based; use lightning::ln::channel_state::{ChannelDetails as LdkChannelDetails, ChannelShutdownState}; use lightning::ln::msgs::{RoutingMessageHandler, SocketAddress}; use lightning::ln::peer_handler::IgnoringMessageHandler; @@ -690,7 +690,7 @@ pub struct CustomTlvRecord { pub value: Vec, } -impl_writeable_tlv_based!(CustomTlvRecord, { +impl_ser_tlv_based!(CustomTlvRecord, { (0, type_num, required), (2, value, required), }); From 4cf36667c65885dee08eecfafaeb06140b7b1d9c Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Wed, 3 Jun 2026 10:35:46 +0200 Subject: [PATCH 02/21] Move BDK wallet helpers onto async KV storage Read and write BDK wallet state through async KVStore helpers while keeping the current WalletPersister entry points bridged through the node runtime. This reduces the wallet persistence surface that still depends on KVStoreSync. Co-Authored-By: HAL 9000 --- src/builder.rs | 7 +++++-- src/io/utils.rs | 27 ++++++++++++++++----------- src/wallet/persist.rs | 39 ++++++++++++++++++++++++++------------- 3 files changed, 47 insertions(+), 26 deletions(-) diff --git a/src/builder.rs b/src/builder.rs index 03ded494f9..c217850e38 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -1537,8 +1537,11 @@ fn build_with_store_internal( let descriptor = Bip84(xprv, KeychainKind::External); let change_descriptor = Bip84(xprv, KeychainKind::Internal); - let mut wallet_persister = - KVStoreWalletPersister::new(Arc::clone(&kv_store), Arc::clone(&logger)); + let mut wallet_persister = KVStoreWalletPersister::new( + Arc::clone(&kv_store), + Arc::clone(&runtime), + Arc::clone(&logger), + ); let wallet_opt = BdkWallet::load() .descriptor(KeychainKind::External, Some(descriptor.clone())) .descriptor(KeychainKind::Internal, Some(change_descriptor.clone())) diff --git a/src/io/utils.rs b/src/io/utils.rs index 89d4afc5c3..11d1255b84 100644 --- a/src/io/utils.rs +++ b/src/io/utils.rs @@ -469,14 +469,15 @@ macro_rules! impl_read_write_change_set_type { $secondary_namespace:expr, $key:expr ) => { - pub(crate) fn $read_name( + pub(crate) async fn $read_name( kv_store: &DynStore, logger: L, ) -> Result, std::io::Error> where L::Target: LdkLogger, { let reader = - match KVStoreSync::read(&*kv_store, $primary_namespace, $secondary_namespace, $key) + match KVStore::read(&*kv_store, $primary_namespace, $secondary_namespace, $key) + .await { Ok(bytes) => bytes, Err(e) => { @@ -510,14 +511,15 @@ macro_rules! impl_read_write_change_set_type { } } - pub(crate) fn $write_name( + pub(crate) async fn $write_name( value: &$change_set_type, kv_store: &DynStore, logger: L, ) -> Result<(), std::io::Error> where L::Target: LdkLogger, { let data = ChangeSetSerWrapper(value).encode(); - KVStoreSync::write(&*kv_store, $primary_namespace, $secondary_namespace, $key, data) + KVStore::write(&*kv_store, $primary_namespace, $secondary_namespace, $key, data) + .await .map_err(|e| { log_error!( logger, @@ -588,36 +590,39 @@ impl_read_write_change_set_type!( ); // Reads the full BdkWalletChangeSet or returns default fields -pub(crate) fn read_bdk_wallet_change_set( +pub(crate) async fn read_bdk_wallet_change_set( kv_store: &DynStore, logger: &Logger, ) -> Result, std::io::Error> { let mut change_set = BdkWalletChangeSet::default(); // We require a descriptor and return `None` to signal creation of a new wallet otherwise. - if let Some(descriptor) = read_bdk_wallet_descriptor(kv_store, logger)? { + if let Some(descriptor) = read_bdk_wallet_descriptor(kv_store, logger).await? { change_set.descriptor = Some(descriptor); } else { return Ok(None); } // We require a change_descriptor and return `None` to signal creation of a new wallet otherwise. - if let Some(change_descriptor) = read_bdk_wallet_change_descriptor(kv_store, logger)? { + if let Some(change_descriptor) = read_bdk_wallet_change_descriptor(kv_store, logger).await? { change_set.change_descriptor = Some(change_descriptor); } else { return Ok(None); } // We require a network and return `None` to signal creation of a new wallet otherwise. - if let Some(network) = read_bdk_wallet_network(kv_store, logger)? { + if let Some(network) = read_bdk_wallet_network(kv_store, logger).await? { change_set.network = Some(network); } else { return Ok(None); } - read_bdk_wallet_local_chain(&*kv_store, logger)? + read_bdk_wallet_local_chain(&*kv_store, logger) + .await? .map(|local_chain| change_set.local_chain = local_chain); - read_bdk_wallet_tx_graph(&*kv_store, logger)?.map(|tx_graph| change_set.tx_graph = tx_graph); - read_bdk_wallet_indexer(&*kv_store, logger)?.map(|indexer| change_set.indexer = indexer); + read_bdk_wallet_tx_graph(&*kv_store, logger) + .await? + .map(|tx_graph| change_set.tx_graph = tx_graph); + read_bdk_wallet_indexer(&*kv_store, logger).await?.map(|indexer| change_set.indexer = indexer); Ok(Some(change_set)) } diff --git a/src/wallet/persist.rs b/src/wallet/persist.rs index 10be1fac08..cc50057808 100644 --- a/src/wallet/persist.rs +++ b/src/wallet/persist.rs @@ -16,16 +16,19 @@ use crate::io::utils::{ write_bdk_wallet_tx_graph, }; use crate::logger::{log_error, LdkLogger, Logger}; +use crate::runtime::Runtime; use crate::types::DynStore; + pub(crate) struct KVStoreWalletPersister { latest_change_set: Option, kv_store: Arc, + runtime: Arc, logger: Arc, } impl KVStoreWalletPersister { - pub(crate) fn new(kv_store: Arc, logger: Arc) -> Self { - Self { latest_change_set: None, kv_store, logger } + pub(crate) fn new(kv_store: Arc, runtime: Arc, logger: Arc) -> Self { + Self { latest_change_set: None, kv_store, runtime, logger } } } @@ -38,7 +41,9 @@ impl WalletPersister for KVStoreWalletPersister { return Ok(latest_change_set.clone()); } - let change_set_opt = read_bdk_wallet_change_set(&*persister.kv_store, &*persister.logger)?; + let change_set_opt = persister + .runtime + .block_on(read_bdk_wallet_change_set(&*persister.kv_store, &*persister.logger))?; let change_set = match change_set_opt { Some(persisted_change_set) => persisted_change_set, @@ -84,7 +89,11 @@ impl WalletPersister for KVStoreWalletPersister { )); } else { latest_change_set.descriptor = Some(descriptor.clone()); - write_bdk_wallet_descriptor(&descriptor, &*persister.kv_store, &*persister.logger)?; + persister.runtime.block_on(write_bdk_wallet_descriptor( + &descriptor, + &*persister.kv_store, + &*persister.logger, + ))?; } } @@ -103,11 +112,11 @@ impl WalletPersister for KVStoreWalletPersister { )); } else { latest_change_set.change_descriptor = Some(change_descriptor.clone()); - write_bdk_wallet_change_descriptor( + persister.runtime.block_on(write_bdk_wallet_change_descriptor( &change_descriptor, &*persister.kv_store, &*persister.logger, - )?; + ))?; } } @@ -124,7 +133,11 @@ impl WalletPersister for KVStoreWalletPersister { )); } else { latest_change_set.network = Some(network); - write_bdk_wallet_network(&network, &*persister.kv_store, &*persister.logger)?; + persister.runtime.block_on(write_bdk_wallet_network( + &network, + &*persister.kv_store, + &*persister.logger, + ))?; } } @@ -144,29 +157,29 @@ impl WalletPersister for KVStoreWalletPersister { // particular order. if !change_set.indexer.is_empty() { latest_change_set.indexer.merge(change_set.indexer.clone()); - write_bdk_wallet_indexer( + persister.runtime.block_on(write_bdk_wallet_indexer( &latest_change_set.indexer, &*persister.kv_store, Arc::clone(&persister.logger), - )?; + ))?; } if !change_set.tx_graph.is_empty() { latest_change_set.tx_graph.merge(change_set.tx_graph.clone()); - write_bdk_wallet_tx_graph( + persister.runtime.block_on(write_bdk_wallet_tx_graph( &latest_change_set.tx_graph, &*persister.kv_store, Arc::clone(&persister.logger), - )?; + ))?; } if !change_set.local_chain.is_empty() { latest_change_set.local_chain.merge(change_set.local_chain.clone()); - write_bdk_wallet_local_chain( + persister.runtime.block_on(write_bdk_wallet_local_chain( &latest_change_set.local_chain, &*persister.kv_store, Arc::clone(&persister.logger), - )?; + ))?; } Ok(()) From bec07191565611364fb95e8557ccefebe61ea8b9 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Wed, 3 Jun 2026 10:39:05 +0200 Subject: [PATCH 03/21] Use async KV storage for static invoices Static invoice persistence already runs from async handlers, so use KVStore directly instead of routing those reads and writes through the blocking KVStoreSync trait. Co-Authored-By: HAL 9000 --- .../asynchronous/static_invoice_store.rs | 44 +++++++++---------- 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/src/payment/asynchronous/static_invoice_store.rs b/src/payment/asynchronous/static_invoice_store.rs index 90c9513fcd..2e89347388 100644 --- a/src/payment/asynchronous/static_invoice_store.rs +++ b/src/payment/asynchronous/static_invoice_store.rs @@ -15,7 +15,7 @@ use bitcoin::hashes::Hash; use lightning::blinded_path::message::BlindedMessagePath; use lightning::impl_ser_tlv_based; use lightning::offers::static_invoice::StaticInvoice; -use lightning::util::persist::KVStoreSync; +use lightning::util::persist::KVStore; use lightning::util::ser::{Readable, Writeable}; use crate::hex_utils; @@ -78,33 +78,28 @@ impl StaticInvoiceStore { let (secondary_namespace, key) = Self::get_storage_location(invoice_slot, recipient_id); - KVStoreSync::read( + let data = match KVStore::read( &*self.kv_store, STATIC_INVOICE_STORE_PRIMARY_NAMESPACE, &secondary_namespace, &key, ) - .and_then(|data| { - PersistedStaticInvoice::read(&mut &*data) - .map(|persisted_invoice| { - Some((persisted_invoice.invoice, persisted_invoice.request_path)) - }) - .map_err(|e| { - lightning::io::Error::new( - lightning::io::ErrorKind::InvalidData, - format!("Failed to parse static invoice: {:?}", e), - ) - }) - }) - .or_else( - |e| { - if e.kind() == lightning::io::ErrorKind::NotFound { - Ok(None) - } else { - Err(e) - } - }, - ) + .await + { + Ok(data) => data, + Err(e) if e.kind() == lightning::io::ErrorKind::NotFound => return Ok(None), + Err(e) => return Err(e), + }; + PersistedStaticInvoice::read(&mut &*data) + .map(|persisted_invoice| { + Some((persisted_invoice.invoice, persisted_invoice.request_path)) + }) + .map_err(|e| { + lightning::io::Error::new( + lightning::io::ErrorKind::InvalidData, + format!("Failed to parse static invoice: {:?}", e), + ) + }) } pub(crate) async fn handle_persist_static_invoice( @@ -124,13 +119,14 @@ impl StaticInvoiceStore { // Static invoices will be persisted at "static_invoices//". // // Example: static_invoices/039058c6f2c0cb492c533b0a4d14ef77cc0f78abccced5287d84a1a2011cfb81/00001 - KVStoreSync::write( + KVStore::write( &*self.kv_store, STATIC_INVOICE_STORE_PRIMARY_NAMESPACE, &secondary_namespace, &key, buf, ) + .await } fn get_storage_location(invoice_slot: u16, recipient_id: &[u8]) -> (String, String) { From 7bfdd0d54ff6e17b94fe052e5b8394d86a076ad0 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Wed, 3 Jun 2026 16:42:52 +0200 Subject: [PATCH 04/21] f Simplify --- .../asynchronous/static_invoice_store.rs | 38 +++++++++++-------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/src/payment/asynchronous/static_invoice_store.rs b/src/payment/asynchronous/static_invoice_store.rs index 2e89347388..85d9479234 100644 --- a/src/payment/asynchronous/static_invoice_store.rs +++ b/src/payment/asynchronous/static_invoice_store.rs @@ -78,28 +78,34 @@ impl StaticInvoiceStore { let (secondary_namespace, key) = Self::get_storage_location(invoice_slot, recipient_id); - let data = match KVStore::read( + KVStore::read( &*self.kv_store, STATIC_INVOICE_STORE_PRIMARY_NAMESPACE, &secondary_namespace, &key, ) .await - { - Ok(data) => data, - Err(e) if e.kind() == lightning::io::ErrorKind::NotFound => return Ok(None), - Err(e) => return Err(e), - }; - PersistedStaticInvoice::read(&mut &*data) - .map(|persisted_invoice| { - Some((persisted_invoice.invoice, persisted_invoice.request_path)) - }) - .map_err(|e| { - lightning::io::Error::new( - lightning::io::ErrorKind::InvalidData, - format!("Failed to parse static invoice: {:?}", e), - ) - }) + .and_then(|data| { + PersistedStaticInvoice::read(&mut &*data) + .map(|persisted_invoice| { + Some((persisted_invoice.invoice, persisted_invoice.request_path)) + }) + .map_err(|e| { + lightning::io::Error::new( + lightning::io::ErrorKind::InvalidData, + format!("Failed to parse static invoice: {:?}", e), + ) + }) + }) + .or_else( + |e| { + if e.kind() == lightning::io::ErrorKind::NotFound { + Ok(None) + } else { + Err(e) + } + }, + ) } pub(crate) async fn handle_persist_static_invoice( From cbf87bb55478195ca17021289748778b6ee293e3 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Wed, 3 Jun 2026 10:42:45 +0200 Subject: [PATCH 05/21] Move peer persistence onto async KV storage Persist peer store updates through async KVStore operations. The synchronous node APIs keep bridging at their runtime boundary while async event handling awaits peer persistence directly. Co-Authored-By: HAL 9000 --- src/event.rs | 62 ++++++++++++++++++++++--------------------- src/lib.rs | 8 +++--- src/payment/bolt11.rs | 2 +- src/peer_store.rs | 51 +++++++++++++++++++---------------- 4 files changed, 65 insertions(+), 58 deletions(-) diff --git a/src/event.rs b/src/event.rs index 5ac64ce97d..c3de8a2391 100644 --- a/src/event.rs +++ b/src/event.rs @@ -1524,36 +1524,38 @@ where }, }; - let network_graph = self.network_graph.read_only(); - let channels = - self.channel_manager.list_channels_with_counterparty(&counterparty_node_id); - if let Some(pending_channel) = - channels.into_iter().find(|c| c.channel_id == channel_id) - { - if !pending_channel.is_outbound - && self.peer_store.get_peer(&counterparty_node_id).is_none() - { - if let Some(address) = network_graph - .nodes() - .get(&NodeId::from_pubkey(&counterparty_node_id)) - .and_then(|node_info| node_info.announcement_info.as_ref()) - .and_then(|ann_info| ann_info.addresses().first()) - { - let peer = PeerInfo { - node_id: counterparty_node_id, - address: address.clone(), - }; - - self.peer_store.add_peer(peer).unwrap_or_else(|e| { - log_error!( - self.logger, - "Failed to add peer {} to peer store: {}", - counterparty_node_id, - e - ); - }); - } - } + let peer_to_store = { + let network_graph = self.network_graph.read_only(); + let channels = + self.channel_manager.list_channels_with_counterparty(&counterparty_node_id); + channels + .into_iter() + .find(|c| c.channel_id == channel_id) + .filter(|pending_channel| { + !pending_channel.is_outbound + && self.peer_store.get_peer(&counterparty_node_id).is_none() + }) + .and_then(|_| { + network_graph + .nodes() + .get(&NodeId::from_pubkey(&counterparty_node_id)) + .and_then(|node_info| node_info.announcement_info.as_ref()) + .and_then(|ann_info| ann_info.addresses().first()) + .map(|address| PeerInfo { + node_id: counterparty_node_id, + address: address.clone(), + }) + }) + }; + if let Some(peer) = peer_to_store { + self.peer_store.add_peer(peer).await.unwrap_or_else(|e| { + log_error!( + self.logger, + "Failed to add peer {} to peer store: {}", + counterparty_node_id, + e + ); + }); } }, LdkEvent::ChannelReady { diff --git a/src/lib.rs b/src/lib.rs index 51edd76173..f20f1e7a99 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1121,7 +1121,7 @@ impl Node { log_info!(self.logger, "Connected to peer {}@{}. ", peer_info.node_id, peer_info.address); if persist { - self.peer_store.add_peer(peer_info)?; + self.runtime.block_on(self.peer_store.add_peer(peer_info))?; } Ok(()) @@ -1138,7 +1138,7 @@ impl Node { log_info!(self.logger, "Disconnecting peer {}..", counterparty_node_id); - match self.peer_store.remove_peer(&counterparty_node_id) { + match self.runtime.block_on(self.peer_store.remove_peer(&counterparty_node_id)) { Ok(()) => {}, Err(e) => { log_error!(self.logger, "Failed to remove peer {}: {}", counterparty_node_id, e) @@ -1255,7 +1255,7 @@ impl Node { zero_reserve_string, peer_info.node_id ); - self.peer_store.add_peer(peer_info)?; + self.runtime.block_on(self.peer_store.add_peer(peer_info))?; Ok(UserChannelId(user_channel_id)) }, Err(e) => { @@ -1861,7 +1861,7 @@ impl Node { // Check if this was the last open channel, if so, forget the peer. if open_channels.len() == 1 { - self.peer_store.remove_peer(&counterparty_node_id)?; + self.runtime.block_on(self.peer_store.remove_peer(&counterparty_node_id))?; } } diff --git a/src/payment/bolt11.rs b/src/payment/bolt11.rs index 4fba38cc83..f1d4938c60 100644 --- a/src/payment/bolt11.rs +++ b/src/payment/bolt11.rs @@ -242,7 +242,7 @@ impl Bolt11Payment { self.payment_store.insert(payment)?; // Persist LSP peer to make sure we reconnect on restart. - self.peer_store.add_peer(peer_info)?; + self.runtime.block_on(self.peer_store.add_peer(peer_info))?; Ok(invoice) } diff --git a/src/peer_store.rs b/src/peer_store.rs index 38d4abc4e1..52513a3dfb 100644 --- a/src/peer_store.rs +++ b/src/peer_store.rs @@ -11,7 +11,7 @@ use std::sync::{Arc, RwLock}; use bitcoin::secp256k1::PublicKey; use lightning::impl_ser_tlv_based; -use lightning::util::persist::KVStoreSync; +use lightning::util::persist::KVStore; use lightning::util::ser::{Readable, ReadableArgs, Writeable, Writer}; use crate::io::{ @@ -40,22 +40,25 @@ where Self { peers, kv_store, logger } } - pub(crate) fn add_peer(&self, peer_info: PeerInfo) -> Result<(), Error> { - let mut locked_peers = self.peers.write().expect("lock"); - - if locked_peers.contains_key(&peer_info.node_id) { - return Ok(()); - } - - locked_peers.insert(peer_info.node_id, peer_info); - self.persist_peers(&*locked_peers) + pub(crate) async fn add_peer(&self, peer_info: PeerInfo) -> Result<(), Error> { + let data = { + let mut locked_peers = self.peers.write().expect("lock"); + if locked_peers.contains_key(&peer_info.node_id) { + return Ok(()); + } + locked_peers.insert(peer_info.node_id, peer_info); + PeerStoreSerWrapper(&locked_peers).encode() + }; + self.persist_peers(data).await } - pub(crate) fn remove_peer(&self, node_id: &PublicKey) -> Result<(), Error> { - let mut locked_peers = self.peers.write().expect("lock"); - - locked_peers.remove(node_id); - self.persist_peers(&*locked_peers) + pub(crate) async fn remove_peer(&self, node_id: &PublicKey) -> Result<(), Error> { + let data = { + let mut locked_peers = self.peers.write().expect("lock"); + locked_peers.remove(node_id); + PeerStoreSerWrapper(&locked_peers).encode() + }; + self.persist_peers(data).await } pub(crate) fn list_peers(&self) -> Vec { @@ -66,15 +69,15 @@ where self.peers.read().expect("lock").get(node_id).cloned() } - fn persist_peers(&self, locked_peers: &HashMap) -> Result<(), Error> { - let data = PeerStoreSerWrapper(&*locked_peers).encode(); - KVStoreSync::write( + async fn persist_peers(&self, data: Vec) -> Result<(), Error> { + KVStore::write( &*self.kv_store, PEER_INFO_PERSISTENCE_PRIMARY_NAMESPACE, PEER_INFO_PERSISTENCE_SECONDARY_NAMESPACE, PEER_INFO_PERSISTENCE_KEY, data, ) + .await .map_err(|e| { log_error!( self.logger, @@ -158,8 +161,8 @@ mod tests { use crate::io::test_utils::InMemoryStore; use crate::types::DynStoreWrapper; - #[test] - fn peer_info_persistence() { + #[tokio::test] + async fn peer_info_persistence() { let store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); let logger = Arc::new(TestLogger::new()); let peer_store = PeerStore::new(Arc::clone(&store), Arc::clone(&logger)); @@ -170,22 +173,24 @@ mod tests { .unwrap(); let address = SocketAddress::from_str("127.0.0.1:9738").unwrap(); let expected_peer_info = PeerInfo { node_id, address }; - assert!(KVStoreSync::read( + assert!(KVStore::read( &*store, PEER_INFO_PERSISTENCE_PRIMARY_NAMESPACE, PEER_INFO_PERSISTENCE_SECONDARY_NAMESPACE, PEER_INFO_PERSISTENCE_KEY, ) + .await .is_err()); - peer_store.add_peer(expected_peer_info.clone()).unwrap(); + peer_store.add_peer(expected_peer_info.clone()).await.unwrap(); // Check we can read back what we persisted. - let persisted_bytes = KVStoreSync::read( + let persisted_bytes = KVStore::read( &*store, PEER_INFO_PERSISTENCE_PRIMARY_NAMESPACE, PEER_INFO_PERSISTENCE_SECONDARY_NAMESPACE, PEER_INFO_PERSISTENCE_KEY, ) + .await .unwrap(); let deser_peer_store = PeerStore::read(&mut &persisted_bytes[..], (Arc::clone(&store), logger)).unwrap(); From 8c949ff2058ef403f3785a347bc668c69f82a438 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Thu, 4 Jun 2026 17:50:43 +0200 Subject: [PATCH 06/21] f - Serialize peer store persistence updates Co-Authored-By: HAL 9000 --- src/peer_store.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/peer_store.rs b/src/peer_store.rs index 52513a3dfb..19993d3f7d 100644 --- a/src/peer_store.rs +++ b/src/peer_store.rs @@ -27,6 +27,7 @@ where L::Target: LdkLogger, { peers: RwLock>, + mutation_lock: tokio::sync::Mutex<()>, kv_store: Arc, logger: L, } @@ -37,10 +38,12 @@ where { pub(crate) fn new(kv_store: Arc, logger: L) -> Self { let peers = RwLock::new(HashMap::new()); - Self { peers, kv_store, logger } + let mutation_lock = tokio::sync::Mutex::new(()); + Self { peers, mutation_lock, kv_store, logger } } pub(crate) async fn add_peer(&self, peer_info: PeerInfo) -> Result<(), Error> { + let _guard = self.mutation_lock.lock().await; let data = { let mut locked_peers = self.peers.write().expect("lock"); if locked_peers.contains_key(&peer_info.node_id) { @@ -53,6 +56,7 @@ where } pub(crate) async fn remove_peer(&self, node_id: &PublicKey) -> Result<(), Error> { + let _guard = self.mutation_lock.lock().await; let data = { let mut locked_peers = self.peers.write().expect("lock"); locked_peers.remove(node_id); @@ -104,7 +108,8 @@ where let (kv_store, logger) = args; let read_peers: PeerStoreDeserWrapper = Readable::read(reader)?; let peers: RwLock> = RwLock::new(read_peers.0); - Ok(Self { peers, kv_store, logger }) + let mutation_lock = tokio::sync::Mutex::new(()); + Ok(Self { peers, mutation_lock, kv_store, logger }) } } From a7c4cbe32a89196fb5fb08dc2d8ba0e15ac26a4e Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Wed, 3 Jun 2026 10:53:05 +0200 Subject: [PATCH 07/21] Move DataStore persistence onto async KV storage Persist DataStore mutations through async KVStore operations while keeping the existing synchronous APIs bridged through the node runtime. Async event handling now awaits payment store writes directly. Co-Authored-By: HAL 9000 --- src/builder.rs | 1 + src/data_store.rs | 134 +++++++++++++++++++++++-------------- src/event.rs | 28 ++++---- src/lib.rs | 6 +- src/payment/bolt11.rs | 14 ++-- src/payment/bolt12.rs | 22 +++--- src/payment/spontaneous.rs | 14 ++-- src/wallet/mod.rs | 39 +++++++---- 8 files changed, 156 insertions(+), 102 deletions(-) diff --git a/src/builder.rs b/src/builder.rs index c217850e38..779b2bd398 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -1622,6 +1622,7 @@ fn build_with_store_internal( Arc::clone(&fee_estimator), Arc::clone(&chain_source), Arc::clone(&payment_store), + Arc::clone(&runtime), Arc::clone(&config), Arc::clone(&logger), Arc::clone(&pending_payment_store), diff --git a/src/data_store.rs b/src/data_store.rs index 2fad814221..c080667bac 100644 --- a/src/data_store.rs +++ b/src/data_store.rs @@ -9,7 +9,7 @@ use std::collections::{hash_map, HashMap}; use std::ops::Deref; use std::sync::{Arc, Mutex}; -use lightning::util::persist::KVStoreSync; +use lightning::util::persist::KVStore; use lightning::util::ser::{Readable, Writeable}; use crate::logger::{log_error, LdkLogger}; @@ -45,6 +45,7 @@ where L::Target: LdkLogger, { objects: Mutex>, + mutation_lock: tokio::sync::Mutex<()>, primary_namespace: String, secondary_namespace: String, kv_store: Arc, @@ -61,50 +62,64 @@ where ) -> Self { let objects = Mutex::new(HashMap::from_iter(objects.into_iter().map(|obj| (obj.id(), obj)))); - Self { objects, primary_namespace, secondary_namespace, kv_store, logger } + Self { + objects, + mutation_lock: tokio::sync::Mutex::new(()), + primary_namespace, + secondary_namespace, + kv_store, + logger, + } } - pub(crate) fn insert(&self, object: SO) -> Result { - let mut locked_objects = self.objects.lock().expect("lock"); + pub(crate) async fn insert(&self, object: SO) -> Result { + let _guard = self.mutation_lock.lock().await; - self.persist(&object)?; + self.persist(&object).await?; + let mut locked_objects = self.objects.lock().expect("lock"); let updated = locked_objects.insert(object.id(), object).is_some(); Ok(updated) } - pub(crate) fn insert_or_update(&self, object: SO) -> Result { - let mut locked_objects = self.objects.lock().expect("lock"); + pub(crate) async fn insert_or_update(&self, object: SO) -> Result { + let _guard = self.mutation_lock.lock().await; + let (updated, data_to_persist) = { + let mut locked_objects = self.objects.lock().expect("lock"); + match locked_objects.entry(object.id()) { + hash_map::Entry::Occupied(mut e) => { + let update = object.to_update(); + let updated = e.get_mut().update(update); + let data_to_persist = + if updated { Some(Self::encode_object(e.get())) } else { None }; + (updated, data_to_persist) + }, + hash_map::Entry::Vacant(e) => { + let data_to_persist = Self::encode_object(&object); + e.insert(object); + (true, Some(data_to_persist)) + }, + } + }; - let updated; - match locked_objects.entry(object.id()) { - hash_map::Entry::Occupied(mut e) => { - let update = object.to_update(); - updated = e.get_mut().update(update); - if updated { - self.persist(&e.get())?; - } - }, - hash_map::Entry::Vacant(e) => { - e.insert(object.clone()); - self.persist(&object)?; - updated = true; - }, + if let Some((store_key, data)) = data_to_persist { + self.persist_encoded(store_key, data).await?; } - Ok(updated) } - pub(crate) fn remove(&self, id: &SO::Id) -> Result<(), Error> { - let removed = self.objects.lock().expect("lock").remove(id).is_some(); + pub(crate) async fn remove(&self, id: &SO::Id) -> Result<(), Error> { + let _guard = self.mutation_lock.lock().await; + let removed = { self.objects.lock().expect("lock").remove(id).is_some() }; if removed { let store_key = id.encode_to_hex_str(); - KVStoreSync::remove( + KVStore::remove( &*self.kv_store, &self.primary_namespace, &self.secondary_namespace, &store_key, false, ) + .await .map_err(|e| { log_error!( self.logger, @@ -124,36 +139,49 @@ where self.objects.lock().expect("lock").get(id).cloned() } - pub(crate) fn update(&self, update: SO::Update) -> Result { - let mut locked_objects = self.objects.lock().expect("lock"); - - if let Some(object) = locked_objects.get_mut(&update.id()) { - let updated = object.update(update); - if updated { - self.persist(&object)?; - Ok(DataStoreUpdateResult::Updated) + pub(crate) async fn update(&self, update: SO::Update) -> Result { + let _guard = self.mutation_lock.lock().await; + let (res, data_to_persist) = { + let mut locked_objects = self.objects.lock().expect("lock"); + if let Some(object) = locked_objects.get_mut(&update.id()) { + let updated = object.update(update); + if updated { + (DataStoreUpdateResult::Updated, Some(Self::encode_object(object))) + } else { + (DataStoreUpdateResult::Unchanged, None) + } } else { - Ok(DataStoreUpdateResult::Unchanged) + (DataStoreUpdateResult::NotFound, None) } - } else { - Ok(DataStoreUpdateResult::NotFound) + }; + if let Some((store_key, data)) = data_to_persist { + self.persist_encoded(store_key, data).await?; } + Ok(res) } pub(crate) fn list_filter bool>(&self, f: F) -> Vec { self.objects.lock().expect("lock").values().filter(f).cloned().collect::>() } - fn persist(&self, object: &SO) -> Result<(), Error> { - let store_key = object.id().encode_to_hex_str(); - let data = object.encode(); - KVStoreSync::write( + async fn persist(&self, object: &SO) -> Result<(), Error> { + let (store_key, data) = Self::encode_object(object); + self.persist_encoded(store_key, data).await + } + + fn encode_object(object: &SO) -> (String, Vec) { + (object.id().encode_to_hex_str(), object.encode()) + } + + async fn persist_encoded(&self, store_key: String, data: Vec) -> Result<(), Error> { + KVStore::write( &*self.kv_store, &self.primary_namespace, &self.secondary_namespace, &store_key, data, ) + .await .map_err(|e| { log_error!( self.logger, @@ -238,8 +266,8 @@ mod tests { (2, data, required), }); - #[test] - fn data_is_persisted() { + #[tokio::test] + async fn data_is_persisted() { let store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); let logger = Arc::new(TestLogger::new()); let primary_namespace = "datastore_test_primary".to_string(); @@ -258,47 +286,49 @@ mod tests { let store_key = id.encode_to_hex_str(); // Check we start empty. - assert!(KVStoreSync::read(&*store, &primary_namespace, &secondary_namespace, &store_key) + assert!(KVStore::read(&*store, &primary_namespace, &secondary_namespace, &store_key) + .await .is_err()); // Check we successfully store an object and return `false` let object = TestObject { id, data: [23u8; 3] }; - assert_eq!(Ok(false), data_store.insert(object.clone())); + assert_eq!(Ok(false), data_store.insert(object.clone()).await); assert_eq!(Some(object), data_store.get(&id)); - assert!(KVStoreSync::read(&*store, &primary_namespace, &secondary_namespace, &store_key) + assert!(KVStore::read(&*store, &primary_namespace, &secondary_namespace, &store_key) + .await .is_ok()); // Test re-insertion returns `true` let mut override_object = object.clone(); override_object.data = [24u8; 3]; - assert_eq!(Ok(true), data_store.insert(override_object)); + assert_eq!(Ok(true), data_store.insert(override_object).await); assert_eq!(Some(override_object), data_store.get(&id)); // Check update returns `Updated` let update = TestObjectUpdate { id, data: [25u8; 3] }; - assert_eq!(Ok(DataStoreUpdateResult::Updated), data_store.update(update)); + assert_eq!(Ok(DataStoreUpdateResult::Updated), data_store.update(update).await); assert_eq!(data_store.get(&id).unwrap().data, [25u8; 3]); // Check no-op update yields `Unchanged` let update = TestObjectUpdate { id, data: [25u8; 3] }; - assert_eq!(Ok(DataStoreUpdateResult::Unchanged), data_store.update(update)); + assert_eq!(Ok(DataStoreUpdateResult::Unchanged), data_store.update(update).await); // Check bogus update yields `NotFound` let bogus_id = TestObjectId { id: [84u8; 4] }; let update = TestObjectUpdate { id: bogus_id, data: [12u8; 3] }; - assert_eq!(Ok(DataStoreUpdateResult::NotFound), data_store.update(update)); + assert_eq!(Ok(DataStoreUpdateResult::NotFound), data_store.update(update).await); // Check `insert_or_update` inserts unknown objects let iou_id = TestObjectId { id: [55u8; 4] }; let iou_object = TestObject { id: iou_id, data: [34u8; 3] }; - assert_eq!(Ok(true), data_store.insert_or_update(iou_object.clone())); + assert_eq!(Ok(true), data_store.insert_or_update(iou_object.clone()).await); // Check `insert_or_update` doesn't update the same object - assert_eq!(Ok(false), data_store.insert_or_update(iou_object.clone())); + assert_eq!(Ok(false), data_store.insert_or_update(iou_object.clone()).await); // Check `insert_or_update` updates if object changed let mut new_iou_object = iou_object; new_iou_object.data[0] += 1; - assert_eq!(Ok(true), data_store.insert_or_update(new_iou_object)); + assert_eq!(Ok(true), data_store.insert_or_update(new_iou_object).await); } } diff --git a/src/event.rs b/src/event.rs index c3de8a2391..bed662cebd 100644 --- a/src/event.rs +++ b/src/event.rs @@ -581,7 +581,7 @@ where } } - fn fail_claimable_payment( + async fn fail_claimable_payment( &self, payment_id: PaymentId, payment_hash: &PaymentHash, ) -> Result<(), ReplayEvent> { self.channel_manager.fail_htlc_backwards(payment_hash); @@ -591,7 +591,7 @@ where status: Some(PaymentStatus::Failed), ..PaymentDetailsUpdate::new(payment_id) }; - match self.payment_store.update(update) { + match self.payment_store.update(update).await { Ok(_) => Ok(()), Err(e) => { log_error!(self.logger, "Failed to access payment store: {}", e); @@ -738,7 +738,7 @@ where status: Some(PaymentStatus::Failed), ..PaymentDetailsUpdate::new(payment_id) }; - match self.payment_store.update(update) { + match self.payment_store.update(update).await { Ok(_) => return Ok(()), Err(e) => { log_error!(self.logger, "Failed to access payment store: {}", e); @@ -781,7 +781,7 @@ where status: Some(PaymentStatus::Failed), ..PaymentDetailsUpdate::new(payment_id) }; - match self.payment_store.update(update) { + match self.payment_store.update(update).await { Ok(_) => return Ok(()), Err(e) => { log_error!(self.logger, "Failed to access payment store: {}", e); @@ -809,7 +809,7 @@ where hex_utils::to_string(&payment_hash.0), counterparty_skimmed_fee_msat, ); - self.fail_claimable_payment(payment_id, &payment_hash)?; + self.fail_claimable_payment(payment_id, &payment_hash).await?; return Ok(()); }; @@ -821,7 +821,7 @@ where counterparty_skimmed_fee_msat, max_total_opening_fee_msat, ); - self.fail_claimable_payment(payment_id, &payment_hash)?; + self.fail_claimable_payment(payment_id, &payment_hash).await?; return Ok(()); } @@ -832,14 +832,14 @@ where counterparty_skimmed_fee_msat: Some(Some(counterparty_skimmed_fee_msat)), ..PaymentDetailsUpdate::new(payment_id) }; - match self.payment_store.update(update) { + match self.payment_store.update(update).await { Ok(_) => (), Err(e) => { log_error!(self.logger, "Failed to access payment store: {}", e); return Err(ReplayEvent()); }, }; - } + }, _ => debug_assert!(false, "We only expect the counterparty to get away with withholding fees for BOLT11 payments."), } } @@ -923,7 +923,7 @@ where PaymentStatus::Pending, ); - match self.payment_store.insert(payment) { + match self.payment_store.insert(payment).await { Ok(false) => (), Ok(true) => { log_error!( @@ -964,7 +964,7 @@ where PaymentStatus::Pending, ); - match self.payment_store.insert(payment) { + match self.payment_store.insert(payment).await { Ok(false) => (), Ok(true) => { log_error!( @@ -1004,7 +1004,7 @@ where status: Some(PaymentStatus::Failed), ..PaymentDetailsUpdate::new(payment_id) }; - match self.payment_store.update(update) { + match self.payment_store.update(update).await { Ok(_) => return Ok(()), Err(e) => { log_error!(self.logger, "Failed to access payment store: {}", e); @@ -1072,7 +1072,7 @@ where }, }; - match self.payment_store.update(update) { + match self.payment_store.update(update).await { Ok(DataStoreUpdateResult::Updated) | Ok(DataStoreUpdateResult::Unchanged) => ( // No need to do anything if the idempotent update was applied, which might // be the result of a replayed event. @@ -1134,7 +1134,7 @@ where ..PaymentDetailsUpdate::new(payment_id) }; - match self.payment_store.update(update) { + match self.payment_store.update(update).await { Ok(_) => {}, Err(e) => { log_error!(self.logger, "Failed to access payment store: {}", e); @@ -1189,7 +1189,7 @@ where status: Some(PaymentStatus::Failed), ..PaymentDetailsUpdate::new(payment_id) }; - match self.payment_store.update(update) { + match self.payment_store.update(update).await { Ok(_) => {}, Err(e) => { log_error!(self.logger, "Failed to access payment store: {}", e); diff --git a/src/lib.rs b/src/lib.rs index f20f1e7a99..03f6de64b4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -925,6 +925,7 @@ impl Node { #[cfg(not(feature = "uniffi"))] pub fn bolt12_payment(&self) -> Bolt12Payment { Bolt12Payment::new( + Arc::clone(&self.runtime), Arc::clone(&self.channel_manager), Arc::clone(&self.keys_manager), Arc::clone(&self.payment_store), @@ -941,6 +942,7 @@ impl Node { #[cfg(feature = "uniffi")] pub fn bolt12_payment(&self) -> Arc { Arc::new(Bolt12Payment::new( + Arc::clone(&self.runtime), Arc::clone(&self.channel_manager), Arc::clone(&self.keys_manager), Arc::clone(&self.payment_store), @@ -955,6 +957,7 @@ impl Node { #[cfg(not(feature = "uniffi"))] pub fn spontaneous_payment(&self) -> SpontaneousPayment { SpontaneousPayment::new( + Arc::clone(&self.runtime), Arc::clone(&self.channel_manager), Arc::clone(&self.keys_manager), Arc::clone(&self.payment_store), @@ -968,6 +971,7 @@ impl Node { #[cfg(feature = "uniffi")] pub fn spontaneous_payment(&self) -> Arc { Arc::new(SpontaneousPayment::new( + Arc::clone(&self.runtime), Arc::clone(&self.channel_manager), Arc::clone(&self.keys_manager), Arc::clone(&self.payment_store), @@ -1899,7 +1903,7 @@ impl Node { /// Remove the payment with the given id from the store. pub fn remove_payment(&self, payment_id: &PaymentId) -> Result<(), Error> { - self.payment_store.remove(&payment_id) + self.runtime.block_on(self.payment_store.remove(&payment_id)) } /// Retrieves an overview of all known balances. diff --git a/src/payment/bolt11.rs b/src/payment/bolt11.rs index f1d4938c60..97dd7a3493 100644 --- a/src/payment/bolt11.rs +++ b/src/payment/bolt11.rs @@ -158,7 +158,7 @@ impl Bolt11Payment { PaymentDirection::Inbound, PaymentStatus::Pending, ); - self.payment_store.insert(payment)?; + self.runtime.block_on(self.payment_store.insert(payment))?; Ok(invoice) } @@ -239,7 +239,7 @@ impl Bolt11Payment { PaymentDirection::Inbound, PaymentStatus::Pending, ); - self.payment_store.insert(payment)?; + self.runtime.block_on(self.payment_store.insert(payment))?; // Persist LSP peer to make sure we reconnect on restart. self.runtime.block_on(self.peer_store.add_peer(peer_info))?; @@ -341,7 +341,7 @@ impl Bolt11Payment { PaymentStatus::Pending, ); - self.payment_store.insert(payment)?; + self.runtime.block_on(self.payment_store.insert(payment))?; Ok(payment_id) }, @@ -371,7 +371,7 @@ impl Bolt11Payment { PaymentStatus::Failed, ); - self.payment_store.insert(payment)?; + self.runtime.block_on(self.payment_store.insert(payment))?; Err(Error::PaymentSendingFailed) }, } @@ -457,7 +457,7 @@ impl Bolt11Payment { PaymentDirection::Outbound, PaymentStatus::Pending, ); - self.payment_store.insert(payment)?; + self.runtime.block_on(self.payment_store.insert(payment))?; Ok(payment_id) }, @@ -488,7 +488,7 @@ impl Bolt11Payment { PaymentStatus::Failed, ); - self.payment_store.insert(payment)?; + self.runtime.block_on(self.payment_store.insert(payment))?; Err(Error::PaymentSendingFailed) }, } @@ -582,7 +582,7 @@ impl Bolt11Payment { ..PaymentDetailsUpdate::new(payment_id) }; - match self.payment_store.update(update) { + match self.runtime.block_on(self.payment_store.update(update)) { Ok(DataStoreUpdateResult::Updated) | Ok(DataStoreUpdateResult::Unchanged) => (), Ok(DataStoreUpdateResult::NotFound) => { log_error!( diff --git a/src/payment/bolt12.rs b/src/payment/bolt12.rs index 2e5a5fb451..d79aca6c24 100644 --- a/src/payment/bolt12.rs +++ b/src/payment/bolt12.rs @@ -29,6 +29,7 @@ use crate::error::Error; use crate::ffi::{maybe_deref, maybe_wrap}; use crate::logger::{log_error, log_info, LdkLogger, Logger}; use crate::payment::store::{PaymentDetails, PaymentDirection, PaymentKind, PaymentStatus}; +use crate::runtime::Runtime; use crate::types::{ChannelManager, KeysManager, PaymentStore}; #[cfg(not(feature = "uniffi"))] @@ -59,6 +60,7 @@ type HumanReadableName = Arc; /// [`Node::bolt12_payment`]: crate::Node::bolt12_payment #[cfg_attr(feature = "uniffi", derive(uniffi::Object))] pub struct Bolt12Payment { + runtime: Arc, channel_manager: Arc, keys_manager: Arc, payment_store: Arc, @@ -70,11 +72,13 @@ pub struct Bolt12Payment { impl Bolt12Payment { pub(crate) fn new( - channel_manager: Arc, keys_manager: Arc, - payment_store: Arc, config: Arc, is_running: Arc>, - logger: Arc, async_payments_role: Option, + runtime: Arc, channel_manager: Arc, + keys_manager: Arc, payment_store: Arc, config: Arc, + is_running: Arc>, logger: Arc, + async_payments_role: Option, ) -> Self { Self { + runtime, channel_manager, keys_manager, payment_store, @@ -163,7 +167,7 @@ impl Bolt12Payment { PaymentDirection::Outbound, PaymentStatus::Pending, ); - self.payment_store.insert(payment)?; + self.runtime.block_on(self.payment_store.insert(payment))?; Ok(payment_id) }, @@ -188,7 +192,7 @@ impl Bolt12Payment { PaymentDirection::Outbound, PaymentStatus::Failed, ); - self.payment_store.insert(payment)?; + self.runtime.block_on(self.payment_store.insert(payment))?; Err(Error::PaymentSendingFailed) }, } @@ -325,7 +329,7 @@ impl Bolt12Payment { PaymentDirection::Outbound, PaymentStatus::Pending, ); - self.payment_store.insert(payment)?; + self.runtime.block_on(self.payment_store.insert(payment))?; Ok(payment_id) }, @@ -350,7 +354,7 @@ impl Bolt12Payment { PaymentDirection::Outbound, PaymentStatus::Failed, ); - self.payment_store.insert(payment)?; + self.runtime.block_on(self.payment_store.insert(payment))?; Err(Error::InvoiceRequestCreationFailed) }, } @@ -457,7 +461,7 @@ impl Bolt12Payment { PaymentStatus::Pending, ); - self.payment_store.insert(payment)?; + self.runtime.block_on(self.payment_store.insert(payment))?; Ok(maybe_wrap(invoice)) } @@ -526,7 +530,7 @@ impl Bolt12Payment { PaymentStatus::Pending, ); - self.payment_store.insert(payment)?; + self.runtime.block_on(self.payment_store.insert(payment))?; Ok(maybe_wrap(refund)) } diff --git a/src/payment/spontaneous.rs b/src/payment/spontaneous.rs index 1c819582e4..45dab644d4 100644 --- a/src/payment/spontaneous.rs +++ b/src/payment/spontaneous.rs @@ -22,6 +22,7 @@ use crate::config::{Config, LDK_PAYMENT_RETRY_TIMEOUT}; use crate::error::Error; use crate::logger::{log_error, log_info, LdkLogger, Logger}; use crate::payment::store::{PaymentDetails, PaymentDirection, PaymentKind, PaymentStatus}; +use crate::runtime::Runtime; use crate::types::{ChannelManager, CustomTlvRecord, KeysManager, PaymentStore}; // The default `final_cltv_expiry_delta` we apply when not set. @@ -34,6 +35,7 @@ const LDK_DEFAULT_FINAL_CLTV_EXPIRY_DELTA: u32 = 144; /// [`Node::spontaneous_payment`]: crate::Node::spontaneous_payment #[cfg_attr(feature = "uniffi", derive(uniffi::Object))] pub struct SpontaneousPayment { + runtime: Arc, channel_manager: Arc, keys_manager: Arc, payment_store: Arc, @@ -44,11 +46,11 @@ pub struct SpontaneousPayment { impl SpontaneousPayment { pub(crate) fn new( - channel_manager: Arc, keys_manager: Arc, - payment_store: Arc, config: Arc, is_running: Arc>, - logger: Arc, + runtime: Arc, channel_manager: Arc, + keys_manager: Arc, payment_store: Arc, config: Arc, + is_running: Arc>, logger: Arc, ) -> Self { - Self { channel_manager, keys_manager, payment_store, config, is_running, logger } + Self { runtime, channel_manager, keys_manager, payment_store, config, is_running, logger } } fn send_inner( @@ -130,7 +132,7 @@ impl SpontaneousPayment { PaymentDirection::Outbound, PaymentStatus::Pending, ); - self.payment_store.insert(payment)?; + self.runtime.block_on(self.payment_store.insert(payment))?; Ok(payment_id) }, @@ -153,7 +155,7 @@ impl SpontaneousPayment { PaymentStatus::Failed, ); - self.payment_store.insert(payment)?; + self.runtime.block_on(self.payment_store.insert(payment))?; Err(Error::PaymentSendingFailed) }, } diff --git a/src/wallet/mod.rs b/src/wallet/mod.rs index 13b1f384f5..cdae34a0aa 100644 --- a/src/wallet/mod.rs +++ b/src/wallet/mod.rs @@ -58,6 +58,7 @@ use crate::payment::store::ConfirmationStatus; use crate::payment::{ PaymentDetails, PaymentDirection, PaymentKind, PaymentStatus, PendingPaymentDetails, }; +use crate::runtime::Runtime; use crate::types::{Broadcaster, PaymentStore, PendingPaymentStore}; use crate::{ChainSource, Error}; @@ -85,6 +86,7 @@ pub(crate) struct Wallet { fee_estimator: Arc, chain_source: Arc, payment_store: Arc, + runtime: Arc, config: Arc, logger: Arc, pending_payment_store: Arc, @@ -95,8 +97,8 @@ impl Wallet { wallet: bdk_wallet::PersistedWallet, wallet_persister: KVStoreWalletPersister, broadcaster: Arc, fee_estimator: Arc, chain_source: Arc, - payment_store: Arc, config: Arc, logger: Arc, - pending_payment_store: Arc, + payment_store: Arc, runtime: Arc, config: Arc, + logger: Arc, pending_payment_store: Arc, ) -> Self { let inner = Mutex::new(wallet); let persister = Mutex::new(wallet_persister); @@ -107,6 +109,7 @@ impl Wallet { fee_estimator, chain_source, payment_store, + runtime, config, logger, pending_payment_store, @@ -278,13 +281,15 @@ impl Wallet { confirmation_status, ); - self.payment_store.insert_or_update(payment.clone())?; + self.runtime.block_on(self.payment_store.insert_or_update(payment.clone()))?; if payment_status == PaymentStatus::Pending { let pending_payment = self.create_pending_payment_from_tx(payment, Vec::new()); - self.pending_payment_store.insert_or_update(pending_payment)?; + self.runtime.block_on( + self.pending_payment_store.insert_or_update(pending_payment), + )?; } }, WalletEvent::ChainTipChanged { new_tip, .. } => { @@ -310,8 +315,11 @@ impl Wallet { let payment_id = payment.details.id; if new_tip.height >= height + ANTI_REORG_DELAY - 1 { payment.details.status = PaymentStatus::Succeeded; - self.payment_store.insert_or_update(payment.details)?; - self.pending_payment_store.remove(&payment_id)?; + self.runtime.block_on( + self.payment_store.insert_or_update(payment.details), + )?; + self.runtime + .block_on(self.pending_payment_store.remove(&payment_id))?; } }, PaymentKind::Onchain { @@ -367,8 +375,9 @@ impl Wallet { ); let pending_payment = self.create_pending_payment_from_tx(payment.clone(), Vec::new()); - self.payment_store.insert_or_update(payment)?; - self.pending_payment_store.insert_or_update(pending_payment)?; + self.runtime.block_on(self.payment_store.insert_or_update(payment))?; + self.runtime + .block_on(self.pending_payment_store.insert_or_update(pending_payment))?; }, WalletEvent::TxReplaced { txid, conflicts, .. } => { let Some(payment_id) = self.find_payment_by_txid(txid) else { @@ -398,7 +407,9 @@ impl Wallet { let pending_payment_details = self.create_pending_payment_from_tx(payment, conflict_txids.clone()); - self.pending_payment_store.insert_or_update(pending_payment_details)?; + self.runtime.block_on( + self.pending_payment_store.insert_or_update(pending_payment_details), + )?; }, WalletEvent::TxDropped { txid, tx } => { let payment_id = self @@ -414,8 +425,9 @@ impl Wallet { ); let pending_payment = self.create_pending_payment_from_tx(payment.clone(), Vec::new()); - self.payment_store.insert_or_update(payment)?; - self.pending_payment_store.insert_or_update(pending_payment)?; + self.runtime.block_on(self.payment_store.insert_or_update(payment))?; + self.runtime + .block_on(self.pending_payment_store.insert_or_update(pending_payment))?; }, _ => { continue; @@ -1416,8 +1428,9 @@ impl Wallet { let pending_payment_store = self.create_pending_payment_from_tx(new_payment.clone(), Vec::new()); - self.pending_payment_store.insert_or_update(pending_payment_store)?; - self.payment_store.insert_or_update(new_payment)?; + self.runtime + .block_on(self.pending_payment_store.insert_or_update(pending_payment_store))?; + self.runtime.block_on(self.payment_store.insert_or_update(new_payment))?; log_info!(self.logger, "RBF successful: replaced {} with {}", txid, new_txid); From ff489f2d39f1ebb69fc8966893338e8724a9b7b3 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Wed, 3 Jun 2026 11:02:55 +0200 Subject: [PATCH 08/21] Move node metrics persistence onto async KV storage Persist node metric updates through async KVStore writes and await them from the chain, gossip, and scoring tasks. This removes the remaining blocking metrics writer while keeping the helper name stable. Co-Authored-By: HAL 9000 --- src/chain/bitcoind.rs | 7 +++-- src/chain/electrum.rs | 62 ++++++++++++++++++++++++------------------- src/chain/esplora.rs | 9 ++++--- src/io/utils.rs | 28 +++++++++---------- src/lib.rs | 1 + src/scoring.rs | 1 + 6 files changed, 62 insertions(+), 46 deletions(-) diff --git a/src/chain/bitcoind.rs b/src/chain/bitcoind.rs index 2582f32f64..f044eec629 100644 --- a/src/chain/bitcoind.rs +++ b/src/chain/bitcoind.rs @@ -204,6 +204,7 @@ impl BitcoindChainSource { m.latest_onchain_wallet_sync_timestamp = unix_time_secs_opt; }, ) + .await .unwrap_or_else(|e| { log_error!(self.logger, "Failed to persist node metrics: {}", e); }); @@ -451,7 +452,8 @@ impl BitcoindChainSource { update_and_persist_node_metrics(&self.node_metrics, &*self.kv_store, &*self.logger, |m| { m.latest_lightning_wallet_sync_timestamp = unix_time_secs_opt; m.latest_onchain_wallet_sync_timestamp = unix_time_secs_opt; - })?; + }) + .await?; Ok(()) } @@ -563,7 +565,8 @@ impl BitcoindChainSource { SystemTime::now().duration_since(UNIX_EPOCH).ok().map(|d| d.as_secs()); update_and_persist_node_metrics(&self.node_metrics, &*self.kv_store, &*self.logger, |m| { m.latest_fee_rate_cache_update_timestamp = unix_time_secs_opt - })?; + }) + .await?; Ok(()) } diff --git a/src/chain/electrum.rs b/src/chain/electrum.rs index 54e7fff0ca..a474cd76aa 100644 --- a/src/chain/electrum.rs +++ b/src/chain/electrum.rs @@ -129,30 +129,36 @@ impl ElectrumChainSource { let incremental_sync = self.node_metrics.read().expect("lock").latest_onchain_wallet_sync_timestamp.is_some(); - let apply_wallet_update = - |update_res: Result, now: Instant| match update_res { - Ok(update) => match onchain_wallet.apply_update(update) { - Ok(()) => { - log_debug!( - self.logger, - "{} of on-chain wallet finished in {}ms.", - if incremental_sync { "Incremental sync" } else { "Sync" }, - now.elapsed().as_millis() - ); - let unix_time_secs_opt = - SystemTime::now().duration_since(UNIX_EPOCH).ok().map(|d| d.as_secs()); - update_and_persist_node_metrics( - &self.node_metrics, - &*self.kv_store, - &*self.logger, - |m| m.latest_onchain_wallet_sync_timestamp = unix_time_secs_opt, - )?; - Ok(()) + macro_rules! apply_wallet_update { + ($update_res:expr, $now:expr) => { + match $update_res { + Ok(update) => match onchain_wallet.apply_update(update) { + Ok(()) => { + log_debug!( + self.logger, + "{} of on-chain wallet finished in {}ms.", + if incremental_sync { "Incremental sync" } else { "Sync" }, + $now.elapsed().as_millis() + ); + let unix_time_secs_opt = SystemTime::now() + .duration_since(UNIX_EPOCH) + .ok() + .map(|d| d.as_secs()); + update_and_persist_node_metrics( + &self.node_metrics, + &*self.kv_store, + &*self.logger, + |m| m.latest_onchain_wallet_sync_timestamp = unix_time_secs_opt, + ) + .await?; + Ok(()) + }, + Err(e) => Err(e), }, Err(e) => Err(e), - }, - Err(e) => Err(e), + } }; + } let cached_txs = onchain_wallet.get_cached_txs(); @@ -162,15 +168,15 @@ impl ElectrumChainSource { .get_incremental_sync_wallet_update(incremental_sync_request, cached_txs); let now = Instant::now(); - let update_res = incremental_sync_fut.await.map(|u| u.into()); - apply_wallet_update(update_res, now) + let update_res: Result = incremental_sync_fut.await.map(|u| u.into()); + apply_wallet_update!(update_res, now) } else { let full_scan_request = onchain_wallet.get_full_scan_request(); let full_scan_fut = electrum_client.get_full_scan_wallet_update(full_scan_request, cached_txs); let now = Instant::now(); - let update_res = full_scan_fut.await.map(|u| u.into()); - apply_wallet_update(update_res, now) + let update_res: Result = full_scan_fut.await.map(|u| u.into()); + apply_wallet_update!(update_res, now) }; res @@ -239,7 +245,8 @@ impl ElectrumChainSource { &*self.kv_store, &*self.logger, |m| m.latest_lightning_wallet_sync_timestamp = unix_time_secs_opt, - )?; + ) + .await?; } res @@ -270,7 +277,8 @@ impl ElectrumChainSource { SystemTime::now().duration_since(UNIX_EPOCH).ok().map(|d| d.as_secs()); update_and_persist_node_metrics(&self.node_metrics, &*self.kv_store, &*self.logger, |m| { m.latest_fee_rate_cache_update_timestamp = unix_time_secs_opt - })?; + }) + .await?; Ok(()) } diff --git a/src/chain/esplora.rs b/src/chain/esplora.rs index 5825a09849..c8ea2368c6 100644 --- a/src/chain/esplora.rs +++ b/src/chain/esplora.rs @@ -127,7 +127,8 @@ impl EsploraChainSource { &*self.kv_store, &*self.logger, |m| m.latest_onchain_wallet_sync_timestamp = unix_time_secs_opt, - )?; + ) + .await?; Ok(()) }, Err(e) => Err(e), @@ -265,7 +266,8 @@ impl EsploraChainSource { &*self.kv_store, &*self.logger, |m| m.latest_lightning_wallet_sync_timestamp = unix_time_secs_opt, - )?; + ) + .await?; Ok(()) }, Err(e) => { @@ -347,7 +349,8 @@ impl EsploraChainSource { SystemTime::now().duration_since(UNIX_EPOCH).ok().map(|d| d.as_secs()); update_and_persist_node_metrics(&self.node_metrics, &*self.kv_store, &*self.logger, |m| { m.latest_fee_rate_cache_update_timestamp = unix_time_secs_opt - })?; + }) + .await?; Ok(()) } diff --git a/src/io/utils.rs b/src/io/utils.rs index 11d1255b84..24c0d3761c 100644 --- a/src/io/utils.rs +++ b/src/io/utils.rs @@ -26,12 +26,12 @@ use lightning::routing::scoring::{ ChannelLiquidities, ProbabilisticScorer, ProbabilisticScoringDecayParameters, }; use lightning::util::persist::{ - migrate_kv_store_data, KVStore, KVStoreSync, KVSTORE_NAMESPACE_KEY_ALPHABET, - KVSTORE_NAMESPACE_KEY_MAX_LEN, NETWORK_GRAPH_PERSISTENCE_KEY, - NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, - OUTPUT_SWEEPER_PERSISTENCE_KEY, OUTPUT_SWEEPER_PERSISTENCE_PRIMARY_NAMESPACE, - OUTPUT_SWEEPER_PERSISTENCE_SECONDARY_NAMESPACE, SCORER_PERSISTENCE_KEY, - SCORER_PERSISTENCE_PRIMARY_NAMESPACE, SCORER_PERSISTENCE_SECONDARY_NAMESPACE, + migrate_kv_store_data, KVStore, KVSTORE_NAMESPACE_KEY_ALPHABET, KVSTORE_NAMESPACE_KEY_MAX_LEN, + NETWORK_GRAPH_PERSISTENCE_KEY, NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, OUTPUT_SWEEPER_PERSISTENCE_KEY, + OUTPUT_SWEEPER_PERSISTENCE_PRIMARY_NAMESPACE, OUTPUT_SWEEPER_PERSISTENCE_SECONDARY_NAMESPACE, + SCORER_PERSISTENCE_KEY, SCORER_PERSISTENCE_PRIMARY_NAMESPACE, + SCORER_PERSISTENCE_SECONDARY_NAMESPACE, }; use lightning::util::ser::{Readable, ReadableArgs, Writeable}; use lightning_persister::fs_store::v1::FilesystemStore; @@ -336,26 +336,26 @@ where } /// Take a write lock on `node_metrics`, apply `update`, and persist the result to `kv_store`. -/// -/// The write lock is held across the KV-store write, preserving the invariant that readers only -/// observe the mutation once it has been durably persisted (or the persist has failed). -pub(crate) fn update_and_persist_node_metrics( +pub(crate) async fn update_and_persist_node_metrics( node_metrics: &RwLock, kv_store: &DynStore, logger: L, update: impl FnOnce(&mut NodeMetrics), ) -> Result<(), Error> where L::Target: LdkLogger, { - let mut locked_node_metrics = node_metrics.write().expect("lock"); - update(&mut *locked_node_metrics); - let data = locked_node_metrics.encode(); - KVStoreSync::write( + let data = { + let mut locked_node_metrics = node_metrics.write().expect("lock"); + update(&mut *locked_node_metrics); + locked_node_metrics.encode() + }; + KVStore::write( &*kv_store, NODE_METRICS_PRIMARY_NAMESPACE, NODE_METRICS_SECONDARY_NAMESPACE, NODE_METRICS_KEY, data, ) + .await .map_err(|e| { log_error!( logger, diff --git a/src/lib.rs b/src/lib.rs index 03f6de64b4..d5c096c02b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -556,6 +556,7 @@ impl Node { Arc::clone(&bcast_logger), |m| m.latest_node_announcement_broadcast_timestamp = unix_time_secs_opt, ) + .await .unwrap_or_else(|e| { log_error!(bcast_logger, "Persistence failed: {}", e); }); diff --git a/src/scoring.rs b/src/scoring.rs index 8abc4eab6a..92f9bd3f2e 100644 --- a/src/scoring.rs +++ b/src/scoring.rs @@ -89,6 +89,7 @@ async fn sync_external_scores( update_and_persist_node_metrics(&node_metrics, &*kv_store, logger, |m| { m.latest_pathfinding_scores_sync_timestamp = Some(duration_since_epoch.as_secs()); }) + .await .unwrap_or_else(|e| { log_error!(logger, "Persisting node metrics failed: {}", e); }); From d71fb064c6c56283e7ba1e7a02712f3ca41db023 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Wed, 3 Jun 2026 17:24:28 +0200 Subject: [PATCH 09/21] f - Move on-chain wallet update helper out of macro Avoid introducing a temporary macro when moving node metrics persistence onto async KV storage. Co-Authored-By: HAL 9000 --- src/chain/electrum.rs | 77 ++++++++++++++++++++++++------------------- 1 file changed, 44 insertions(+), 33 deletions(-) diff --git a/src/chain/electrum.rs b/src/chain/electrum.rs index a474cd76aa..b794b40056 100644 --- a/src/chain/electrum.rs +++ b/src/chain/electrum.rs @@ -129,37 +129,6 @@ impl ElectrumChainSource { let incremental_sync = self.node_metrics.read().expect("lock").latest_onchain_wallet_sync_timestamp.is_some(); - macro_rules! apply_wallet_update { - ($update_res:expr, $now:expr) => { - match $update_res { - Ok(update) => match onchain_wallet.apply_update(update) { - Ok(()) => { - log_debug!( - self.logger, - "{} of on-chain wallet finished in {}ms.", - if incremental_sync { "Incremental sync" } else { "Sync" }, - $now.elapsed().as_millis() - ); - let unix_time_secs_opt = SystemTime::now() - .duration_since(UNIX_EPOCH) - .ok() - .map(|d| d.as_secs()); - update_and_persist_node_metrics( - &self.node_metrics, - &*self.kv_store, - &*self.logger, - |m| m.latest_onchain_wallet_sync_timestamp = unix_time_secs_opt, - ) - .await?; - Ok(()) - }, - Err(e) => Err(e), - }, - Err(e) => Err(e), - } - }; - } - let cached_txs = onchain_wallet.get_cached_txs(); let res = if incremental_sync { @@ -169,19 +138,61 @@ impl ElectrumChainSource { let now = Instant::now(); let update_res: Result = incremental_sync_fut.await.map(|u| u.into()); - apply_wallet_update!(update_res, now) + self.apply_onchain_wallet_update( + onchain_wallet.as_ref(), + incremental_sync, + update_res, + now, + ) + .await } else { let full_scan_request = onchain_wallet.get_full_scan_request(); let full_scan_fut = electrum_client.get_full_scan_wallet_update(full_scan_request, cached_txs); let now = Instant::now(); let update_res: Result = full_scan_fut.await.map(|u| u.into()); - apply_wallet_update!(update_res, now) + self.apply_onchain_wallet_update( + onchain_wallet.as_ref(), + incremental_sync, + update_res, + now, + ) + .await }; res } + async fn apply_onchain_wallet_update( + &self, onchain_wallet: &Wallet, incremental_sync: bool, + update_res: Result, now: Instant, + ) -> Result<(), Error> { + match update_res { + Ok(update) => match onchain_wallet.apply_update(update) { + Ok(()) => { + log_debug!( + self.logger, + "{} of on-chain wallet finished in {}ms.", + if incremental_sync { "Incremental sync" } else { "Sync" }, + now.elapsed().as_millis() + ); + let unix_time_secs_opt = + SystemTime::now().duration_since(UNIX_EPOCH).ok().map(|d| d.as_secs()); + update_and_persist_node_metrics( + &self.node_metrics, + &*self.kv_store, + &*self.logger, + |m| m.latest_onchain_wallet_sync_timestamp = unix_time_secs_opt, + ) + .await?; + Ok(()) + }, + Err(e) => Err(e), + }, + Err(e) => Err(e), + } + } + pub(crate) async fn sync_lightning_wallet( &self, channel_manager: Arc, chain_monitor: Arc, output_sweeper: Arc, From 1ecafce4c9b407c36be949c751b794d90547fd89 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Thu, 4 Jun 2026 17:53:34 +0200 Subject: [PATCH 10/21] f - Serialize node metrics persistence updates Co-Authored-By: HAL 9000 --- src/builder.rs | 6 +++--- src/chain/bitcoind.rs | 8 ++++---- src/chain/electrum.rs | 6 +++--- src/chain/esplora.rs | 8 ++++---- src/chain/mod.rs | 12 ++++++------ src/io/utils.rs | 7 ++++--- src/lib.rs | 29 ++++++++++++++++++++++++++++- src/scoring.rs | 10 +++++----- 8 files changed, 57 insertions(+), 29 deletions(-) diff --git a/src/builder.rs b/src/builder.rs index 779b2bd398..15f656600f 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -85,7 +85,7 @@ use crate::types::{ }; use crate::wallet::persist::KVStoreWalletPersister; use crate::wallet::Wallet; -use crate::{Node, NodeMetrics}; +use crate::{Node, NodeMetrics, PersistedNodeMetrics}; const LSPS_HARDENED_CHILD_INDEX: u32 = 577; const PERSISTER_MAX_PENDING_UPDATES: u64 = 100; @@ -1415,10 +1415,10 @@ fn build_with_store_internal( // Initialize the status fields. let node_metrics = match node_metris_res { - Ok(metrics) => Arc::new(RwLock::new(metrics)), + Ok(metrics) => Arc::new(PersistedNodeMetrics::new(metrics)), Err(e) => { if e.kind() == std::io::ErrorKind::NotFound { - Arc::new(RwLock::new(NodeMetrics::default())) + Arc::new(PersistedNodeMetrics::new(NodeMetrics::default())) } else { log_error!(logger, "Failed to read node metrics from store: {}", e); return Err(BuildError::ReadFailed); diff --git a/src/chain/bitcoind.rs b/src/chain/bitcoind.rs index f044eec629..6bfa8ffd27 100644 --- a/src/chain/bitcoind.rs +++ b/src/chain/bitcoind.rs @@ -42,7 +42,7 @@ use crate::fee_estimator::{ use crate::io::utils::update_and_persist_node_metrics; use crate::logger::{log_bytes, log_debug, log_error, log_info, log_trace, LdkLogger, Logger}; use crate::types::{ChainMonitor, ChannelManager, DynStore, Sweeper, Wallet}; -use crate::{Error, NodeMetrics}; +use crate::{Error, PersistedNodeMetrics}; const CHAIN_POLLING_INTERVAL_SECS: u64 = 2; const CHAIN_POLLING_TIMEOUT_SECS: u64 = 10; @@ -55,14 +55,14 @@ pub(super) struct BitcoindChainSource { kv_store: Arc, config: Arc, logger: Arc, - node_metrics: Arc>, + node_metrics: Arc, } impl BitcoindChainSource { pub(crate) fn new_rpc( rpc_host: String, rpc_port: u16, rpc_user: String, rpc_password: String, fee_estimator: Arc, kv_store: Arc, config: Arc, - logger: Arc, node_metrics: Arc>, + logger: Arc, node_metrics: Arc, ) -> Self { let api_client = Arc::new(BitcoindClient::new_rpc( rpc_host.clone(), @@ -89,7 +89,7 @@ impl BitcoindChainSource { rpc_host: String, rpc_port: u16, rpc_user: String, rpc_password: String, fee_estimator: Arc, kv_store: Arc, config: Arc, rest_client_config: BitcoindRestClientConfig, logger: Arc, - node_metrics: Arc>, + node_metrics: Arc, ) -> Self { let api_client = Arc::new(BitcoindClient::new_rest( rest_client_config.rest_host, diff --git a/src/chain/electrum.rs b/src/chain/electrum.rs index b794b40056..ad0ef1b7ba 100644 --- a/src/chain/electrum.rs +++ b/src/chain/electrum.rs @@ -34,7 +34,7 @@ use crate::io::utils::update_and_persist_node_metrics; use crate::logger::{log_bytes, log_debug, log_error, log_trace, LdkLogger, Logger}; use crate::runtime::Runtime; use crate::types::{ChainMonitor, ChannelManager, DynStore, Sweeper, Wallet}; -use crate::NodeMetrics; +use crate::PersistedNodeMetrics; const BDK_ELECTRUM_CLIENT_BATCH_SIZE: usize = 5; const ELECTRUM_CLIENT_NUM_RETRIES: u8 = 3; @@ -49,14 +49,14 @@ pub(super) struct ElectrumChainSource { kv_store: Arc, config: Arc, logger: Arc, - node_metrics: Arc>, + node_metrics: Arc, } impl ElectrumChainSource { pub(super) fn new( server_url: String, sync_config: ElectrumSyncConfig, fee_estimator: Arc, kv_store: Arc, config: Arc, - logger: Arc, node_metrics: Arc>, + logger: Arc, node_metrics: Arc, ) -> Self { let electrum_runtime_status = RwLock::new(ElectrumRuntimeStatus::new()); let onchain_wallet_sync_status = Mutex::new(WalletSyncStatus::Completed); diff --git a/src/chain/esplora.rs b/src/chain/esplora.rs index c8ea2368c6..eb23a395d3 100644 --- a/src/chain/esplora.rs +++ b/src/chain/esplora.rs @@ -6,7 +6,7 @@ // accordance with one or both of these licenses. use std::collections::HashMap; -use std::sync::{Arc, Mutex, RwLock}; +use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use bdk_esplora::EsploraAsyncExt; @@ -25,7 +25,7 @@ use crate::fee_estimator::{ use crate::io::utils::update_and_persist_node_metrics; use crate::logger::{log_bytes, log_debug, log_error, log_trace, LdkLogger, Logger}; use crate::types::{ChainMonitor, ChannelManager, DynStore, Sweeper, Wallet}; -use crate::{Error, NodeMetrics}; +use crate::{Error, PersistedNodeMetrics}; pub(super) struct EsploraChainSource { pub(super) sync_config: EsploraSyncConfig, @@ -37,14 +37,14 @@ pub(super) struct EsploraChainSource { kv_store: Arc, config: Arc, logger: Arc, - node_metrics: Arc>, + node_metrics: Arc, } impl EsploraChainSource { pub(crate) fn new( server_url: String, headers: HashMap, sync_config: EsploraSyncConfig, fee_estimator: Arc, kv_store: Arc, config: Arc, - logger: Arc, node_metrics: Arc>, + logger: Arc, node_metrics: Arc, ) -> Result { let mut client_builder = esplora_client::Builder::new(&server_url); client_builder = diff --git a/src/chain/mod.rs b/src/chain/mod.rs index cb8541be6a..92c4bdb641 100644 --- a/src/chain/mod.rs +++ b/src/chain/mod.rs @@ -10,7 +10,7 @@ mod electrum; mod esplora; use std::collections::HashMap; -use std::sync::{Arc, Mutex, RwLock}; +use std::sync::{Arc, Mutex}; use std::time::Duration; use bitcoin::{Script, Txid}; @@ -27,7 +27,7 @@ use crate::fee_estimator::OnchainFeeEstimator; use crate::logger::{log_debug, log_info, log_trace, LdkLogger, Logger}; use crate::runtime::Runtime; use crate::types::{Broadcaster, ChainMonitor, ChannelManager, DynStore, Sweeper, Wallet}; -use crate::{Error, NodeMetrics}; +use crate::{Error, PersistedNodeMetrics}; pub(crate) enum WalletSyncStatus { Completed, @@ -100,7 +100,7 @@ impl ChainSource { server_url: String, headers: HashMap, sync_config: EsploraSyncConfig, fee_estimator: Arc, tx_broadcaster: Arc, kv_store: Arc, config: Arc, logger: Arc, - node_metrics: Arc>, + node_metrics: Arc, ) -> Result<(Self, Option), ()> { let esplora_chain_source = EsploraChainSource::new( server_url, @@ -121,7 +121,7 @@ impl ChainSource { server_url: String, sync_config: ElectrumSyncConfig, fee_estimator: Arc, tx_broadcaster: Arc, kv_store: Arc, config: Arc, logger: Arc, - node_metrics: Arc>, + node_metrics: Arc, ) -> (Self, Option) { let electrum_chain_source = ElectrumChainSource::new( server_url, @@ -141,7 +141,7 @@ impl ChainSource { rpc_host: String, rpc_port: u16, rpc_user: String, rpc_password: String, fee_estimator: Arc, tx_broadcaster: Arc, kv_store: Arc, config: Arc, logger: Arc, - node_metrics: Arc>, + node_metrics: Arc, ) -> (Self, Option) { let bitcoind_chain_source = BitcoindChainSource::new_rpc( rpc_host, @@ -164,7 +164,7 @@ impl ChainSource { rpc_host: String, rpc_port: u16, rpc_user: String, rpc_password: String, fee_estimator: Arc, tx_broadcaster: Arc, kv_store: Arc, config: Arc, rest_client_config: BitcoindRestClientConfig, - logger: Arc, node_metrics: Arc>, + logger: Arc, node_metrics: Arc, ) -> (Self, Option) { let bitcoind_chain_source = BitcoindChainSource::new_rest( rpc_host, diff --git a/src/io/utils.rs b/src/io/utils.rs index 24c0d3761c..b13e0cf5d9 100644 --- a/src/io/utils.rs +++ b/src/io/utils.rs @@ -11,7 +11,7 @@ use std::ops::Deref; #[cfg(unix)] use std::os::unix::fs::OpenOptionsExt; use std::path::{Path, PathBuf}; -use std::sync::{Arc, RwLock}; +use std::sync::Arc; use bdk_chain::indexer::keychain_txout::ChangeSet as BdkIndexerChangeSet; use bdk_chain::local_chain::ChangeSet as BdkLocalChainChangeSet; @@ -49,7 +49,7 @@ use crate::logger::{log_error, LdkLogger, Logger}; use crate::peer_store::PeerStore; use crate::types::{Broadcaster, DynStore, KeysManager, Sweeper}; use crate::wallet::ser::{ChangeSetDeserWrapper, ChangeSetSerWrapper}; -use crate::{BuildError, Error, EventQueue, NodeMetrics}; +use crate::{BuildError, Error, EventQueue, NodeMetrics, PersistedNodeMetrics}; pub const EXTERNAL_PATHFINDING_SCORES_CACHE_KEY: &str = "external_pathfinding_scores_cache"; @@ -337,12 +337,13 @@ where /// Take a write lock on `node_metrics`, apply `update`, and persist the result to `kv_store`. pub(crate) async fn update_and_persist_node_metrics( - node_metrics: &RwLock, kv_store: &DynStore, logger: L, + node_metrics: &PersistedNodeMetrics, kv_store: &DynStore, logger: L, update: impl FnOnce(&mut NodeMetrics), ) -> Result<(), Error> where L::Target: LdkLogger, { + let _guard = node_metrics.lock_mutation().await; let data = { let mut locked_node_metrics = node_metrics.write().expect("lock"); update(&mut *locked_node_metrics); diff --git a/src/lib.rs b/src/lib.rs index d5c096c02b..e0a67cf48c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -243,7 +243,7 @@ pub struct Node { payment_store: Arc, lnurl_auth: Arc, is_running: Arc>, - node_metrics: Arc>, + node_metrics: Arc, om_mailbox: Option>, async_payments_role: Option, hrn_resolver: HRNResolver, @@ -2186,6 +2186,33 @@ impl Default for NodeMetrics { } } +pub(crate) struct PersistedNodeMetrics { + metrics: RwLock, + mutation_lock: tokio::sync::Mutex<()>, +} + +impl PersistedNodeMetrics { + pub(crate) fn new(metrics: NodeMetrics) -> Self { + Self { metrics: RwLock::new(metrics), mutation_lock: tokio::sync::Mutex::new(()) } + } + + pub(crate) fn read( + &self, + ) -> std::sync::LockResult> { + self.metrics.read() + } + + pub(crate) fn write( + &self, + ) -> std::sync::LockResult> { + self.metrics.write() + } + + pub(crate) async fn lock_mutation(&self) -> tokio::sync::MutexGuard<'_, ()> { + self.mutation_lock.lock().await + } +} + impl_ser_tlv_based!(NodeMetrics, { (0, latest_lightning_wallet_sync_timestamp, option), (1, latest_pathfinding_scores_sync_timestamp, option), diff --git a/src/scoring.rs b/src/scoring.rs index 92f9bd3f2e..401d9c3f1f 100644 --- a/src/scoring.rs +++ b/src/scoring.rs @@ -1,4 +1,4 @@ -use std::sync::{Arc, Mutex, RwLock}; +use std::sync::{Arc, Mutex}; use std::time::SystemTime; use lightning::routing::scoring::ChannelLiquidities; @@ -13,12 +13,12 @@ use crate::io::utils::write_external_pathfinding_scores_to_cache; use crate::logger::LdkLogger; use crate::runtime::Runtime; use crate::types::DynStore; -use crate::{update_and_persist_node_metrics, Logger, NodeMetrics, Scorer}; +use crate::{update_and_persist_node_metrics, Logger, PersistedNodeMetrics, Scorer}; /// Start a background task that periodically downloads scores via an external url and merges them into the local /// pathfinding scores. pub fn setup_background_pathfinding_scores_sync( - url: String, scorer: Arc>, node_metrics: Arc>, + url: String, scorer: Arc>, node_metrics: Arc, kv_store: Arc, logger: Arc, runtime: Arc, mut stop_receiver: tokio::sync::watch::Receiver<()>, ) { @@ -51,7 +51,7 @@ pub fn setup_background_pathfinding_scores_sync( } async fn sync_external_scores( - logger: &Logger, scorer: &Mutex, node_metrics: &RwLock, + logger: &Logger, scorer: &Mutex, node_metrics: &PersistedNodeMetrics, kv_store: Arc, url: &String, ) -> () { let request = bitreq::get(url) @@ -86,7 +86,7 @@ async fn sync_external_scores( .duration_since(SystemTime::UNIX_EPOCH) .expect("system time must be after Unix epoch"); scorer.lock().expect("lock").merge(liquidities, duration_since_epoch); - update_and_persist_node_metrics(&node_metrics, &*kv_store, logger, |m| { + update_and_persist_node_metrics(node_metrics, &*kv_store, logger, |m| { m.latest_pathfinding_scores_sync_timestamp = Some(duration_since_epoch.as_secs()); }) .await From a454ce114d2402bd2cfa04336374922b7387c5bf Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Wed, 3 Jun 2026 11:07:30 +0200 Subject: [PATCH 11/21] Use BDK's async wallet persister Persist the on-chain wallet through BDK's AsyncWalletPersister so wallet state writes use the async KVStore path. Existing synchronous wallet APIs keep bridging through the node runtime until their callers are made async. Co-Authored-By: HAL 9000 --- src/builder.rs | 33 ++++++++------ src/wallet/mod.rs | 46 +++++++++++-------- src/wallet/persist.rs | 103 ++++++++++++++++++++++-------------------- 3 files changed, 100 insertions(+), 82 deletions(-) diff --git a/src/builder.rs b/src/builder.rs index 15f656600f..4107ca686f 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -1537,17 +1537,18 @@ fn build_with_store_internal( let descriptor = Bip84(xprv, KeychainKind::External); let change_descriptor = Bip84(xprv, KeychainKind::Internal); - let mut wallet_persister = KVStoreWalletPersister::new( - Arc::clone(&kv_store), - Arc::clone(&runtime), - Arc::clone(&logger), - ); - let wallet_opt = BdkWallet::load() - .descriptor(KeychainKind::External, Some(descriptor.clone())) - .descriptor(KeychainKind::Internal, Some(change_descriptor.clone())) - .extract_keys() - .check_network(config.network) - .load_wallet(&mut wallet_persister) + let mut wallet_persister = + KVStoreWalletPersister::new(Arc::clone(&kv_store), Arc::clone(&logger)); + let wallet_opt = runtime + .block_on(async { + BdkWallet::load() + .descriptor(KeychainKind::External, Some(descriptor.clone())) + .descriptor(KeychainKind::Internal, Some(change_descriptor.clone())) + .extract_keys() + .check_network(config.network) + .load_wallet_async(&mut wallet_persister) + .await + }) .map_err(|e| match e { bdk_wallet::LoadWithPersistError::InvalidChangeSet( bdk_wallet::LoadError::Mismatch(bdk_wallet::LoadMismatch::Network { @@ -1571,9 +1572,13 @@ fn build_with_store_internal( let bdk_wallet = match wallet_opt { Some(wallet) => wallet, None => { - let mut wallet = BdkWallet::create(descriptor, change_descriptor) - .network(config.network) - .create_wallet(&mut wallet_persister) + let mut wallet = runtime + .block_on(async { + BdkWallet::create(descriptor, change_descriptor) + .network(config.network) + .create_wallet_async(&mut wallet_persister) + .await + }) .map_err(|e| { log_error!(logger, "Failed to set up wallet: {}", e); BuildError::WalletSetupFailed diff --git a/src/wallet/mod.rs b/src/wallet/mod.rs index cdae34a0aa..76f2aa9ce6 100644 --- a/src/wallet/mod.rs +++ b/src/wallet/mod.rs @@ -161,10 +161,12 @@ impl Wallet { })?; let mut locked_persister = self.persister.lock().expect("lock"); - locked_wallet.persist(&mut locked_persister).map_err(|e| { - log_error!(self.logger, "Failed to persist wallet: {}", e); - Error::PersistenceFailed - })?; + self.runtime.block_on(locked_wallet.persist_async(&mut locked_persister)).map_err( + |e| { + log_error!(self.logger, "Failed to persist wallet: {}", e); + Error::PersistenceFailed + }, + )?; Ok(()) }, @@ -214,7 +216,7 @@ impl Wallet { })?; let mut locked_persister = self.persister.lock().expect("lock"); - locked_wallet.persist(&mut locked_persister).map_err(|e| { + self.runtime.block_on(locked_wallet.persist_async(&mut locked_persister)).map_err(|e| { log_error!(self.logger, "Failed to persist wallet: {}", e); Error::PersistenceFailed })?; @@ -474,7 +476,7 @@ impl Wallet { } let mut locked_persister = self.persister.lock().expect("lock"); - locked_wallet.persist(&mut locked_persister).map_err(|e| { + self.runtime.block_on(locked_wallet.persist_async(&mut locked_persister)).map_err(|e| { log_error!(self.logger, "Failed to persist wallet: {}", e); Error::PersistenceFailed })?; @@ -492,7 +494,7 @@ impl Wallet { let mut locked_persister = self.persister.lock().expect("lock"); let address_info = locked_wallet.reveal_next_address(KeychainKind::External); - locked_wallet.persist(&mut locked_persister).map_err(|e| { + self.runtime.block_on(locked_wallet.persist_async(&mut locked_persister)).map_err(|e| { log_error!(self.logger, "Failed to persist wallet: {}", e); Error::PersistenceFailed })?; @@ -504,7 +506,7 @@ impl Wallet { let mut locked_persister = self.persister.lock().expect("lock"); let address_info = locked_wallet.next_unused_address(KeychainKind::Internal); - locked_wallet.persist(&mut locked_persister).map_err(|e| { + self.runtime.block_on(locked_wallet.persist_async(&mut locked_persister)).map_err(|e| { log_error!(self.logger, "Failed to persist wallet: {}", e); Error::PersistenceFailed })?; @@ -516,7 +518,7 @@ impl Wallet { let mut locked_persister = self.persister.lock().expect("lock"); locked_wallet.cancel_tx(tx); - locked_wallet.persist(&mut locked_persister).map_err(|e| { + self.runtime.block_on(locked_wallet.persist_async(&mut locked_persister)).map_err(|e| { log_error!(self.logger, "Failed to persist wallet: {}", e); Error::PersistenceFailed })?; @@ -854,10 +856,12 @@ impl Wallet { } let mut locked_persister = self.persister.lock().expect("lock"); - locked_wallet.persist(&mut locked_persister).map_err(|e| { - log_error!(self.logger, "Failed to persist wallet: {}", e); - Error::PersistenceFailed - })?; + self.runtime.block_on(locked_wallet.persist_async(&mut locked_persister)).map_err( + |e| { + log_error!(self.logger, "Failed to persist wallet: {}", e); + Error::PersistenceFailed + }, + )?; psbt.extract_tx().map_err(|e| { log_error!(self.logger, "Failed to extract transaction: {}", e); @@ -972,10 +976,12 @@ impl Wallet { .find(|txout| must_pay_to.iter().all(|output| output != txout)); if change_output.is_some() { - locked_wallet.persist(&mut locked_persister).map_err(|e| { - log_error!(self.logger, "Failed to persist wallet: {}", e); - () - })?; + self.runtime.block_on(locked_wallet.persist_async(&mut locked_persister)).map_err( + |e| { + log_error!(self.logger, "Failed to persist wallet: {}", e); + () + }, + )?; } Ok(CoinSelection { confirmed_utxos, change_output }) @@ -1080,7 +1086,7 @@ impl Wallet { let mut locked_persister = self.persister.lock().expect("lock"); let address_info = locked_wallet.next_unused_address(KeychainKind::Internal); - locked_wallet.persist(&mut locked_persister).map_err(|e| { + self.runtime.block_on(locked_wallet.persist_async(&mut locked_persister)).map_err(|e| { log_error!(self.logger, "Failed to persist wallet: {}", e); () })?; @@ -1399,7 +1405,7 @@ impl Wallet { } let mut locked_persister = self.persister.lock().expect("lock"); - locked_wallet.persist(&mut locked_persister).map_err(|e| { + self.runtime.block_on(locked_wallet.persist_async(&mut locked_persister)).map_err(|e| { log_error!(self.logger, "Failed to persist wallet after fee bump of {}: {}", txid, e); Error::PersistenceFailed })?; @@ -1501,7 +1507,7 @@ impl Listen for Wallet { }; let mut locked_persister = self.persister.lock().expect("lock"); - match locked_wallet.persist(&mut locked_persister) { + match self.runtime.block_on(locked_wallet.persist_async(&mut locked_persister)) { Ok(_) => (), Err(e) => { log_error!(self.logger, "Failed to persist on-chain wallet: {}", e); diff --git a/src/wallet/persist.rs b/src/wallet/persist.rs index cc50057808..364dc4b475 100644 --- a/src/wallet/persist.rs +++ b/src/wallet/persist.rs @@ -5,10 +5,12 @@ // http://opensource.org/licenses/MIT>, at your option. You may not use this file except in // accordance with one or both of these licenses. +use std::future::Future; +use std::pin::Pin; use std::sync::Arc; use bdk_chain::Merge; -use bdk_wallet::{ChangeSet, WalletPersister}; +use bdk_wallet::{AsyncWalletPersister, ChangeSet}; use crate::io::utils::{ read_bdk_wallet_change_set, write_bdk_wallet_change_descriptor, write_bdk_wallet_descriptor, @@ -16,34 +18,26 @@ use crate::io::utils::{ write_bdk_wallet_tx_graph, }; use crate::logger::{log_error, LdkLogger, Logger}; -use crate::runtime::Runtime; use crate::types::DynStore; pub(crate) struct KVStoreWalletPersister { latest_change_set: Option, kv_store: Arc, - runtime: Arc, logger: Arc, } impl KVStoreWalletPersister { - pub(crate) fn new(kv_store: Arc, runtime: Arc, logger: Arc) -> Self { - Self { latest_change_set: None, kv_store, runtime, logger } + pub(crate) fn new(kv_store: Arc, logger: Arc) -> Self { + Self { latest_change_set: None, kv_store, logger } } -} - -impl WalletPersister for KVStoreWalletPersister { - type Error = std::io::Error; - fn initialize(persister: &mut Self) -> Result { + async fn initialize_inner(&mut self) -> Result { // Return immediately if we have already been initialized. - if let Some(latest_change_set) = persister.latest_change_set.as_ref() { + if let Some(latest_change_set) = self.latest_change_set.as_ref() { return Ok(latest_change_set.clone()); } - let change_set_opt = persister - .runtime - .block_on(read_bdk_wallet_change_set(&*persister.kv_store, &*persister.logger))?; + let change_set_opt = read_bdk_wallet_change_set(&*self.kv_store, &*self.logger).await?; let change_set = match change_set_opt { Some(persisted_change_set) => persisted_change_set, @@ -54,18 +48,21 @@ impl WalletPersister for KVStoreWalletPersister { ChangeSet::default() }, }; - persister.latest_change_set = Some(change_set.clone()); + self.latest_change_set = Some(change_set.clone()); Ok(change_set) } - fn persist(persister: &mut Self, change_set: &ChangeSet) -> Result<(), Self::Error> { + async fn persist_inner(&mut self, change_set: &ChangeSet) -> Result<(), std::io::Error> { if change_set.is_empty() { return Ok(()); } + let kv_store = Arc::clone(&self.kv_store); + let logger = Arc::clone(&self.logger); + // We're allowed to fail here if we're not initialized, BDK docs state: "This method can fail if the // persister is not initialized." - let latest_change_set = persister.latest_change_set.as_mut().ok_or_else(|| { + let latest_change_set = self.latest_change_set.as_mut().ok_or_else(|| { std::io::Error::new( std::io::ErrorKind::Other, "Wallet must be initialized before calling persist", @@ -80,7 +77,7 @@ impl WalletPersister for KVStoreWalletPersister { { debug_assert!(false, "Wallet descriptor must never change"); log_error!( - persister.logger, + logger, "Wallet change set doesn't match persisted descriptor. This should never happen." ); return Err(std::io::Error::new( @@ -89,11 +86,7 @@ impl WalletPersister for KVStoreWalletPersister { )); } else { latest_change_set.descriptor = Some(descriptor.clone()); - persister.runtime.block_on(write_bdk_wallet_descriptor( - &descriptor, - &*persister.kv_store, - &*persister.logger, - ))?; + write_bdk_wallet_descriptor(&descriptor, &*kv_store, Arc::clone(&logger)).await?; } } @@ -103,7 +96,7 @@ impl WalletPersister for KVStoreWalletPersister { { debug_assert!(false, "Wallet change_descriptor must never change"); log_error!( - persister.logger, + logger, "Wallet change set doesn't match persisted change_descriptor. This should never happen." ); return Err(std::io::Error::new( @@ -112,11 +105,12 @@ impl WalletPersister for KVStoreWalletPersister { )); } else { latest_change_set.change_descriptor = Some(change_descriptor.clone()); - persister.runtime.block_on(write_bdk_wallet_change_descriptor( + write_bdk_wallet_change_descriptor( &change_descriptor, - &*persister.kv_store, - &*persister.logger, - ))?; + &*kv_store, + Arc::clone(&logger), + ) + .await?; } } @@ -124,7 +118,7 @@ impl WalletPersister for KVStoreWalletPersister { if latest_change_set.network.is_some() && latest_change_set.network != Some(network) { debug_assert!(false, "Wallet network must never change"); log_error!( - persister.logger, + logger, "Wallet change set doesn't match persisted network. This should never happen." ); return Err(std::io::Error::new( @@ -133,11 +127,7 @@ impl WalletPersister for KVStoreWalletPersister { )); } else { latest_change_set.network = Some(network); - persister.runtime.block_on(write_bdk_wallet_network( - &network, - &*persister.kv_store, - &*persister.logger, - ))?; + write_bdk_wallet_network(&network, &*kv_store, Arc::clone(&logger)).await?; } } @@ -157,31 +147,48 @@ impl WalletPersister for KVStoreWalletPersister { // particular order. if !change_set.indexer.is_empty() { latest_change_set.indexer.merge(change_set.indexer.clone()); - persister.runtime.block_on(write_bdk_wallet_indexer( - &latest_change_set.indexer, - &*persister.kv_store, - Arc::clone(&persister.logger), - ))?; + write_bdk_wallet_indexer(&latest_change_set.indexer, &*kv_store, Arc::clone(&logger)) + .await?; } if !change_set.tx_graph.is_empty() { latest_change_set.tx_graph.merge(change_set.tx_graph.clone()); - persister.runtime.block_on(write_bdk_wallet_tx_graph( - &latest_change_set.tx_graph, - &*persister.kv_store, - Arc::clone(&persister.logger), - ))?; + write_bdk_wallet_tx_graph(&latest_change_set.tx_graph, &*kv_store, Arc::clone(&logger)) + .await?; } if !change_set.local_chain.is_empty() { latest_change_set.local_chain.merge(change_set.local_chain.clone()); - persister.runtime.block_on(write_bdk_wallet_local_chain( + write_bdk_wallet_local_chain( &latest_change_set.local_chain, - &*persister.kv_store, - Arc::clone(&persister.logger), - ))?; + &*kv_store, + Arc::clone(&logger), + ) + .await?; } Ok(()) } } + +impl AsyncWalletPersister for KVStoreWalletPersister { + type Error = std::io::Error; + + fn initialize<'a>( + persister: &'a mut Self, + ) -> Pin> + Send + 'a>> + where + Self: 'a, + { + Box::pin(persister.initialize_inner()) + } + + fn persist<'a>( + persister: &'a mut Self, change_set: &'a ChangeSet, + ) -> Pin> + Send + 'a>> + where + Self: 'a, + { + Box::pin(persister.persist_inner(change_set)) + } +} From de48a6057be12065030e8b70a3223a37f7443ab3 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Wed, 3 Jun 2026 11:09:53 +0200 Subject: [PATCH 12/21] Use async KVStore migration for filesystem stores Open filesystem stores through the async LDK migration helper so v1-to-v2 store migration no longer depends on the blocking KVStoreSync migration path. Co-Authored-By: HAL 9000 --- src/builder.rs | 5 +++-- src/io/utils.rs | 51 +++++++++++++++++++++++++------------------------ 2 files changed, 29 insertions(+), 27 deletions(-) diff --git a/src/builder.rs b/src/builder.rs index 4107ca686f..9fd7f832c6 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -697,11 +697,12 @@ impl NodeBuilder { /// [`FilesystemStoreV2`]: lightning_persister::fs_store::v2::FilesystemStoreV2 pub fn build_with_fs_store(&self, node_entropy: NodeEntropy) -> Result { let logger = setup_logger(&self.log_writer_config, &self.config)?; + let runtime = self.setup_runtime(&logger)?; let mut storage_dir_path: PathBuf = self.config.storage_dir_path.clone().into(); storage_dir_path.push("fs_store"); - let kv_store = open_or_migrate_fs_store(storage_dir_path)?; - self.build_with_store_and_logger(node_entropy, kv_store, logger) + let kv_store = runtime.block_on(open_or_migrate_fs_store(storage_dir_path))?; + self.build_with_store_runtime_and_logger(node_entropy, kv_store, runtime, logger) } /// Builds a [`Node`] instance with a [VSS] backend and according to the options diff --git a/src/io/utils.rs b/src/io/utils.rs index b13e0cf5d9..cf3905332d 100644 --- a/src/io/utils.rs +++ b/src/io/utils.rs @@ -26,12 +26,12 @@ use lightning::routing::scoring::{ ChannelLiquidities, ProbabilisticScorer, ProbabilisticScoringDecayParameters, }; use lightning::util::persist::{ - migrate_kv_store_data, KVStore, KVSTORE_NAMESPACE_KEY_ALPHABET, KVSTORE_NAMESPACE_KEY_MAX_LEN, - NETWORK_GRAPH_PERSISTENCE_KEY, NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, - NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, OUTPUT_SWEEPER_PERSISTENCE_KEY, - OUTPUT_SWEEPER_PERSISTENCE_PRIMARY_NAMESPACE, OUTPUT_SWEEPER_PERSISTENCE_SECONDARY_NAMESPACE, - SCORER_PERSISTENCE_KEY, SCORER_PERSISTENCE_PRIMARY_NAMESPACE, - SCORER_PERSISTENCE_SECONDARY_NAMESPACE, + migrate_kv_store_data_async, KVStore, KVSTORE_NAMESPACE_KEY_ALPHABET, + KVSTORE_NAMESPACE_KEY_MAX_LEN, NETWORK_GRAPH_PERSISTENCE_KEY, + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + OUTPUT_SWEEPER_PERSISTENCE_KEY, OUTPUT_SWEEPER_PERSISTENCE_PRIMARY_NAMESPACE, + OUTPUT_SWEEPER_PERSISTENCE_SECONDARY_NAMESPACE, SCORER_PERSISTENCE_KEY, + SCORER_PERSISTENCE_PRIMARY_NAMESPACE, SCORER_PERSISTENCE_SECONDARY_NAMESPACE, }; use lightning::util::ser::{Readable, ReadableArgs, Writeable}; use lightning_persister::fs_store::v1::FilesystemStore; @@ -632,7 +632,7 @@ pub(crate) async fn read_bdk_wallet_change_set( /// If the directory contains v1 data (files at the top level), the data is migrated to v2 format /// in a temporary directory, the original is renamed to `fs_store_v1_backup`, and the migrated /// directory is moved into place. -pub(crate) fn open_or_migrate_fs_store( +pub(crate) async fn open_or_migrate_fs_store( storage_dir_path: PathBuf, ) -> Result { let parent_dir = storage_dir_path.parent().ok_or(BuildError::StoragePathAccessFailed)?; @@ -647,14 +647,15 @@ pub(crate) fn open_or_migrate_fs_store( Ok(store) => Ok(store), Err(FilesystemStoreV2Error::V1DataDetected(_)) => { // The directory contains v1 data, migrate to v2. - let mut v1_store = FilesystemStore::new(storage_dir_path.clone()); + let v1_store = FilesystemStore::new(storage_dir_path.clone()); let v2_dir = fs_store_sibling_path(&storage_dir_path, "fs_store_v2_migrating"); fs::create_dir_all(v2_dir.clone()).map_err(|_| BuildError::StoragePathAccessFailed)?; - let mut v2_store = FilesystemStoreV2::new(v2_dir.clone()) + let v2_store = FilesystemStoreV2::new(v2_dir.clone()) .map_err(|_| BuildError::KVStoreSetupFailed)?; - migrate_kv_store_data(&mut v1_store, &mut v2_store) + migrate_kv_store_data_async(&v1_store, &v2_store) + .await .map_err(|_| BuildError::KVStoreSetupFailed)?; // Swap directories: rename v1 out of the way, move v2 into place. @@ -739,15 +740,15 @@ mod tests { assert_eq!(expected_seed_bytes, read_seed_bytes); } - #[test] - fn fs_store_migration_recovers_before_v1_backup_rename() { + #[tokio::test] + async fn fs_store_migration_recovers_before_v1_backup_rename() { let fs_store_path = fs_store_path(); let mut v1_store = write_v1_test_data(&fs_store_path); let v2_migrating_path = sibling_path(&fs_store_path, "fs_store_v2_migrating"); let mut v2_store = FilesystemStoreV2::new(v2_migrating_path.clone()).unwrap(); migrate_kv_store_data(&mut v1_store, &mut v2_store).unwrap(); - let migrated_store = open_or_migrate_fs_store(fs_store_path.clone()).unwrap(); + let migrated_store = open_or_migrate_fs_store(fs_store_path.clone()).await.unwrap(); assert_eq!( KVStoreSync::read( &migrated_store, @@ -762,8 +763,8 @@ mod tests { assert!(!v2_migrating_path.exists()); } - #[test] - fn fs_store_migration_recovers_after_v1_backup_rename() { + #[tokio::test] + async fn fs_store_migration_recovers_after_v1_backup_rename() { let fs_store_path = fs_store_path(); let mut v1_store = write_v1_test_data(&fs_store_path); let v2_migrating_path = sibling_path(&fs_store_path, "fs_store_v2_migrating"); @@ -773,7 +774,7 @@ mod tests { let backup_path = sibling_path(&fs_store_path, "fs_store_v1_backup"); fs::rename(&fs_store_path, backup_path).unwrap(); - let migrated_store = open_or_migrate_fs_store(fs_store_path.clone()).unwrap(); + let migrated_store = open_or_migrate_fs_store(fs_store_path.clone()).await.unwrap(); assert_eq!( KVStoreSync::read( &migrated_store, @@ -788,8 +789,8 @@ mod tests { assert!(!v2_migrating_path.exists()); } - #[test] - fn fs_store_migration_recovers_after_v2_rename() { + #[tokio::test] + async fn fs_store_migration_recovers_after_v2_rename() { let fs_store_path = fs_store_path(); let mut v1_store = write_v1_test_data(&fs_store_path); let v2_migrating_path = sibling_path(&fs_store_path, "fs_store_v2_migrating"); @@ -800,7 +801,7 @@ mod tests { fs::rename(&fs_store_path, &backup_path).unwrap(); fs::rename(&v2_migrating_path, &fs_store_path).unwrap(); - let migrated_store = open_or_migrate_fs_store(fs_store_path.clone()).unwrap(); + let migrated_store = open_or_migrate_fs_store(fs_store_path.clone()).await.unwrap(); assert_eq!( KVStoreSync::read( &migrated_store, @@ -816,15 +817,15 @@ mod tests { assert!(!v2_migrating_path.exists()); } - #[test] - fn fs_store_migration_recovers_backup_without_migrating_dir() { + #[tokio::test] + async fn fs_store_migration_recovers_backup_without_migrating_dir() { let fs_store_path = fs_store_path(); write_v1_test_data(&fs_store_path); let backup_path = sibling_path(&fs_store_path, "fs_store_v1_backup"); fs::rename(&fs_store_path, backup_path).unwrap(); - let migrated_store = open_or_migrate_fs_store(fs_store_path.clone()).unwrap(); + let migrated_store = open_or_migrate_fs_store(fs_store_path.clone()).await.unwrap(); assert_eq!( KVStoreSync::read( &migrated_store, @@ -839,8 +840,8 @@ mod tests { assert!(!sibling_path(&fs_store_path, "fs_store_v1_backup").exists()); } - #[test] - fn fs_store_migration_recovers_unexpected_migrating_dir_without_backup() { + #[tokio::test] + async fn fs_store_migration_recovers_unexpected_migrating_dir_without_backup() { let fs_store_path = fs_store_path(); let v2_migrating_path = sibling_path(&fs_store_path, "fs_store_v2_migrating"); let v2_store = FilesystemStoreV2::new(v2_migrating_path.clone()).unwrap(); @@ -853,7 +854,7 @@ mod tests { ) .unwrap(); - let migrated_store = open_or_migrate_fs_store(fs_store_path.clone()).unwrap(); + let migrated_store = open_or_migrate_fs_store(fs_store_path.clone()).await.unwrap(); assert_eq!( KVStoreSync::read( &migrated_store, From 0b95f04e56f4aedb001a42456733c98940eb0dd3 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Wed, 3 Jun 2026 11:57:31 +0200 Subject: [PATCH 13/21] Move test InMemoryStore into shared module Move the existing in-memory test store into a shared module without changing its behavior. This lets integration tests reuse it while keeping the later async TestSyncStore change separate. Co-Authored-By: HAL 9000 --- src/io/in_memory_store.rs | 223 ++++++++++++++++++++++++++++++++++++++ src/io/test_utils.rs | 220 +------------------------------------ 2 files changed, 228 insertions(+), 215 deletions(-) create mode 100644 src/io/in_memory_store.rs diff --git a/src/io/in_memory_store.rs b/src/io/in_memory_store.rs new file mode 100644 index 0000000000..9ab5751152 --- /dev/null +++ b/src/io/in_memory_store.rs @@ -0,0 +1,223 @@ +// This file is Copyright its original authors, visible in version control history. +// +// This file is licensed under the Apache License, Version 2.0 or the MIT license , at your option. You may not use this file except in +// accordance with one or both of these licenses. + +use std::collections::{hash_map, HashMap}; +use std::future::Future; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Mutex; + +use lightning::io; +use lightning::util::persist::{ + KVStore, KVStoreSync, PageToken, PaginatedKVStore, PaginatedKVStoreSync, PaginatedListResponse, +}; + +const IN_MEMORY_PAGE_SIZE: usize = 50; + +pub struct InMemoryStore { + persisted_bytes: Mutex>>>, + creation_counter: AtomicU64, + creation_times: Mutex>>, +} + +impl InMemoryStore { + pub fn new() -> Self { + let persisted_bytes = Mutex::new(HashMap::new()); + let creation_counter = AtomicU64::new(1); + let creation_times = Mutex::new(HashMap::new()); + Self { persisted_bytes, creation_counter, creation_times } + } + + fn read_internal( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, + ) -> io::Result> { + let persisted_lock = self.persisted_bytes.lock().unwrap(); + let prefixed = format!("{primary_namespace}/{secondary_namespace}"); + + if let Some(outer_ref) = persisted_lock.get(&prefixed) { + if let Some(inner_ref) = outer_ref.get(key) { + let bytes = inner_ref.clone(); + Ok(bytes) + } else { + Err(io::Error::new(io::ErrorKind::NotFound, "Key not found")) + } + } else { + Err(io::Error::new(io::ErrorKind::NotFound, "Namespace not found")) + } + } + + fn write_internal( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, + ) -> io::Result<()> { + let mut persisted_lock = self.persisted_bytes.lock().unwrap(); + + let prefixed = format!("{primary_namespace}/{secondary_namespace}"); + let outer_e = persisted_lock.entry(prefixed.clone()).or_insert(HashMap::new()); + outer_e.insert(key.to_string(), buf); + + let mut ct_lock = self.creation_times.lock().unwrap(); + let ct_ns = ct_lock.entry(prefixed).or_insert(HashMap::new()); + ct_ns + .entry(key.to_string()) + .or_insert_with(|| self.creation_counter.fetch_add(1, Ordering::Relaxed)); + + Ok(()) + } + + fn remove_internal( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, _lazy: bool, + ) -> io::Result<()> { + let mut persisted_lock = self.persisted_bytes.lock().unwrap(); + + let prefixed = format!("{primary_namespace}/{secondary_namespace}"); + if let Some(outer_ref) = persisted_lock.get_mut(&prefixed) { + outer_ref.remove(&key.to_string()); + } + + let mut ct_lock = self.creation_times.lock().unwrap(); + if let Some(ct_ns) = ct_lock.get_mut(&prefixed) { + ct_ns.remove(key); + } + + Ok(()) + } + + fn list_internal( + &self, primary_namespace: &str, secondary_namespace: &str, + ) -> io::Result> { + let mut persisted_lock = self.persisted_bytes.lock().unwrap(); + + let prefixed = format!("{primary_namespace}/{secondary_namespace}"); + match persisted_lock.entry(prefixed) { + hash_map::Entry::Occupied(e) => Ok(e.get().keys().cloned().collect()), + hash_map::Entry::Vacant(_) => Ok(Vec::new()), + } + } +} + +impl KVStore for InMemoryStore { + fn read( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, + ) -> impl Future, io::Error>> + 'static + Send { + let res = self.read_internal(&primary_namespace, &secondary_namespace, &key); + async move { res } + } + + fn write( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, + ) -> impl Future> + 'static + Send { + let res = self.write_internal(&primary_namespace, &secondary_namespace, &key, buf); + async move { res } + } + + fn remove( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, + ) -> impl Future> + 'static + Send { + let res = self.remove_internal(&primary_namespace, &secondary_namespace, &key, lazy); + async move { res } + } + + fn list( + &self, primary_namespace: &str, secondary_namespace: &str, + ) -> impl Future, io::Error>> + 'static + Send { + let res = self.list_internal(primary_namespace, secondary_namespace); + async move { res } + } +} + +impl KVStoreSync for InMemoryStore { + fn read( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, + ) -> io::Result> { + self.read_internal(primary_namespace, secondary_namespace, key) + } + + fn write( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, + ) -> io::Result<()> { + self.write_internal(primary_namespace, secondary_namespace, key, buf) + } + + fn remove( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, + ) -> io::Result<()> { + self.remove_internal(primary_namespace, secondary_namespace, key, lazy) + } + + fn list(&self, primary_namespace: &str, secondary_namespace: &str) -> io::Result> { + self.list_internal(primary_namespace, secondary_namespace) + } +} + +impl InMemoryStore { + fn list_paginated_internal( + &self, primary_namespace: &str, secondary_namespace: &str, page_token: Option, + ) -> io::Result { + let ct_lock = self.creation_times.lock().unwrap(); + let prefixed = format!("{primary_namespace}/{secondary_namespace}"); + + let ct_ns = match ct_lock.get(&prefixed) { + Some(m) => m, + None => { + return Ok(PaginatedListResponse { keys: Vec::new(), next_page_token: None }); + }, + }; + + let mut entries: Vec<(&String, &u64)> = ct_ns.iter().collect(); + entries.sort_by(|a, b| b.1.cmp(a.1)); + + let start_idx = if let Some(ref token) = page_token { + let token_sort_order: u64 = token + .as_str() + .parse() + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "Invalid page token"))?; + + entries + .iter() + .position(|(_, sort_order)| **sort_order < token_sort_order) + .unwrap_or(entries.len()) + } else { + 0 + }; + + let mut page: Vec<(&String, &u64)> = + entries[start_idx..].iter().take(IN_MEMORY_PAGE_SIZE + 1).cloned().collect(); + + let has_more = page.len() > IN_MEMORY_PAGE_SIZE; + page.truncate(IN_MEMORY_PAGE_SIZE); + + let next_page_token = if has_more { + let (_, last_sort_order) = page.last().unwrap(); + Some(PageToken::new(last_sort_order.to_string())) + } else { + None + }; + + let page: Vec = page.into_iter().map(|(k, _)| k.clone()).collect(); + + Ok(PaginatedListResponse { keys: page, next_page_token }) + } +} + +impl PaginatedKVStoreSync for InMemoryStore { + fn list_paginated( + &self, primary_namespace: &str, secondary_namespace: &str, page_token: Option, + ) -> io::Result { + self.list_paginated_internal(primary_namespace, secondary_namespace, page_token) + } +} + +impl PaginatedKVStore for InMemoryStore { + fn list_paginated( + &self, primary_namespace: &str, secondary_namespace: &str, page_token: Option, + ) -> impl Future> + 'static + Send { + let res = self.list_paginated_internal(primary_namespace, secondary_namespace, page_token); + async move { res } + } +} + +unsafe impl Sync for InMemoryStore {} +unsafe impl Send for InMemoryStore {} diff --git a/src/io/test_utils.rs b/src/io/test_utils.rs index eed8c3e2da..0b16e525ed 100644 --- a/src/io/test_utils.rs +++ b/src/io/test_utils.rs @@ -5,15 +5,10 @@ // http://opensource.org/licenses/MIT>, at your option. You may not use this file except in // accordance with one or both of these licenses. -use std::collections::{hash_map, HashMap}; -use std::future::Future; use std::panic::RefUnwindSafe; use std::path::PathBuf; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::Mutex; use lightning::events::ClosureReason; -use lightning::io; use lightning::ln::functional_test_utils::{ check_added_monitors, check_closed_broadcast, check_closed_event, connect_block, create_announced_chan_between_nodes, create_chanmon_cfgs, create_dummy_block, create_network, @@ -21,13 +16,15 @@ use lightning::ln::functional_test_utils::{ TestChanMonCfg, }; use lightning::util::persist::{ - KVStore, KVStoreSync, MonitorUpdatingPersister, PageToken, PaginatedKVStore, - PaginatedKVStoreSync, PaginatedListResponse, KVSTORE_NAMESPACE_KEY_MAX_LEN, + KVStoreSync, MonitorUpdatingPersister, KVSTORE_NAMESPACE_KEY_MAX_LEN, }; use lightning::util::test_utils; use rand::distr::Alphanumeric; use rand::{rng, Rng}; +#[path = "in_memory_store.rs"] +mod in_memory_store; + type TestMonitorUpdatePersister<'a, K> = MonitorUpdatingPersister< &'a K, &'a test_utils::TestLogger, @@ -39,214 +36,7 @@ type TestMonitorUpdatePersister<'a, K> = MonitorUpdatingPersister< const EXPECTED_UPDATES_PER_PAYMENT: u64 = 5; -const IN_MEMORY_PAGE_SIZE: usize = 50; - -pub struct InMemoryStore { - persisted_bytes: Mutex>>>, - creation_counter: AtomicU64, - creation_times: Mutex>>, -} - -impl InMemoryStore { - pub fn new() -> Self { - let persisted_bytes = Mutex::new(HashMap::new()); - let creation_counter = AtomicU64::new(1); - let creation_times = Mutex::new(HashMap::new()); - Self { persisted_bytes, creation_counter, creation_times } - } - - fn read_internal( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, - ) -> io::Result> { - let persisted_lock = self.persisted_bytes.lock().unwrap(); - let prefixed = format!("{primary_namespace}/{secondary_namespace}"); - - if let Some(outer_ref) = persisted_lock.get(&prefixed) { - if let Some(inner_ref) = outer_ref.get(key) { - let bytes = inner_ref.clone(); - Ok(bytes) - } else { - Err(io::Error::new(io::ErrorKind::NotFound, "Key not found")) - } - } else { - Err(io::Error::new(io::ErrorKind::NotFound, "Namespace not found")) - } - } - - fn write_internal( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, - ) -> io::Result<()> { - let mut persisted_lock = self.persisted_bytes.lock().unwrap(); - - let prefixed = format!("{primary_namespace}/{secondary_namespace}"); - let outer_e = persisted_lock.entry(prefixed.clone()).or_insert(HashMap::new()); - outer_e.insert(key.to_string(), buf); - - // Only assign creation time on first write (not on update) - let mut ct_lock = self.creation_times.lock().unwrap(); - let ct_ns = ct_lock.entry(prefixed).or_insert(HashMap::new()); - ct_ns - .entry(key.to_string()) - .or_insert_with(|| self.creation_counter.fetch_add(1, Ordering::Relaxed)); - - Ok(()) - } - - fn remove_internal( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, _lazy: bool, - ) -> io::Result<()> { - let mut persisted_lock = self.persisted_bytes.lock().unwrap(); - - let prefixed = format!("{primary_namespace}/{secondary_namespace}"); - if let Some(outer_ref) = persisted_lock.get_mut(&prefixed) { - outer_ref.remove(&key.to_string()); - } - - // Remove creation time entry - let mut ct_lock = self.creation_times.lock().unwrap(); - if let Some(ct_ns) = ct_lock.get_mut(&prefixed) { - ct_ns.remove(key); - } - - Ok(()) - } - - fn list_internal( - &self, primary_namespace: &str, secondary_namespace: &str, - ) -> io::Result> { - let mut persisted_lock = self.persisted_bytes.lock().unwrap(); - - let prefixed = format!("{primary_namespace}/{secondary_namespace}"); - match persisted_lock.entry(prefixed) { - hash_map::Entry::Occupied(e) => Ok(e.get().keys().cloned().collect()), - hash_map::Entry::Vacant(_) => Ok(Vec::new()), - } - } -} - -impl KVStore for InMemoryStore { - fn read( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, - ) -> impl Future, io::Error>> + 'static + Send { - let res = self.read_internal(&primary_namespace, &secondary_namespace, &key); - async move { res } - } - fn write( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, - ) -> impl Future> + 'static + Send { - let res = self.write_internal(&primary_namespace, &secondary_namespace, &key, buf); - async move { res } - } - fn remove( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, - ) -> impl Future> + 'static + Send { - let res = self.remove_internal(&primary_namespace, &secondary_namespace, &key, lazy); - async move { res } - } - fn list( - &self, primary_namespace: &str, secondary_namespace: &str, - ) -> impl Future, io::Error>> + 'static + Send { - let res = self.list_internal(primary_namespace, secondary_namespace); - async move { res } - } -} - -impl KVStoreSync for InMemoryStore { - fn read( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, - ) -> io::Result> { - self.read_internal(primary_namespace, secondary_namespace, key) - } - - fn write( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, - ) -> io::Result<()> { - self.write_internal(primary_namespace, secondary_namespace, key, buf) - } - - fn remove( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, - ) -> io::Result<()> { - self.remove_internal(primary_namespace, secondary_namespace, key, lazy) - } - - fn list(&self, primary_namespace: &str, secondary_namespace: &str) -> io::Result> { - self.list_internal(primary_namespace, secondary_namespace) - } -} - -impl InMemoryStore { - fn list_paginated_internal( - &self, primary_namespace: &str, secondary_namespace: &str, page_token: Option, - ) -> io::Result { - let ct_lock = self.creation_times.lock().unwrap(); - let prefixed = format!("{primary_namespace}/{secondary_namespace}"); - - let ct_ns = match ct_lock.get(&prefixed) { - Some(m) => m, - None => { - return Ok(PaginatedListResponse { keys: Vec::new(), next_page_token: None }); - }, - }; - - // Build list of (key, sort_order) sorted by sort_order DESC (newest first). - let mut entries: Vec<(&String, &u64)> = ct_ns.iter().collect(); - entries.sort_by(|a, b| b.1.cmp(a.1)); - - // Apply page token filter - let start_idx = if let Some(ref token) = page_token { - let token_sort_order: u64 = token - .as_str() - .parse() - .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "Invalid page token"))?; - - entries - .iter() - .position(|(_, sort_order)| **sort_order < token_sort_order) - .unwrap_or(entries.len()) - } else { - 0 - }; - - // Fetch one extra entry beyond page size to determine whether a next page exists. - let mut page: Vec<(&String, &u64)> = - entries[start_idx..].iter().take(IN_MEMORY_PAGE_SIZE + 1).cloned().collect(); - - let has_more = page.len() > IN_MEMORY_PAGE_SIZE; - page.truncate(IN_MEMORY_PAGE_SIZE); - - let next_page_token = if has_more { - let (_, last_sort_order) = page.last().unwrap(); - Some(PageToken::new(last_sort_order.to_string())) - } else { - None - }; - - let page: Vec = page.into_iter().map(|(k, _)| k.clone()).collect(); - - Ok(PaginatedListResponse { keys: page, next_page_token }) - } -} - -impl PaginatedKVStoreSync for InMemoryStore { - fn list_paginated( - &self, primary_namespace: &str, secondary_namespace: &str, page_token: Option, - ) -> io::Result { - self.list_paginated_internal(primary_namespace, secondary_namespace, page_token) - } -} - -impl PaginatedKVStore for InMemoryStore { - fn list_paginated( - &self, primary_namespace: &str, secondary_namespace: &str, page_token: Option, - ) -> impl Future> + 'static + Send { - let res = self.list_paginated_internal(primary_namespace, secondary_namespace, page_token); - async move { res } - } -} - -unsafe impl Sync for InMemoryStore {} -unsafe impl Send for InMemoryStore {} +pub(crate) use in_memory_store::InMemoryStore; pub(crate) fn random_storage_path() -> PathBuf { let mut temp_path = std::env::temp_dir(); From 45d52bb08dd31d0c672403e87b2ffe9f0d21e02d Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Wed, 3 Jun 2026 17:36:17 +0200 Subject: [PATCH 14/21] f - Preserve moved InMemoryStore code exactly Keep the shared test store move as a pure code move by restoring the original comments and spacing. Co-Authored-By: HAL 9000 --- src/io/in_memory_store.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/io/in_memory_store.rs b/src/io/in_memory_store.rs index 9ab5751152..8d0d24a4e9 100644 --- a/src/io/in_memory_store.rs +++ b/src/io/in_memory_store.rs @@ -58,6 +58,7 @@ impl InMemoryStore { let outer_e = persisted_lock.entry(prefixed.clone()).or_insert(HashMap::new()); outer_e.insert(key.to_string(), buf); + // Only assign creation time on first write (not on update) let mut ct_lock = self.creation_times.lock().unwrap(); let ct_ns = ct_lock.entry(prefixed).or_insert(HashMap::new()); ct_ns @@ -77,6 +78,7 @@ impl InMemoryStore { outer_ref.remove(&key.to_string()); } + // Remove creation time entry let mut ct_lock = self.creation_times.lock().unwrap(); if let Some(ct_ns) = ct_lock.get_mut(&prefixed) { ct_ns.remove(key); @@ -105,21 +107,18 @@ impl KVStore for InMemoryStore { let res = self.read_internal(&primary_namespace, &secondary_namespace, &key); async move { res } } - fn write( &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, ) -> impl Future> + 'static + Send { let res = self.write_internal(&primary_namespace, &secondary_namespace, &key, buf); async move { res } } - fn remove( &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, ) -> impl Future> + 'static + Send { let res = self.remove_internal(&primary_namespace, &secondary_namespace, &key, lazy); async move { res } } - fn list( &self, primary_namespace: &str, secondary_namespace: &str, ) -> impl Future, io::Error>> + 'static + Send { @@ -166,9 +165,11 @@ impl InMemoryStore { }, }; + // Build list of (key, sort_order) sorted by sort_order DESC (newest first). let mut entries: Vec<(&String, &u64)> = ct_ns.iter().collect(); entries.sort_by(|a, b| b.1.cmp(a.1)); + // Apply page token filter let start_idx = if let Some(ref token) = page_token { let token_sort_order: u64 = token .as_str() @@ -183,6 +184,7 @@ impl InMemoryStore { 0 }; + // Fetch one extra entry beyond page size to determine whether a next page exists. let mut page: Vec<(&String, &u64)> = entries[start_idx..].iter().take(IN_MEMORY_PAGE_SIZE + 1).cloned().collect(); From d781d910e62564f0ce59459c5c646600ca086b4c Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Wed, 3 Jun 2026 11:58:16 +0200 Subject: [PATCH 15/21] Move test store checks onto async KV storage Exercise async KVStore operations in TestSyncStore and filesystem migration tests while keeping the temporary sync comparison path until the final KVStoreSync removal. Also route pathfinding score export through async KVStore reads. Co-Authored-By: HAL 9000 --- src/io/utils.rs | 45 ++++++---- src/lib.rs | 31 +++---- tests/common/mod.rs | 211 +++++++++++++++++++++++++++++++++++--------- 3 files changed, 211 insertions(+), 76 deletions(-) diff --git a/src/io/utils.rs b/src/io/utils.rs index cf3905332d..4657688f51 100644 --- a/src/io/utils.rs +++ b/src/io/utils.rs @@ -719,7 +719,7 @@ mod tests { use std::fs; use std::path::{Path, PathBuf}; - use lightning::util::persist::{migrate_kv_store_data, KVStoreSync}; + use lightning::util::persist::{migrate_kv_store_data_async, KVStore}; use lightning_persister::fs_store::v1::FilesystemStore; use lightning_persister::fs_store::v2::FilesystemStoreV2; @@ -743,19 +743,20 @@ mod tests { #[tokio::test] async fn fs_store_migration_recovers_before_v1_backup_rename() { let fs_store_path = fs_store_path(); - let mut v1_store = write_v1_test_data(&fs_store_path); + let v1_store = write_v1_test_data(&fs_store_path).await; let v2_migrating_path = sibling_path(&fs_store_path, "fs_store_v2_migrating"); - let mut v2_store = FilesystemStoreV2::new(v2_migrating_path.clone()).unwrap(); - migrate_kv_store_data(&mut v1_store, &mut v2_store).unwrap(); + let v2_store = FilesystemStoreV2::new(v2_migrating_path.clone()).unwrap(); + migrate_kv_store_data_async(&v1_store, &v2_store).await.unwrap(); let migrated_store = open_or_migrate_fs_store(fs_store_path.clone()).await.unwrap(); assert_eq!( - KVStoreSync::read( + KVStore::read( &migrated_store, TEST_PRIMARY_NAMESPACE, TEST_SECONDARY_NAMESPACE, TEST_KEY ) + .await .unwrap(), TEST_VALUE ); @@ -766,22 +767,23 @@ mod tests { #[tokio::test] async fn fs_store_migration_recovers_after_v1_backup_rename() { let fs_store_path = fs_store_path(); - let mut v1_store = write_v1_test_data(&fs_store_path); + let v1_store = write_v1_test_data(&fs_store_path).await; let v2_migrating_path = sibling_path(&fs_store_path, "fs_store_v2_migrating"); - let mut v2_store = FilesystemStoreV2::new(v2_migrating_path.clone()).unwrap(); - migrate_kv_store_data(&mut v1_store, &mut v2_store).unwrap(); + let v2_store = FilesystemStoreV2::new(v2_migrating_path.clone()).unwrap(); + migrate_kv_store_data_async(&v1_store, &v2_store).await.unwrap(); let backup_path = sibling_path(&fs_store_path, "fs_store_v1_backup"); fs::rename(&fs_store_path, backup_path).unwrap(); let migrated_store = open_or_migrate_fs_store(fs_store_path.clone()).await.unwrap(); assert_eq!( - KVStoreSync::read( + KVStore::read( &migrated_store, TEST_PRIMARY_NAMESPACE, TEST_SECONDARY_NAMESPACE, TEST_KEY ) + .await .unwrap(), TEST_VALUE ); @@ -792,10 +794,10 @@ mod tests { #[tokio::test] async fn fs_store_migration_recovers_after_v2_rename() { let fs_store_path = fs_store_path(); - let mut v1_store = write_v1_test_data(&fs_store_path); + let v1_store = write_v1_test_data(&fs_store_path).await; let v2_migrating_path = sibling_path(&fs_store_path, "fs_store_v2_migrating"); - let mut v2_store = FilesystemStoreV2::new(v2_migrating_path.clone()).unwrap(); - migrate_kv_store_data(&mut v1_store, &mut v2_store).unwrap(); + let v2_store = FilesystemStoreV2::new(v2_migrating_path.clone()).unwrap(); + migrate_kv_store_data_async(&v1_store, &v2_store).await.unwrap(); let backup_path = sibling_path(&fs_store_path, "fs_store_v1_backup"); fs::rename(&fs_store_path, &backup_path).unwrap(); @@ -803,12 +805,13 @@ mod tests { let migrated_store = open_or_migrate_fs_store(fs_store_path.clone()).await.unwrap(); assert_eq!( - KVStoreSync::read( + KVStore::read( &migrated_store, TEST_PRIMARY_NAMESPACE, TEST_SECONDARY_NAMESPACE, TEST_KEY ) + .await .unwrap(), TEST_VALUE ); @@ -820,19 +823,20 @@ mod tests { #[tokio::test] async fn fs_store_migration_recovers_backup_without_migrating_dir() { let fs_store_path = fs_store_path(); - write_v1_test_data(&fs_store_path); + write_v1_test_data(&fs_store_path).await; let backup_path = sibling_path(&fs_store_path, "fs_store_v1_backup"); fs::rename(&fs_store_path, backup_path).unwrap(); let migrated_store = open_or_migrate_fs_store(fs_store_path.clone()).await.unwrap(); assert_eq!( - KVStoreSync::read( + KVStore::read( &migrated_store, TEST_PRIMARY_NAMESPACE, TEST_SECONDARY_NAMESPACE, TEST_KEY ) + .await .unwrap(), TEST_VALUE ); @@ -845,23 +849,25 @@ mod tests { let fs_store_path = fs_store_path(); let v2_migrating_path = sibling_path(&fs_store_path, "fs_store_v2_migrating"); let v2_store = FilesystemStoreV2::new(v2_migrating_path.clone()).unwrap(); - KVStoreSync::write( + KVStore::write( &v2_store, TEST_PRIMARY_NAMESPACE, TEST_SECONDARY_NAMESPACE, TEST_KEY, TEST_VALUE.to_vec(), ) + .await .unwrap(); let migrated_store = open_or_migrate_fs_store(fs_store_path.clone()).await.unwrap(); assert_eq!( - KVStoreSync::read( + KVStore::read( &migrated_store, TEST_PRIMARY_NAMESPACE, TEST_SECONDARY_NAMESPACE, TEST_KEY ) + .await .unwrap(), TEST_VALUE ); @@ -881,15 +887,16 @@ mod tests { sibling_path } - fn write_v1_test_data(fs_store_path: &Path) -> FilesystemStore { + async fn write_v1_test_data(fs_store_path: &Path) -> FilesystemStore { let v1_store = FilesystemStore::new(fs_store_path.to_path_buf()); - KVStoreSync::write( + KVStore::write( &v1_store, TEST_PRIMARY_NAMESPACE, TEST_SECONDARY_NAMESPACE, TEST_KEY, TEST_VALUE.to_vec(), ) + .await .unwrap(); v1_store } diff --git a/src/lib.rs b/src/lib.rs index e0a67cf48c..5ad0492989 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -156,7 +156,7 @@ use lightning::ln::msgs::{BaseMessageHandler, SocketAddress}; use lightning::ln::peer_handler::CustomMessageHandler; use lightning::routing::gossip::NodeAlias; use lightning::sign::EntropySource; -use lightning::util::persist::KVStoreSync; +use lightning::util::persist::KVStore; use lightning::util::wallet_utils::{Input, Wallet as LdkWallet}; use lightning_background_processor::process_events_async; pub use lightning_invoice; @@ -2062,20 +2062,21 @@ impl Node { /// Exports the current state of the scorer. The result can be shared with and merged by light nodes that only have /// a limited view of the network. pub fn export_pathfinding_scores(&self) -> Result, Error> { - KVStoreSync::read( - &*self.kv_store, - lightning::util::persist::SCORER_PERSISTENCE_PRIMARY_NAMESPACE, - lightning::util::persist::SCORER_PERSISTENCE_SECONDARY_NAMESPACE, - lightning::util::persist::SCORER_PERSISTENCE_KEY, - ) - .map_err(|e| { - log_error!( - self.logger, - "Failed to access store while exporting pathfinding scores: {}", - e - ); - Error::PersistenceFailed - }) + self.runtime + .block_on(KVStore::read( + &*self.kv_store, + lightning::util::persist::SCORER_PERSISTENCE_PRIMARY_NAMESPACE, + lightning::util::persist::SCORER_PERSISTENCE_SECONDARY_NAMESPACE, + lightning::util::persist::SCORER_PERSISTENCE_KEY, + )) + .map_err(|e| { + log_error!( + self.logger, + "Failed to access store while exporting pathfinding scores: {}", + e + ); + Error::PersistenceFailed + }) } /// Return the features used in node announcement. diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 30d9a4387e..f2e7191ae4 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -24,7 +24,7 @@ use std::future::Future; use std::path::PathBuf; use std::str::FromStr; use std::sync::atomic::{AtomicU16, Ordering}; -use std::sync::{Arc, RwLock}; +use std::sync::Arc; use std::time::Duration; use bitcoin::hashes::hex::FromHex; @@ -51,7 +51,6 @@ use lightning::io; use lightning::ln::msgs::SocketAddress; use lightning::routing::gossip::NodeAlias; use lightning::util::persist::{KVStore, KVStoreSync}; -use lightning::util::test_utils::TestStore; use lightning_invoice::{Bolt11InvoiceDescription, Description}; use lightning_persister::fs_store::v1::FilesystemStore; use lightning_types::payment::{PaymentHash, PaymentPreimage}; @@ -60,6 +59,10 @@ use rand::distr::Alphanumeric; use rand::{rng, Rng}; use serde_json::{json, Value}; +#[path = "../../src/io/in_memory_store.rs"] +mod in_memory_store; +use in_memory_store::InMemoryStore; + /// Shared timeout (in seconds) for waiting on LDK events and external node operations. pub(crate) const INTEROP_TIMEOUT_SECS: u64 = 60; @@ -1660,16 +1663,9 @@ impl KVStore for TestSyncStore { let secondary_namespace = secondary_namespace.to_string(); let key = key.to_string(); let inner = Arc::clone(&self.inner); - let fut = tokio::task::spawn_blocking(move || { - inner.read_internal(&primary_namespace, &secondary_namespace, &key) - }); - async move { - fut.await.unwrap_or_else(|e| { - let msg = format!("Failed to IO operation due join error: {}", e); - Err(io::Error::new(io::ErrorKind::Other, msg)) - }) - } + async move { inner.read_internal_async(&primary_namespace, &secondary_namespace, &key).await } } + fn write( &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, ) -> impl Future> + 'static + Send { @@ -1677,16 +1673,11 @@ impl KVStore for TestSyncStore { let secondary_namespace = secondary_namespace.to_string(); let key = key.to_string(); let inner = Arc::clone(&self.inner); - let fut = tokio::task::spawn_blocking(move || { - inner.write_internal(&primary_namespace, &secondary_namespace, &key, buf) - }); async move { - fut.await.unwrap_or_else(|e| { - let msg = format!("Failed to IO operation due join error: {}", e); - Err(io::Error::new(io::ErrorKind::Other, msg)) - }) + inner.write_internal_async(&primary_namespace, &secondary_namespace, &key, buf).await } } + fn remove( &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, ) -> impl Future> + 'static + Send { @@ -1694,31 +1685,18 @@ impl KVStore for TestSyncStore { let secondary_namespace = secondary_namespace.to_string(); let key = key.to_string(); let inner = Arc::clone(&self.inner); - let fut = tokio::task::spawn_blocking(move || { - inner.remove_internal(&primary_namespace, &secondary_namespace, &key, lazy) - }); async move { - fut.await.unwrap_or_else(|e| { - let msg = format!("Failed to IO operation due join error: {}", e); - Err(io::Error::new(io::ErrorKind::Other, msg)) - }) + inner.remove_internal_async(&primary_namespace, &secondary_namespace, &key, lazy).await } } + fn list( &self, primary_namespace: &str, secondary_namespace: &str, ) -> impl Future, io::Error>> + 'static + Send { let primary_namespace = primary_namespace.to_string(); let secondary_namespace = secondary_namespace.to_string(); let inner = Arc::clone(&self.inner); - let fut = tokio::task::spawn_blocking(move || { - inner.list_internal(&primary_namespace, &secondary_namespace) - }); - async move { - fut.await.unwrap_or_else(|e| { - let msg = format!("Failed to IO operation due join error: {}", e); - Err(io::Error::new(io::ErrorKind::Other, msg)) - }) - } + async move { inner.list_internal_async(&primary_namespace, &secondary_namespace).await } } } @@ -1749,15 +1727,15 @@ impl KVStoreSync for TestSyncStore { } struct TestSyncStoreInner { - serializer: RwLock<()>, - test_store: TestStore, + serializer: tokio::sync::RwLock<()>, + test_store: InMemoryStore, fs_store: FilesystemStore, sqlite_store: SqliteStore, } impl TestSyncStoreInner { fn new(dest_dir: PathBuf) -> Self { - let serializer = RwLock::new(()); + let serializer = tokio::sync::RwLock::new(()); let mut fs_dir = dest_dir.clone(); fs_dir.push("fs_store"); let fs_store = FilesystemStore::new(fs_dir); @@ -1769,7 +1747,7 @@ impl TestSyncStoreInner { Some("test_sync_table".to_string()), ) .unwrap(); - let test_store = TestStore::new(false); + let test_store = InMemoryStore::new(); Self { serializer, fs_store, sqlite_store, test_store } } @@ -1803,10 +1781,159 @@ impl TestSyncStoreInner { } } + async fn do_list_async( + &self, primary_namespace: &str, secondary_namespace: &str, + ) -> lightning::io::Result> { + let fs_res = KVStore::list(&self.fs_store, primary_namespace, secondary_namespace).await; + let sqlite_res = + KVStore::list(&self.sqlite_store, primary_namespace, secondary_namespace).await; + let test_res = + KVStore::list(&self.test_store, primary_namespace, secondary_namespace).await; + + match fs_res { + Ok(mut list) => { + list.sort(); + + let mut sqlite_list = sqlite_res.unwrap(); + sqlite_list.sort(); + assert_eq!(list, sqlite_list); + + let mut test_list = test_res.unwrap(); + test_list.sort(); + assert_eq!(list, test_list); + + Ok(list) + }, + Err(e) => { + assert!(sqlite_res.is_err()); + assert!(test_res.is_err()); + Err(e) + }, + } + } + + async fn list_internal_async( + &self, primary_namespace: &str, secondary_namespace: &str, + ) -> lightning::io::Result> { + let _guard = self.serializer.read().await; + self.do_list_async(primary_namespace, secondary_namespace).await + } + + async fn read_internal_async( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, + ) -> lightning::io::Result> { + let _guard = self.serializer.read().await; + + let fs_res = + KVStore::read(&self.fs_store, primary_namespace, secondary_namespace, key).await; + let sqlite_res = + KVStore::read(&self.sqlite_store, primary_namespace, secondary_namespace, key).await; + let test_res = + KVStore::read(&self.test_store, primary_namespace, secondary_namespace, key).await; + + match fs_res { + Ok(read) => { + assert_eq!(read, sqlite_res.unwrap()); + assert_eq!(read, test_res.unwrap()); + Ok(read) + }, + Err(e) => { + assert!(sqlite_res.is_err()); + assert_eq!(e.kind(), unsafe { sqlite_res.unwrap_err_unchecked().kind() }); + assert!(test_res.is_err()); + assert_eq!(e.kind(), unsafe { test_res.unwrap_err_unchecked().kind() }); + Err(e) + }, + } + } + + async fn write_internal_async( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, + ) -> lightning::io::Result<()> { + let _guard = self.serializer.write().await; + let fs_res = KVStore::write( + &self.fs_store, + primary_namespace, + secondary_namespace, + key, + buf.clone(), + ) + .await; + let sqlite_res = KVStore::write( + &self.sqlite_store, + primary_namespace, + secondary_namespace, + key, + buf.clone(), + ) + .await; + let test_res = KVStore::write( + &self.test_store, + primary_namespace, + secondary_namespace, + key, + buf.clone(), + ) + .await; + + assert!(self + .do_list_async(primary_namespace, secondary_namespace) + .await + .unwrap() + .contains(&key.to_string())); + + match fs_res { + Ok(()) => { + assert!(sqlite_res.is_ok()); + assert!(test_res.is_ok()); + Ok(()) + }, + Err(e) => { + assert!(sqlite_res.is_err()); + assert!(test_res.is_err()); + Err(e) + }, + } + } + + async fn remove_internal_async( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, + ) -> lightning::io::Result<()> { + let _guard = self.serializer.write().await; + let fs_res = + KVStore::remove(&self.fs_store, primary_namespace, secondary_namespace, key, lazy) + .await; + let sqlite_res = + KVStore::remove(&self.sqlite_store, primary_namespace, secondary_namespace, key, lazy) + .await; + let test_res = + KVStore::remove(&self.test_store, primary_namespace, secondary_namespace, key, lazy) + .await; + + assert!(!self + .do_list_async(primary_namespace, secondary_namespace) + .await + .unwrap() + .contains(&key.to_string())); + + match fs_res { + Ok(()) => { + assert!(sqlite_res.is_ok()); + assert!(test_res.is_ok()); + Ok(()) + }, + Err(e) => { + assert!(sqlite_res.is_err()); + assert!(test_res.is_err()); + Err(e) + }, + } + } + fn read_internal( &self, primary_namespace: &str, secondary_namespace: &str, key: &str, ) -> lightning::io::Result> { - let _guard = self.serializer.read().unwrap(); + let _guard = self.serializer.blocking_read(); let fs_res = KVStoreSync::read(&self.fs_store, primary_namespace, secondary_namespace, key); let sqlite_res = @@ -1833,7 +1960,7 @@ impl TestSyncStoreInner { fn write_internal( &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, ) -> lightning::io::Result<()> { - let _guard = self.serializer.write().unwrap(); + let _guard = self.serializer.blocking_write(); let fs_res = KVStoreSync::write( &self.fs_store, primary_namespace, @@ -1878,7 +2005,7 @@ impl TestSyncStoreInner { fn remove_internal( &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, ) -> lightning::io::Result<()> { - let _guard = self.serializer.write().unwrap(); + let _guard = self.serializer.blocking_write(); let fs_res = KVStoreSync::remove(&self.fs_store, primary_namespace, secondary_namespace, key, lazy); let sqlite_res = KVStoreSync::remove( @@ -1918,7 +2045,7 @@ impl TestSyncStoreInner { fn list_internal( &self, primary_namespace: &str, secondary_namespace: &str, ) -> lightning::io::Result> { - let _guard = self.serializer.read().unwrap(); + let _guard = self.serializer.blocking_read(); self.do_list(primary_namespace, secondary_namespace) } } From fe2f09f3600d59a46413d865248e64eb06f63e92 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Wed, 3 Jun 2026 11:29:24 +0200 Subject: [PATCH 16/21] Remove blocking KV store support Drop the remaining synchronous KV store trait bounds and implementations. After the preceding migrations, custom stores only need to provide async KVStore persistence. Co-Authored-By: HAL 9000 --- src/builder.rs | 16 +-- src/io/in_memory_store.rs | 36 +---- src/io/postgres_store/mod.rs | 178 ++++++++--------------- src/io/sqlite_store/migrations.rs | 24 ++-- src/io/sqlite_store/mod.rs | 212 ++++++++++----------------- src/io/test_utils.rs | 228 ++++++++++++++++++++++++------ src/io/vss_store.rs | 122 +--------------- src/lib.rs | 2 +- src/types.rs | 80 +---------- tests/common/mod.rs | 177 +---------------------- 10 files changed, 358 insertions(+), 717 deletions(-) diff --git a/src/builder.rs b/src/builder.rs index 9fd7f832c6..c88c867cc1 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -81,7 +81,7 @@ use crate::tx_broadcaster::TransactionBroadcaster; use crate::types::{ AsyncPersister, ChainMonitor, ChannelManager, DynStore, DynStoreRef, DynStoreWrapper, GossipSync, Graph, HRNResolver, KeysManager, MessageRouter, OnionMessenger, PaymentStore, - PeerManager, PendingPaymentStore, SyncAndAsyncKVStore, + PeerManager, PendingPaymentStore, }; use crate::wallet::persist::KVStoreWalletPersister; use crate::wallet::Wallet; @@ -176,17 +176,17 @@ pub enum BuildError { RuntimeSetupFailed, /// We failed to read data from the [`KVStore`]. /// - /// [`KVStore`]: lightning::util::persist::KVStoreSync + /// [`KVStore`]: lightning::util::persist::KVStore ReadFailed, /// We failed to write data to the [`KVStore`]. /// - /// [`KVStore`]: lightning::util::persist::KVStoreSync + /// [`KVStore`]: lightning::util::persist::KVStore WriteFailed, /// We failed to access the given `storage_dir_path`. StoragePathAccessFailed, /// We failed to setup our [`KVStore`]. /// - /// [`KVStore`]: lightning::util::persist::KVStoreSync + /// [`KVStore`]: lightning::util::persist::KVStore KVStoreSetupFailed, /// We failed to setup the onchain wallet. WalletSetupFailed, @@ -826,7 +826,7 @@ impl NodeBuilder { } /// Builds a [`Node`] instance according to the options previously configured. - pub fn build_with_store( + pub fn build_with_store( &self, node_entropy: NodeEntropy, kv_store: S, ) -> Result { let logger = setup_logger(&self.log_writer_config, &self.config)?; @@ -845,14 +845,14 @@ impl NodeBuilder { } } - fn build_with_store_and_logger( + fn build_with_store_and_logger( &self, node_entropy: NodeEntropy, kv_store: S, logger: Arc, ) -> Result { let runtime = self.setup_runtime(&logger)?; self.build_with_store_runtime_and_logger(node_entropy, kv_store, runtime, logger) } - fn build_with_store_runtime_and_logger( + fn build_with_store_runtime_and_logger( &self, node_entropy: NodeEntropy, kv_store: S, runtime: Arc, logger: Arc, ) -> Result { let seed_bytes = node_entropy.to_seed_bytes(); @@ -1346,7 +1346,7 @@ impl ArcedNodeBuilder { /// Builds a [`Node`] instance according to the options previously configured. // Note that the generics here don't actually work for Uniffi, but we don't currently expose // this so its not needed. - pub fn build_with_store( + pub fn build_with_store( &self, node_entropy: Arc, kv_store: S, ) -> Result, BuildError> { self.inner.read().expect("lock").build_with_store(*node_entropy, kv_store).map(Arc::new) diff --git a/src/io/in_memory_store.rs b/src/io/in_memory_store.rs index 8d0d24a4e9..8b7d41c843 100644 --- a/src/io/in_memory_store.rs +++ b/src/io/in_memory_store.rs @@ -11,9 +11,7 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Mutex; use lightning::io; -use lightning::util::persist::{ - KVStore, KVStoreSync, PageToken, PaginatedKVStore, PaginatedKVStoreSync, PaginatedListResponse, -}; +use lightning::util::persist::{KVStore, PageToken, PaginatedKVStore, PaginatedListResponse}; const IN_MEMORY_PAGE_SIZE: usize = 50; @@ -127,30 +125,6 @@ impl KVStore for InMemoryStore { } } -impl KVStoreSync for InMemoryStore { - fn read( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, - ) -> io::Result> { - self.read_internal(primary_namespace, secondary_namespace, key) - } - - fn write( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, - ) -> io::Result<()> { - self.write_internal(primary_namespace, secondary_namespace, key, buf) - } - - fn remove( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, - ) -> io::Result<()> { - self.remove_internal(primary_namespace, secondary_namespace, key, lazy) - } - - fn list(&self, primary_namespace: &str, secondary_namespace: &str) -> io::Result> { - self.list_internal(primary_namespace, secondary_namespace) - } -} - impl InMemoryStore { fn list_paginated_internal( &self, primary_namespace: &str, secondary_namespace: &str, page_token: Option, @@ -204,14 +178,6 @@ impl InMemoryStore { } } -impl PaginatedKVStoreSync for InMemoryStore { - fn list_paginated( - &self, primary_namespace: &str, secondary_namespace: &str, page_token: Option, - ) -> io::Result { - self.list_paginated_internal(primary_namespace, secondary_namespace, page_token) - } -} - impl PaginatedKVStore for InMemoryStore { fn list_paginated( &self, primary_namespace: &str, secondary_namespace: &str, page_token: Option, diff --git a/src/io/postgres_store/mod.rs b/src/io/postgres_store/mod.rs index 7319d08985..c54cbdef12 100644 --- a/src/io/postgres_store/mod.rs +++ b/src/io/postgres_store/mod.rs @@ -12,9 +12,7 @@ use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; use lightning::io; -use lightning::util::persist::{ - KVStore, KVStoreSync, PageToken, PaginatedKVStore, PaginatedKVStoreSync, PaginatedListResponse, -}; +use lightning::util::persist::{KVStore, PageToken, PaginatedKVStore, PaginatedListResponse}; use lightning_types::string::PrintableString; use native_tls::TlsConnector; use postgres_native_tls::MakeTlsConnector; @@ -91,10 +89,9 @@ macro_rules! query_with_retry { }}; } -/// A [`KVStoreSync`] implementation that writes to and reads from a [PostgreSQL] database. +/// A [`KVStore`] implementation that writes to and reads from a [PostgreSQL] database. /// -/// Maintains an internal runtime for the underlying tokio-postgres connection drivers and for -/// synchronous [`KVStoreSync`] and [`PaginatedKVStoreSync`] calls. +/// Maintains an internal runtime for the underlying tokio-postgres connection drivers. /// /// [PostgreSQL]: https://www.postgresql.org pub struct PostgresStore { @@ -104,14 +101,14 @@ pub struct PostgresStore { // operations aren't sensitive to the order of execution. next_write_version: AtomicU64, - // A store-internal runtime used for setup, connection driver tasks, and sync store access. + // A store-internal runtime used for setup and connection driver tasks. internal_runtime: Option, } // tokio::sync::Mutex (used for the DB client) contains UnsafeCell which opts out of // RefUnwindSafe. std::sync::Mutex (used by SqliteStore) doesn't have this issue because // it poisons on panic. This impl is needed for do_read_write_remove_list_persist which -// requires K: KVStoreSync + RefUnwindSafe. +// requires K: KVStore + RefUnwindSafe. #[cfg(test)] impl std::panic::RefUnwindSafe for PostgresStore {} @@ -302,48 +299,6 @@ impl PostgresStore { io::Error::new(io::ErrorKind::Other, "Failed to access internal PostgreSQL runtime") }) } - - fn block_on(&self, fut: F) -> io::Result { - let internal_runtime = self.internal_runtime()?; - Ok(tokio::task::block_in_place(move || internal_runtime.block_on(fut))) - } -} - -impl KVStoreSync for PostgresStore { - fn read( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, - ) -> io::Result> { - self.block_on(KVStore::read(self, primary_namespace, secondary_namespace, key))? - } - - fn write( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, - ) -> io::Result<()> { - self.block_on(KVStore::write(self, primary_namespace, secondary_namespace, key, buf))? - } - - fn remove( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, - ) -> io::Result<()> { - self.block_on(KVStore::remove(self, primary_namespace, secondary_namespace, key, lazy))? - } - - fn list(&self, primary_namespace: &str, secondary_namespace: &str) -> io::Result> { - self.block_on(KVStore::list(self, primary_namespace, secondary_namespace))? - } -} - -impl PaginatedKVStoreSync for PostgresStore { - fn list_paginated( - &self, primary_namespace: &str, secondary_namespace: &str, page_token: Option, - ) -> io::Result { - self.block_on(PaginatedKVStore::list_paginated( - self, - primary_namespace, - secondary_namespace, - page_token, - ))? - } } impl PaginatedKVStore for PostgresStore { @@ -901,7 +856,7 @@ mod tests { #[tokio::test(flavor = "multi_thread")] async fn read_write_remove_list_persist() { let store = create_test_store("test_rwrl").await; - do_read_write_remove_list_persist(&store); + do_read_write_remove_list_persist(&store).await; cleanup_store(&store).await; } @@ -931,17 +886,17 @@ mod tests { let sub = "test_sub"; // Write a value before disconnecting. - KVStoreSync::write(&store, ns, sub, "key_a", vec![1u8; 8]).unwrap(); + KVStore::write(&store, ns, sub, "key_a", vec![1u8; 8]).await.unwrap(); // Read should auto-reconnect and return the previously written value. kill_connection(&store).await; - let data = KVStoreSync::read(&store, ns, sub, "key_a").unwrap(); + let data = KVStore::read(&store, ns, sub, "key_a").await.unwrap(); assert_eq!(data, vec![1u8; 8]); // Write should auto-reconnect without a preceding read. kill_connection(&store).await; - KVStoreSync::write(&store, ns, sub, "key_b", vec![2u8; 8]).unwrap(); - let data = KVStoreSync::read(&store, ns, sub, "key_b").unwrap(); + KVStore::write(&store, ns, sub, "key_b", vec![2u8; 8]).await.unwrap(); + let data = KVStore::read(&store, ns, sub, "key_b").await.unwrap(); assert_eq!(data, vec![2u8; 8]); cleanup_store(&store).await; @@ -958,7 +913,9 @@ mod tests { for i in 0..num_entries { let key = format!("key_{:04}", i); let data = vec![i as u8; 32]; - KVStoreSync::write(&store, primary_namespace, secondary_namespace, &key, data).unwrap(); + KVStore::write(&store, primary_namespace, secondary_namespace, &key, data) + .await + .unwrap(); } // Paginate through all entries and collect them @@ -967,12 +924,13 @@ mod tests { let mut page_count = 0; loop { - let response = PaginatedKVStoreSync::list_paginated( + let response = PaginatedKVStore::list_paginated( &store, primary_namespace, secondary_namespace, page_token, ) + .await .unwrap(); all_keys.extend(response.keys.clone()); @@ -1010,32 +968,33 @@ mod tests { let primary_namespace = "test_ns"; let secondary_namespace = "test_sub"; - KVStoreSync::write(&store, primary_namespace, secondary_namespace, "first", vec![1u8; 8]) + KVStore::write(&store, primary_namespace, secondary_namespace, "first", vec![1u8; 8]) + .await .unwrap(); - KVStoreSync::write(&store, primary_namespace, secondary_namespace, "second", vec![2u8; 8]) + KVStore::write(&store, primary_namespace, secondary_namespace, "second", vec![2u8; 8]) + .await .unwrap(); - KVStoreSync::write(&store, primary_namespace, secondary_namespace, "third", vec![3u8; 8]) + KVStore::write(&store, primary_namespace, secondary_namespace, "third", vec![3u8; 8]) + .await .unwrap(); // Update the first entry - KVStoreSync::write(&store, primary_namespace, secondary_namespace, "first", vec![99u8; 8]) + KVStore::write(&store, primary_namespace, secondary_namespace, "first", vec![99u8; 8]) + .await .unwrap(); // Paginated listing should still show "first" with its original creation order - let response = PaginatedKVStoreSync::list_paginated( - &store, - primary_namespace, - secondary_namespace, - None, - ) - .unwrap(); + let response = + PaginatedKVStore::list_paginated(&store, primary_namespace, secondary_namespace, None) + .await + .unwrap(); // Newest first: third, second, first assert_eq!(response.keys, vec!["third", "second", "first"]); // Verify the updated value was persisted let data = - KVStoreSync::read(&store, primary_namespace, secondary_namespace, "first").unwrap(); + KVStore::read(&store, primary_namespace, secondary_namespace, "first").await.unwrap(); assert_eq!(data, vec![99u8; 8]); cleanup_store(&store).await; @@ -1047,7 +1006,7 @@ mod tests { // Paginating an empty or unknown namespace returns an empty result with no token. let response = - PaginatedKVStoreSync::list_paginated(&store, "nonexistent", "ns", None).unwrap(); + PaginatedKVStore::list_paginated(&store, "nonexistent", "ns", None).await.unwrap(); assert!(response.keys.is_empty()); assert!(response.next_page_token.is_none()); @@ -1058,22 +1017,23 @@ mod tests { async fn test_postgres_store_paginated_namespace_isolation() { let store = create_test_store("test_pg_paginated_isolation").await; - KVStoreSync::write(&store, "ns_a", "sub", "key_1", vec![1u8; 8]).unwrap(); - KVStoreSync::write(&store, "ns_a", "sub", "key_2", vec![2u8; 8]).unwrap(); - KVStoreSync::write(&store, "ns_b", "sub", "key_3", vec![3u8; 8]).unwrap(); - KVStoreSync::write(&store, "ns_a", "other", "key_4", vec![4u8; 8]).unwrap(); + KVStore::write(&store, "ns_a", "sub", "key_1", vec![1u8; 8]).await.unwrap(); + KVStore::write(&store, "ns_a", "sub", "key_2", vec![2u8; 8]).await.unwrap(); + KVStore::write(&store, "ns_b", "sub", "key_3", vec![3u8; 8]).await.unwrap(); + KVStore::write(&store, "ns_a", "other", "key_4", vec![4u8; 8]).await.unwrap(); // ns_a/sub should only contain key_1 and key_2 (newest first). - let response = PaginatedKVStoreSync::list_paginated(&store, "ns_a", "sub", None).unwrap(); + let response = PaginatedKVStore::list_paginated(&store, "ns_a", "sub", None).await.unwrap(); assert_eq!(response.keys, vec!["key_2", "key_1"]); assert!(response.next_page_token.is_none()); // ns_b/sub should only contain key_3. - let response = PaginatedKVStoreSync::list_paginated(&store, "ns_b", "sub", None).unwrap(); + let response = PaginatedKVStore::list_paginated(&store, "ns_b", "sub", None).await.unwrap(); assert_eq!(response.keys, vec!["key_3"]); // ns_a/other should only contain key_4. - let response = PaginatedKVStoreSync::list_paginated(&store, "ns_a", "other", None).unwrap(); + let response = + PaginatedKVStore::list_paginated(&store, "ns_a", "other", None).await.unwrap(); assert_eq!(response.keys, vec!["key_4"]); cleanup_store(&store).await; @@ -1086,13 +1046,13 @@ mod tests { let ns = "test_ns"; let sub = "test_sub"; - KVStoreSync::write(&store, ns, sub, "a", vec![1u8; 8]).unwrap(); - KVStoreSync::write(&store, ns, sub, "b", vec![2u8; 8]).unwrap(); - KVStoreSync::write(&store, ns, sub, "c", vec![3u8; 8]).unwrap(); + KVStore::write(&store, ns, sub, "a", vec![1u8; 8]).await.unwrap(); + KVStore::write(&store, ns, sub, "b", vec![2u8; 8]).await.unwrap(); + KVStore::write(&store, ns, sub, "c", vec![3u8; 8]).await.unwrap(); - KVStoreSync::remove(&store, ns, sub, "b", false).unwrap(); + KVStore::remove(&store, ns, sub, "b", false).await.unwrap(); - let response = PaginatedKVStoreSync::list_paginated(&store, ns, sub, None).unwrap(); + let response = PaginatedKVStore::list_paginated(&store, ns, sub, None).await.unwrap(); assert_eq!(response.keys, vec!["c", "a"]); assert!(response.next_page_token.is_none()); @@ -1109,24 +1069,24 @@ mod tests { // Write exactly PAGE_SIZE entries (50). for i in 0..PAGE_SIZE { let key = format!("key_{:04}", i); - KVStoreSync::write(&store, ns, sub, &key, vec![i as u8; 8]).unwrap(); + KVStore::write(&store, ns, sub, &key, vec![i as u8; 8]).await.unwrap(); } // Exactly PAGE_SIZE entries: all returned in one page with no next-page token. - let response = PaginatedKVStoreSync::list_paginated(&store, ns, sub, None).unwrap(); + let response = PaginatedKVStore::list_paginated(&store, ns, sub, None).await.unwrap(); assert_eq!(response.keys.len(), PAGE_SIZE); assert!(response.next_page_token.is_none()); // Add one more entry (PAGE_SIZE + 1 total). First page should now have a token. - KVStoreSync::write(&store, ns, sub, "key_extra", vec![0u8; 8]).unwrap(); - let response = PaginatedKVStoreSync::list_paginated(&store, ns, sub, None).unwrap(); + KVStore::write(&store, ns, sub, "key_extra", vec![0u8; 8]).await.unwrap(); + let response = PaginatedKVStore::list_paginated(&store, ns, sub, None).await.unwrap(); assert_eq!(response.keys.len(), PAGE_SIZE); assert!(response.next_page_token.is_some()); // Second page should have exactly 1 entry and no token. - let response = - PaginatedKVStoreSync::list_paginated(&store, ns, sub, response.next_page_token) - .unwrap(); + let response = PaginatedKVStore::list_paginated(&store, ns, sub, response.next_page_token) + .await + .unwrap(); assert_eq!(response.keys.len(), 1); assert!(response.next_page_token.is_none()); @@ -1143,10 +1103,10 @@ mod tests { // Write fewer entries than PAGE_SIZE. for i in 0..5 { let key = format!("key_{i}"); - KVStoreSync::write(&store, ns, sub, &key, vec![i as u8; 8]).unwrap(); + KVStore::write(&store, ns, sub, &key, vec![i as u8; 8]).await.unwrap(); } - let response = PaginatedKVStoreSync::list_paginated(&store, ns, sub, None).unwrap(); + let response = PaginatedKVStore::list_paginated(&store, ns, sub, None).await.unwrap(); assert_eq!(response.keys.len(), 5); // Fewer than PAGE_SIZE means no next page. assert!(response.next_page_token.is_none()); @@ -1165,22 +1125,12 @@ mod tests { { let store = create_test_store(table_name).await; - KVStoreSync::write( - &store, - primary_namespace, - secondary_namespace, - "key_a", - vec![1u8; 8], - ) - .unwrap(); - KVStoreSync::write( - &store, - primary_namespace, - secondary_namespace, - "key_b", - vec![2u8; 8], - ) - .unwrap(); + KVStore::write(&store, primary_namespace, secondary_namespace, "key_a", vec![1u8; 8]) + .await + .unwrap(); + KVStore::write(&store, primary_namespace, secondary_namespace, "key_b", vec![2u8; 8]) + .await + .unwrap(); // Don't clean up since we want to reopen } @@ -1189,22 +1139,18 @@ mod tests { { let store = create_test_store(table_name).await; - KVStoreSync::write( - &store, - primary_namespace, - secondary_namespace, - "key_c", - vec![3u8; 8], - ) - .unwrap(); + KVStore::write(&store, primary_namespace, secondary_namespace, "key_c", vec![3u8; 8]) + .await + .unwrap(); // Paginated listing should show newest first: key_c, key_b, key_a - let response = PaginatedKVStoreSync::list_paginated( + let response = PaginatedKVStore::list_paginated( &store, primary_namespace, secondary_namespace, None, ) + .await .unwrap(); assert_eq!(response.keys, vec!["key_c", "key_b", "key_a"]); diff --git a/src/io/sqlite_store/migrations.rs b/src/io/sqlite_store/migrations.rs index f596b1a42f..1b4de9aa07 100644 --- a/src/io/sqlite_store/migrations.rs +++ b/src/io/sqlite_store/migrations.rs @@ -169,14 +169,14 @@ fn migrate_v2_to_v3(connection: &mut Connection, kv_table_name: &str) -> io::Res mod tests { use std::fs; - use lightning::util::persist::{KVStoreSync, PaginatedKVStoreSync}; + use lightning::util::persist::{KVStore, PaginatedKVStore}; use rusqlite::{named_params, Connection}; use crate::io::sqlite_store::SqliteStore; use crate::io::test_utils::{do_read_write_remove_list_persist, random_storage_path}; - #[test] - fn rwrl_post_schema_1_migration() { + #[tokio::test] + async fn rwrl_post_schema_1_migration() { let old_schema_version = 1; let mut temp_path = random_storage_path(); @@ -253,15 +253,15 @@ mod tests { // Check we migrate the db just fine without losing our written data. let store = SqliteStore::new(temp_path, Some(db_file_name), Some(kv_table_name)).unwrap(); - let res = store.read(&test_namespace, "", &test_key).unwrap(); + let res = KVStore::read(&store, &test_namespace, "", &test_key).await.unwrap(); assert_eq!(res, test_data); // Check we can continue to use the store just fine. - do_read_write_remove_list_persist(&store); + do_read_write_remove_list_persist(&store).await; } - #[test] - fn rwrl_post_schema_2_migration() { + #[tokio::test] + async fn rwrl_post_schema_2_migration() { let old_schema_version = 2u16; let mut temp_path = random_storage_path(); @@ -325,24 +325,24 @@ mod tests { // Verify data survived for i in 0..3 { let key = format!("key_{}", i); - let data = store.read(test_ns, test_sub, &key).unwrap(); + let data = KVStore::read(&store, test_ns, test_sub, &key).await.unwrap(); assert_eq!(data, vec![i as u8; 8]); } // Verify paginated listing works and returns entries in ROWID-backfilled order (newest first) let response = - PaginatedKVStoreSync::list_paginated(&store, test_ns, test_sub, None).unwrap(); + PaginatedKVStore::list_paginated(&store, test_ns, test_sub, None).await.unwrap(); assert_eq!(response.keys.len(), 3); // ROWIDs were 1, 2, 3 so sort_order was backfilled as 1, 2, 3; newest first assert_eq!(response.keys, vec!["key_2", "key_1", "key_0"]); // Verify we can write new entries and they get proper ordering - KVStoreSync::write(&store, test_ns, test_sub, "key_new", vec![99u8; 8]).unwrap(); + KVStore::write(&store, test_ns, test_sub, "key_new", vec![99u8; 8]).await.unwrap(); let response = - PaginatedKVStoreSync::list_paginated(&store, test_ns, test_sub, None).unwrap(); + PaginatedKVStore::list_paginated(&store, test_ns, test_sub, None).await.unwrap(); assert_eq!(response.keys[0], "key_new"); // Check we can continue to use the store just fine. - do_read_write_remove_list_persist(&store); + do_read_write_remove_list_persist(&store).await; } } diff --git a/src/io/sqlite_store/mod.rs b/src/io/sqlite_store/mod.rs index 84af03adc5..076aeef9bd 100644 --- a/src/io/sqlite_store/mod.rs +++ b/src/io/sqlite_store/mod.rs @@ -14,9 +14,7 @@ use std::sync::atomic::{AtomicI64, AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; use lightning::io; -use lightning::util::persist::{ - KVStore, KVStoreSync, PageToken, PaginatedKVStore, PaginatedKVStoreSync, PaginatedListResponse, -}; +use lightning::util::persist::{KVStore, PageToken, PaginatedKVStore, PaginatedListResponse}; use lightning_types::string::PrintableString; use rusqlite::{named_params, Connection}; @@ -41,7 +39,7 @@ const SCHEMA_USER_VERSION: u16 = 3; // The number of entries returned per page in paginated list operations. const PAGE_SIZE: usize = 50; -/// A [`KVStoreSync`] implementation that writes to and reads from an [SQLite] database. +/// A [`KVStore`] implementation that writes to and reads from an [SQLite] database. /// /// [SQLite]: https://sqlite.org pub struct SqliteStore { @@ -185,57 +183,6 @@ impl KVStore for SqliteStore { } } -impl KVStoreSync for SqliteStore { - fn read( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, - ) -> io::Result> { - self.inner.read_internal(primary_namespace, secondary_namespace, key) - } - - fn write( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, - ) -> io::Result<()> { - let locking_key = self.build_locking_key(primary_namespace, secondary_namespace, key); - let (inner_lock_ref, version) = self.get_new_version_and_lock_ref(locking_key.clone()); - self.inner.write_internal( - inner_lock_ref, - locking_key, - version, - primary_namespace, - secondary_namespace, - key, - buf, - ) - } - - fn remove( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, _lazy: bool, - ) -> io::Result<()> { - let locking_key = self.build_locking_key(primary_namespace, secondary_namespace, key); - let (inner_lock_ref, version) = self.get_new_version_and_lock_ref(locking_key.clone()); - self.inner.remove_internal( - inner_lock_ref, - locking_key, - version, - primary_namespace, - secondary_namespace, - key, - ) - } - - fn list(&self, primary_namespace: &str, secondary_namespace: &str) -> io::Result> { - self.inner.list_internal(primary_namespace, secondary_namespace) - } -} - -impl PaginatedKVStoreSync for SqliteStore { - fn list_paginated( - &self, primary_namespace: &str, secondary_namespace: &str, page_token: Option, - ) -> io::Result { - self.inner.list_paginated_internal(primary_namespace, secondary_namespace, page_token) - } -} - impl PaginatedKVStore for SqliteStore { fn list_paginated( &self, primary_namespace: &str, secondary_namespace: &str, page_token: Option, @@ -700,8 +647,8 @@ mod tests { } } - #[test] - fn read_write_remove_list_persist() { + #[tokio::test] + async fn read_write_remove_list_persist() { let mut temp_path = random_storage_path(); temp_path.push("read_write_remove_list_persist"); let store = SqliteStore::new( @@ -710,11 +657,11 @@ mod tests { Some("test_table".to_string()), ) .unwrap(); - do_read_write_remove_list_persist(&store); + do_read_write_remove_list_persist(&store).await; } - #[test] - fn test_sqlite_store() { + #[tokio::test(flavor = "multi_thread")] + async fn test_sqlite_store() { let mut temp_path = random_storage_path(); temp_path.push("test_sqlite_store"); let store_0 = SqliteStore::new( @@ -732,8 +679,8 @@ mod tests { do_test_store(&store_0, &store_1) } - #[test] - fn test_sqlite_store_paginated_listing() { + #[tokio::test] + async fn test_sqlite_store_paginated_listing() { let mut temp_path = random_storage_path(); temp_path.push("test_sqlite_store_paginated_listing"); let store = SqliteStore::new( @@ -750,7 +697,9 @@ mod tests { for i in 0..num_entries { let key = format!("key_{:04}", i); let data = vec![i as u8; 32]; - KVStoreSync::write(&store, primary_namespace, secondary_namespace, &key, data).unwrap(); + KVStore::write(&store, primary_namespace, secondary_namespace, &key, data) + .await + .unwrap(); } // Paginate through all entries and collect them @@ -759,12 +708,13 @@ mod tests { let mut page_count = 0; loop { - let response = PaginatedKVStoreSync::list_paginated( + let response = PaginatedKVStore::list_paginated( &store, primary_namespace, secondary_namespace, page_token, ) + .await .unwrap(); all_keys.extend(response.keys.clone()); @@ -795,8 +745,8 @@ mod tests { assert_eq!(all_keys[num_entries - 1], "key_0000"); } - #[test] - fn test_sqlite_store_paginated_update_preserves_order() { + #[tokio::test] + async fn test_sqlite_store_paginated_update_preserves_order() { let mut temp_path = random_storage_path(); temp_path.push("test_sqlite_store_paginated_update"); let store = SqliteStore::new( @@ -809,37 +759,38 @@ mod tests { let primary_namespace = "test_ns"; let secondary_namespace = "test_sub"; - KVStoreSync::write(&store, primary_namespace, secondary_namespace, "first", vec![1u8; 8]) + KVStore::write(&store, primary_namespace, secondary_namespace, "first", vec![1u8; 8]) + .await .unwrap(); - KVStoreSync::write(&store, primary_namespace, secondary_namespace, "second", vec![2u8; 8]) + KVStore::write(&store, primary_namespace, secondary_namespace, "second", vec![2u8; 8]) + .await .unwrap(); - KVStoreSync::write(&store, primary_namespace, secondary_namespace, "third", vec![3u8; 8]) + KVStore::write(&store, primary_namespace, secondary_namespace, "third", vec![3u8; 8]) + .await .unwrap(); // Update the first entry - KVStoreSync::write(&store, primary_namespace, secondary_namespace, "first", vec![99u8; 8]) + KVStore::write(&store, primary_namespace, secondary_namespace, "first", vec![99u8; 8]) + .await .unwrap(); // Paginated listing should still show "first" with its original creation order - let response = PaginatedKVStoreSync::list_paginated( - &store, - primary_namespace, - secondary_namespace, - None, - ) - .unwrap(); + let response = + PaginatedKVStore::list_paginated(&store, primary_namespace, secondary_namespace, None) + .await + .unwrap(); // Newest first: third, second, first assert_eq!(response.keys, vec!["third", "second", "first"]); // Verify the updated value was persisted let data = - KVStoreSync::read(&store, primary_namespace, secondary_namespace, "first").unwrap(); + KVStore::read(&store, primary_namespace, secondary_namespace, "first").await.unwrap(); assert_eq!(data, vec![99u8; 8]); } - #[test] - fn test_sqlite_store_paginated_empty_namespace() { + #[tokio::test] + async fn test_sqlite_store_paginated_empty_namespace() { let mut temp_path = random_storage_path(); temp_path.push("test_sqlite_store_paginated_empty"); let store = SqliteStore::new( @@ -851,13 +802,13 @@ mod tests { // Paginating an empty or unknown namespace returns an empty result with no token. let response = - PaginatedKVStoreSync::list_paginated(&store, "nonexistent", "ns", None).unwrap(); + PaginatedKVStore::list_paginated(&store, "nonexistent", "ns", None).await.unwrap(); assert!(response.keys.is_empty()); assert!(response.next_page_token.is_none()); } - #[test] - fn test_sqlite_store_paginated_namespace_isolation() { + #[tokio::test] + async fn test_sqlite_store_paginated_namespace_isolation() { let mut temp_path = random_storage_path(); temp_path.push("test_sqlite_store_paginated_isolation"); let store = SqliteStore::new( @@ -867,27 +818,28 @@ mod tests { ) .unwrap(); - KVStoreSync::write(&store, "ns_a", "sub", "key_1", vec![1u8; 8]).unwrap(); - KVStoreSync::write(&store, "ns_a", "sub", "key_2", vec![2u8; 8]).unwrap(); - KVStoreSync::write(&store, "ns_b", "sub", "key_3", vec![3u8; 8]).unwrap(); - KVStoreSync::write(&store, "ns_a", "other", "key_4", vec![4u8; 8]).unwrap(); + KVStore::write(&store, "ns_a", "sub", "key_1", vec![1u8; 8]).await.unwrap(); + KVStore::write(&store, "ns_a", "sub", "key_2", vec![2u8; 8]).await.unwrap(); + KVStore::write(&store, "ns_b", "sub", "key_3", vec![3u8; 8]).await.unwrap(); + KVStore::write(&store, "ns_a", "other", "key_4", vec![4u8; 8]).await.unwrap(); // ns_a/sub should only contain key_1 and key_2 (newest first). - let response = PaginatedKVStoreSync::list_paginated(&store, "ns_a", "sub", None).unwrap(); + let response = PaginatedKVStore::list_paginated(&store, "ns_a", "sub", None).await.unwrap(); assert_eq!(response.keys, vec!["key_2", "key_1"]); assert!(response.next_page_token.is_none()); // ns_b/sub should only contain key_3. - let response = PaginatedKVStoreSync::list_paginated(&store, "ns_b", "sub", None).unwrap(); + let response = PaginatedKVStore::list_paginated(&store, "ns_b", "sub", None).await.unwrap(); assert_eq!(response.keys, vec!["key_3"]); // ns_a/other should only contain key_4. - let response = PaginatedKVStoreSync::list_paginated(&store, "ns_a", "other", None).unwrap(); + let response = + PaginatedKVStore::list_paginated(&store, "ns_a", "other", None).await.unwrap(); assert_eq!(response.keys, vec!["key_4"]); } - #[test] - fn test_sqlite_store_paginated_removal() { + #[tokio::test] + async fn test_sqlite_store_paginated_removal() { let mut temp_path = random_storage_path(); temp_path.push("test_sqlite_store_paginated_removal"); let store = SqliteStore::new( @@ -900,19 +852,19 @@ mod tests { let ns = "test_ns"; let sub = "test_sub"; - KVStoreSync::write(&store, ns, sub, "a", vec![1u8; 8]).unwrap(); - KVStoreSync::write(&store, ns, sub, "b", vec![2u8; 8]).unwrap(); - KVStoreSync::write(&store, ns, sub, "c", vec![3u8; 8]).unwrap(); + KVStore::write(&store, ns, sub, "a", vec![1u8; 8]).await.unwrap(); + KVStore::write(&store, ns, sub, "b", vec![2u8; 8]).await.unwrap(); + KVStore::write(&store, ns, sub, "c", vec![3u8; 8]).await.unwrap(); - KVStoreSync::remove(&store, ns, sub, "b", false).unwrap(); + KVStore::remove(&store, ns, sub, "b", false).await.unwrap(); - let response = PaginatedKVStoreSync::list_paginated(&store, ns, sub, None).unwrap(); + let response = PaginatedKVStore::list_paginated(&store, ns, sub, None).await.unwrap(); assert_eq!(response.keys, vec!["c", "a"]); assert!(response.next_page_token.is_none()); } - #[test] - fn test_sqlite_store_paginated_exact_page_boundary() { + #[tokio::test] + async fn test_sqlite_store_paginated_exact_page_boundary() { let mut temp_path = random_storage_path(); temp_path.push("test_sqlite_store_paginated_boundary"); let store = SqliteStore::new( @@ -928,30 +880,30 @@ mod tests { // Write exactly PAGE_SIZE entries (50). for i in 0..PAGE_SIZE { let key = format!("key_{:04}", i); - KVStoreSync::write(&store, ns, sub, &key, vec![i as u8; 8]).unwrap(); + KVStore::write(&store, ns, sub, &key, vec![i as u8; 8]).await.unwrap(); } // Exactly PAGE_SIZE entries: all returned in one page with no next-page token. - let response = PaginatedKVStoreSync::list_paginated(&store, ns, sub, None).unwrap(); + let response = PaginatedKVStore::list_paginated(&store, ns, sub, None).await.unwrap(); assert_eq!(response.keys.len(), PAGE_SIZE); assert!(response.next_page_token.is_none()); // Add one more entry (PAGE_SIZE + 1 total). First page should now have a token. - KVStoreSync::write(&store, ns, sub, "key_extra", vec![0u8; 8]).unwrap(); - let response = PaginatedKVStoreSync::list_paginated(&store, ns, sub, None).unwrap(); + KVStore::write(&store, ns, sub, "key_extra", vec![0u8; 8]).await.unwrap(); + let response = PaginatedKVStore::list_paginated(&store, ns, sub, None).await.unwrap(); assert_eq!(response.keys.len(), PAGE_SIZE); assert!(response.next_page_token.is_some()); // Second page should have exactly 1 entry and no token. - let response = - PaginatedKVStoreSync::list_paginated(&store, ns, sub, response.next_page_token) - .unwrap(); + let response = PaginatedKVStore::list_paginated(&store, ns, sub, response.next_page_token) + .await + .unwrap(); assert_eq!(response.keys.len(), 1); assert!(response.next_page_token.is_none()); } - #[test] - fn test_sqlite_store_paginated_fewer_than_page_size() { + #[tokio::test] + async fn test_sqlite_store_paginated_fewer_than_page_size() { let mut temp_path = random_storage_path(); temp_path.push("test_sqlite_store_paginated_few"); let store = SqliteStore::new( @@ -967,10 +919,10 @@ mod tests { // Write fewer entries than PAGE_SIZE. for i in 0..5 { let key = format!("key_{}", i); - KVStoreSync::write(&store, ns, sub, &key, vec![i as u8; 8]).unwrap(); + KVStore::write(&store, ns, sub, &key, vec![i as u8; 8]).await.unwrap(); } - let response = PaginatedKVStoreSync::list_paginated(&store, ns, sub, None).unwrap(); + let response = PaginatedKVStore::list_paginated(&store, ns, sub, None).await.unwrap(); assert_eq!(response.keys.len(), 5); // Fewer than PAGE_SIZE means no next page. assert!(response.next_page_token.is_none()); @@ -978,8 +930,8 @@ mod tests { assert_eq!(response.keys, vec!["key_4", "key_3", "key_2", "key_1", "key_0"]); } - #[test] - fn test_sqlite_store_write_version_persists_across_restart() { + #[tokio::test] + async fn test_sqlite_store_write_version_persists_across_restart() { let mut temp_path = random_storage_path(); temp_path.push("test_sqlite_store_write_version_restart"); @@ -994,22 +946,12 @@ mod tests { ) .unwrap(); - KVStoreSync::write( - &store, - primary_namespace, - secondary_namespace, - "key_a", - vec![1u8; 8], - ) - .unwrap(); - KVStoreSync::write( - &store, - primary_namespace, - secondary_namespace, - "key_b", - vec![2u8; 8], - ) - .unwrap(); + KVStore::write(&store, primary_namespace, secondary_namespace, "key_a", vec![1u8; 8]) + .await + .unwrap(); + KVStore::write(&store, primary_namespace, secondary_namespace, "key_b", vec![2u8; 8]) + .await + .unwrap(); // Don't drop/cleanup since we want to reopen std::mem::forget(store); @@ -1024,22 +966,18 @@ mod tests { ) .unwrap(); - KVStoreSync::write( - &store, - primary_namespace, - secondary_namespace, - "key_c", - vec![3u8; 8], - ) - .unwrap(); + KVStore::write(&store, primary_namespace, secondary_namespace, "key_c", vec![3u8; 8]) + .await + .unwrap(); // Paginated listing should show newest first: key_c, key_b, key_a - let response = PaginatedKVStoreSync::list_paginated( + let response = PaginatedKVStore::list_paginated( &store, primary_namespace, secondary_namespace, None, ) + .await .unwrap(); assert_eq!(response.keys, vec!["key_c", "key_b", "key_a"]); diff --git a/src/io/test_utils.rs b/src/io/test_utils.rs index 0b16e525ed..aadb4b79a8 100644 --- a/src/io/test_utils.rs +++ b/src/io/test_utils.rs @@ -5,10 +5,15 @@ // http://opensource.org/licenses/MIT>, at your option. You may not use this file except in // accordance with one or both of these licenses. +use std::future::Future; use std::panic::RefUnwindSafe; use std::path::PathBuf; +use std::sync::Arc; +use lightning::chain::channelmonitor::{ChannelMonitor, ChannelMonitorUpdate}; +use lightning::chain::{chainmonitor, BlockLocator, ChannelMonitorUpdateStatus}; use lightning::events::ClosureReason; +use lightning::io; use lightning::ln::functional_test_utils::{ check_added_monitors, check_closed_broadcast, check_closed_event, connect_block, create_announced_chan_between_nodes, create_chanmon_cfgs, create_dummy_block, create_network, @@ -16,8 +21,13 @@ use lightning::ln::functional_test_utils::{ TestChanMonCfg, }; use lightning::util::persist::{ - KVStoreSync, MonitorUpdatingPersister, KVSTORE_NAMESPACE_KEY_MAX_LEN, + KVStore, MonitorName, ARCHIVED_CHANNEL_MONITOR_PERSISTENCE_PRIMARY_NAMESPACE, + ARCHIVED_CHANNEL_MONITOR_PERSISTENCE_SECONDARY_NAMESPACE, + CHANNEL_MONITOR_PERSISTENCE_PRIMARY_NAMESPACE, CHANNEL_MONITOR_PERSISTENCE_SECONDARY_NAMESPACE, + KVSTORE_NAMESPACE_KEY_MAX_LEN, }; +use lightning::util::ser::{ReadableArgs, Writeable}; +use lightning::util::test_channel_signer::TestChannelSigner; use lightning::util::test_utils; use rand::distr::Alphanumeric; use rand::{rng, Rng}; @@ -25,14 +35,127 @@ use rand::{rng, Rng}; #[path = "in_memory_store.rs"] mod in_memory_store; -type TestMonitorUpdatePersister<'a, K> = MonitorUpdatingPersister< - &'a K, - &'a test_utils::TestLogger, - &'a test_utils::TestKeysInterface, - &'a test_utils::TestKeysInterface, - &'a test_utils::TestBroadcaster, - &'a test_utils::TestFeeEstimator, ->; +use crate::logger::Logger; +use crate::runtime::Runtime; + +pub(crate) struct TestMonitorUpdatePersister<'a, K> { + store: &'a K, + runtime: Runtime, + entropy_source: &'a test_utils::TestKeysInterface, + signer_provider: &'a test_utils::TestKeysInterface, +} + +impl TestMonitorUpdatePersister<'_, K> { + pub(crate) fn read_all_channel_monitors_with_updates( + &self, + ) -> Result)>, io::Error> { + self.runtime.block_on(async { + let stored_keys = KVStore::list( + self.store, + CHANNEL_MONITOR_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MONITOR_PERSISTENCE_SECONDARY_NAMESPACE, + ) + .await?; + + let mut res = Vec::with_capacity(stored_keys.len()); + for stored_key in stored_keys { + let data = KVStore::read( + self.store, + CHANNEL_MONITOR_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MONITOR_PERSISTENCE_SECONDARY_NAMESPACE, + &stored_key, + ) + .await?; + match )>>::read( + &mut io::Cursor::new(data), + (self.entropy_source, self.signer_provider), + ) { + Ok(Some((best_block, channel_monitor))) => { + res.push((best_block, channel_monitor)); + }, + Ok(None) => {}, + Err(_) => { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Failed to read ChannelMonitor", + )); + }, + } + } + Ok(res) + }) + } + + fn write_monitor( + &self, monitor_name: MonitorName, monitor: &ChannelMonitor, + ) -> ChannelMonitorUpdateStatus { + let write_res = self.runtime.block_on(KVStore::write( + self.store, + CHANNEL_MONITOR_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MONITOR_PERSISTENCE_SECONDARY_NAMESPACE, + &monitor_name.to_string(), + monitor.encode(), + )); + match write_res { + Ok(()) => ChannelMonitorUpdateStatus::Completed, + Err(_) => ChannelMonitorUpdateStatus::UnrecoverableError, + } + } +} + +impl chainmonitor::Persist + for TestMonitorUpdatePersister<'_, K> +{ + fn persist_new_channel( + &self, monitor_name: MonitorName, monitor: &ChannelMonitor, + ) -> ChannelMonitorUpdateStatus { + self.write_monitor(monitor_name, monitor) + } + + fn update_persisted_channel( + &self, monitor_name: MonitorName, _monitor_update: Option<&ChannelMonitorUpdate>, + monitor: &ChannelMonitor, + ) -> ChannelMonitorUpdateStatus { + self.write_monitor(monitor_name, monitor) + } + + fn archive_persisted_channel(&self, monitor_name: MonitorName) { + let key = monitor_name.to_string(); + self.runtime.block_on(async { + let monitor = match KVStore::read( + self.store, + CHANNEL_MONITOR_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MONITOR_PERSISTENCE_SECONDARY_NAMESPACE, + &key, + ) + .await + { + Ok(monitor) => monitor, + Err(_) => return, + }; + + if KVStore::write( + self.store, + ARCHIVED_CHANNEL_MONITOR_PERSISTENCE_PRIMARY_NAMESPACE, + ARCHIVED_CHANNEL_MONITOR_PERSISTENCE_SECONDARY_NAMESPACE, + &key, + monitor, + ) + .await + .is_ok() + { + let _ = KVStore::remove( + self.store, + CHANNEL_MONITOR_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MONITOR_PERSISTENCE_SECONDARY_NAMESPACE, + &key, + true, + ) + .await; + } + }); + } +} const EXPECTED_UPDATES_PER_PAYMENT: u64 = 5; @@ -46,7 +169,32 @@ pub(crate) fn random_storage_path() -> PathBuf { temp_path } -pub(crate) fn do_read_write_remove_list_persist(kv_store: &K) { +async fn catch_future_unwind(future: F) -> std::thread::Result { + let mut future = std::pin::pin!(future); + std::future::poll_fn(|cx| { + match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| future.as_mut().poll(cx))) { + Ok(std::task::Poll::Ready(output)) => std::task::Poll::Ready(Ok(output)), + Ok(std::task::Poll::Pending) => std::task::Poll::Pending, + Err(panic) => std::task::Poll::Ready(Err(panic)), + } + }) + .await +} + +async fn assert_invalid_write_fails( + kv_store: &K, primary_namespace: &str, secondary_namespace: &str, key: &str, data: Vec, +) { + let res = std::panic::catch_unwind(|| { + KVStore::write(kv_store, primary_namespace, secondary_namespace, key, data) + }); + if let Ok(fut) = res { + if let Ok(write_res) = catch_future_unwind(fut).await { + assert!(write_res.is_err()); + } + } +} + +pub(crate) async fn do_read_write_remove_list_persist(kv_store: &K) { let data = vec![42u8; 32]; let primary_namespace = "testspace"; @@ -54,63 +202,63 @@ pub(crate) fn do_read_write_remove_list_persist( let key = "testkey"; // Test the basic KVStore operations. - kv_store.write(primary_namespace, secondary_namespace, key, data.clone()).unwrap(); + KVStore::write(kv_store, primary_namespace, secondary_namespace, key, data.clone()) + .await + .unwrap(); // Test empty primary/secondary namespaces are allowed, but not empty primary namespace and non-empty // secondary primary_namespace, and not empty key. - kv_store.write("", "", key, data.clone()).unwrap(); - let res = - std::panic::catch_unwind(|| kv_store.write("", secondary_namespace, key, data.clone())); - assert!(res.is_err()); - let res = std::panic::catch_unwind(|| { - kv_store.write(primary_namespace, secondary_namespace, "", data.clone()) - }); - assert!(res.is_err()); + KVStore::write(kv_store, "", "", key, data.clone()).await.unwrap(); + assert_invalid_write_fails(kv_store, "", secondary_namespace, key, data.clone()).await; + assert_invalid_write_fails(kv_store, primary_namespace, secondary_namespace, "", data.clone()) + .await; - let listed_keys = kv_store.list(primary_namespace, secondary_namespace).unwrap(); + let listed_keys = + KVStore::list(kv_store, primary_namespace, secondary_namespace).await.unwrap(); assert_eq!(listed_keys.len(), 1); assert_eq!(listed_keys[0], key); - let read_data = kv_store.read(primary_namespace, secondary_namespace, key).unwrap(); + let read_data = + KVStore::read(kv_store, primary_namespace, secondary_namespace, key).await.unwrap(); assert_eq!(data, &*read_data); - kv_store.remove(primary_namespace, secondary_namespace, key, false).unwrap(); + KVStore::remove(kv_store, primary_namespace, secondary_namespace, key, false).await.unwrap(); - let listed_keys = kv_store.list(primary_namespace, secondary_namespace).unwrap(); + let listed_keys = + KVStore::list(kv_store, primary_namespace, secondary_namespace).await.unwrap(); assert_eq!(listed_keys.len(), 0); // Ensure we have no issue operating with primary_namespace/secondary_namespace/key being KVSTORE_NAMESPACE_KEY_MAX_LEN let max_chars: String = std::iter::repeat('A').take(KVSTORE_NAMESPACE_KEY_MAX_LEN).collect(); - kv_store.write(&max_chars, &max_chars, &max_chars, data.clone()).unwrap(); + KVStore::write(kv_store, &max_chars, &max_chars, &max_chars, data.clone()).await.unwrap(); - let listed_keys = kv_store.list(&max_chars, &max_chars).unwrap(); + let listed_keys = KVStore::list(kv_store, &max_chars, &max_chars).await.unwrap(); assert_eq!(listed_keys.len(), 1); assert_eq!(listed_keys[0], max_chars); - let read_data = kv_store.read(&max_chars, &max_chars, &max_chars).unwrap(); + let read_data = KVStore::read(kv_store, &max_chars, &max_chars, &max_chars).await.unwrap(); assert_eq!(data, &*read_data); - kv_store.remove(&max_chars, &max_chars, &max_chars, false).unwrap(); + KVStore::remove(kv_store, &max_chars, &max_chars, &max_chars, false).await.unwrap(); - let listed_keys = kv_store.list(&max_chars, &max_chars).unwrap(); + let listed_keys = KVStore::list(kv_store, &max_chars, &max_chars).await.unwrap(); assert_eq!(listed_keys.len(), 0); } -pub(crate) fn create_persister<'a, K: KVStoreSync + Sync>( - store: &'a K, chanmon_cfg: &'a TestChanMonCfg, max_pending_updates: u64, +pub(crate) fn create_persister<'a, K: KVStore + Sync>( + store: &'a K, chanmon_cfg: &'a TestChanMonCfg, _max_pending_updates: u64, ) -> TestMonitorUpdatePersister<'a, K> { - MonitorUpdatingPersister::new( + let runtime = + Runtime::new(Arc::new(Logger::new_log_facade())).expect("Failed to setup runtime"); + TestMonitorUpdatePersister { store, - &chanmon_cfg.logger, - max_pending_updates, - &chanmon_cfg.keys_manager, - &chanmon_cfg.keys_manager, - &chanmon_cfg.tx_broadcaster, - &chanmon_cfg.fee_estimator, - ) + runtime, + entropy_source: &chanmon_cfg.keys_manager, + signer_provider: &chanmon_cfg.keys_manager, + } } -pub(crate) fn create_chain_monitor<'a, K: KVStoreSync + Sync>( +pub(crate) fn create_chain_monitor<'a, K: KVStore + Sync>( chanmon_cfg: &'a TestChanMonCfg, persister: &'a TestMonitorUpdatePersister<'a, K>, ) -> test_utils::TestChainMonitor<'a> { test_utils::TestChainMonitor::new( @@ -125,7 +273,7 @@ pub(crate) fn create_chain_monitor<'a, K: KVStoreSync + Sync>( // Integration-test the given KVStore implementation. Test relaying a few payments and check that // the persisted data is updated the appropriate number of times. -pub(crate) fn do_test_store(store_0: &K, store_1: &K) { +pub(crate) fn do_test_store(store_0: &K, store_1: &K) { // This value is used later to limit how many iterations we perform. let persister_0_max_pending_updates = 7; // Intentionally set this to a smaller value to test a different alignment. diff --git a/src/io/vss_store.rs b/src/io/vss_store.rs index 007f7b9b48..e7e8d031e3 100644 --- a/src/io/vss_store.rs +++ b/src/io/vss_store.rs @@ -24,7 +24,7 @@ use bitcoin::Network; use lightning::impl_ser_tlv_based_enum; use lightning::io::{self, Error, ErrorKind}; use lightning::sign::{EntropySource as LdkEntropySource, RandomBytes}; -use lightning::util::persist::{KVStore, KVStoreSync}; +use lightning::util::persist::KVStore; use lightning::util::ser::{Readable, Writeable}; use prost::Message; use vss_client::client::VssClient; @@ -77,7 +77,7 @@ const VSS_SCHEMA_VERSION_KEY: &str = "vss_schema_version"; // would hit a blocking case const INTERNAL_RUNTIME_WORKERS: usize = 2; -/// A [`KVStore`]/[`KVStoreSync`] implementation that writes to and reads from a [VSS] backend. +/// A [`KVStore`] implementation that writes to and reads from a [VSS] backend. /// /// [VSS]: https://github.com/lightningdevkit/vss-server/blob/main/README.md pub struct VssStore { @@ -149,7 +149,6 @@ impl VssStore { let inner = Arc::new(VssStoreInner::new( schema_version, - blocking_client, async_client, store_id, data_encryption_key, @@ -193,111 +192,6 @@ impl VssStore { } } -impl KVStoreSync for VssStore { - fn read( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, - ) -> io::Result> { - let internal_runtime = self.internal_runtime.as_ref().ok_or_else(|| { - debug_assert!(false, "Failed to access internal runtime"); - let msg = format!("Failed to access internal runtime"); - Error::new(ErrorKind::Other, msg) - })?; - let primary_namespace = primary_namespace.to_string(); - let secondary_namespace = secondary_namespace.to_string(); - let key = key.to_string(); - let inner = Arc::clone(&self.inner); - let fut = async move { - inner - .read_internal(&inner.blocking_client, primary_namespace, secondary_namespace, key) - .await - }; - tokio::task::block_in_place(move || internal_runtime.block_on(fut)) - } - - fn write( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, - ) -> io::Result<()> { - let internal_runtime = self.internal_runtime.as_ref().ok_or_else(|| { - debug_assert!(false, "Failed to access internal runtime"); - let msg = format!("Failed to access internal runtime"); - Error::new(ErrorKind::Other, msg) - })?; - let primary_namespace = primary_namespace.to_string(); - let secondary_namespace = secondary_namespace.to_string(); - let key = key.to_string(); - let inner = Arc::clone(&self.inner); - let locking_key = self.build_locking_key(&primary_namespace, &secondary_namespace, &key); - let (inner_lock_ref, version) = self.get_new_version_and_lock_ref(locking_key.clone()); - let fut = async move { - inner - .write_internal( - &inner.blocking_client, - inner_lock_ref, - locking_key, - version, - primary_namespace, - secondary_namespace, - key, - buf, - ) - .await - }; - tokio::task::block_in_place(move || internal_runtime.block_on(fut)) - } - - fn remove( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, - ) -> io::Result<()> { - let internal_runtime = self.internal_runtime.as_ref().ok_or_else(|| { - debug_assert!(false, "Failed to access internal runtime"); - let msg = format!("Failed to access internal runtime"); - Error::new(ErrorKind::Other, msg) - })?; - let primary_namespace = primary_namespace.to_string(); - let secondary_namespace = secondary_namespace.to_string(); - let key = key.to_string(); - let inner = Arc::clone(&self.inner); - let locking_key = self.build_locking_key(&primary_namespace, &secondary_namespace, &key); - let (inner_lock_ref, version) = self.get_new_version_and_lock_ref(locking_key.clone()); - let fut = async move { - inner - .remove_internal( - &inner.blocking_client, - inner_lock_ref, - locking_key, - version, - primary_namespace, - secondary_namespace, - key, - ) - .await - }; - if lazy { - internal_runtime.spawn(async { fut.await }); - Ok(()) - } else { - tokio::task::block_in_place(move || internal_runtime.block_on(fut)) - } - } - - fn list(&self, primary_namespace: &str, secondary_namespace: &str) -> io::Result> { - let internal_runtime = self.internal_runtime.as_ref().ok_or_else(|| { - debug_assert!(false, "Failed to access internal runtime"); - let msg = format!("Failed to access internal runtime"); - Error::new(ErrorKind::Other, msg) - })?; - let primary_namespace = primary_namespace.to_string(); - let secondary_namespace = secondary_namespace.to_string(); - let inner = Arc::clone(&self.inner); - let fut = async move { - inner - .list_internal(&inner.blocking_client, primary_namespace, secondary_namespace) - .await - }; - tokio::task::block_in_place(move || internal_runtime.block_on(fut)) - } -} - impl KVStore for VssStore { fn read( &self, primary_namespace: &str, secondary_namespace: &str, key: &str, @@ -388,7 +282,6 @@ impl Drop for VssStore { struct VssStoreInner { schema_version: VssSchemaVersion, - blocking_client: VssClient, // A secondary client that will only be used for async persistence via `KVStore`, to ensure TCP // connections aren't shared between our outer and the internal runtime. async_client: VssClient, @@ -403,14 +296,13 @@ struct VssStoreInner { impl VssStoreInner { pub(crate) fn new( - schema_version: VssSchemaVersion, blocking_client: VssClient, - async_client: VssClient, store_id: String, - data_encryption_key: [u8; 32], key_obfuscator: KeyObfuscator, entropy_source: RandomBytes, + schema_version: VssSchemaVersion, async_client: VssClient, + store_id: String, data_encryption_key: [u8; 32], key_obfuscator: KeyObfuscator, + entropy_source: RandomBytes, ) -> Self { let locks = Mutex::new(HashMap::new()); Self { schema_version, - blocking_client, async_client, store_id, data_encryption_key, @@ -1026,8 +918,8 @@ mod tests { use super::*; use crate::io::test_utils::do_read_write_remove_list_persist; - #[test] - fn vss_read_write_remove_list_persist() { + #[tokio::test] + async fn vss_read_write_remove_list_persist() { let vss_base_url = std::env::var("TEST_VSS_BASE_URL").unwrap(); let mut rng = rng(); let rand_store_id: String = (0..7).map(|_| rng.sample(Alphanumeric) as char).collect(); diff --git a/src/lib.rs b/src/lib.rs index 5ad0492989..4f79c5adf2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -180,7 +180,7 @@ use types::{ HRNResolver, KeysManager, OnionMessenger, PaymentStore, PeerManager, Router, Scorer, Sweeper, Wallet, }; -pub use types::{ChannelDetails, CustomTlvRecord, PeerDetails, SyncAndAsyncKVStore, UserChannelId}; +pub use types::{ChannelDetails, CustomTlvRecord, PeerDetails, UserChannelId}; pub use vss_client; use crate::scoring::setup_background_pathfinding_scores_sync; diff --git a/src/types.rs b/src/types.rs index 8ff5b29660..a8ce812b8e 100644 --- a/src/types.rs +++ b/src/types.rs @@ -29,7 +29,7 @@ use lightning::routing::gossip; use lightning::routing::router::DefaultRouter; use lightning::routing::scoring::{CombinedScorer, ProbabilisticScoringFeeParameters}; use lightning::sign::InMemorySigner; -use lightning::util::persist::{KVStore, KVStoreSync, MonitorUpdatingPersisterAsync}; +use lightning::util::persist::{KVStore, MonitorUpdatingPersisterAsync}; use lightning::util::ser::{Readable, Writeable, Writer}; use lightning::util::sweep::OutputSweeper; use lightning_block_sync::gossip::GossipVerifier; @@ -46,17 +46,6 @@ use crate::message_handler::NodeCustomMessageHandler; use crate::payment::{PaymentDetails, PendingPaymentDetails}; use crate::runtime::RuntimeSpawner; -/// A supertrait that requires that a type implements both [`KVStore`] and [`KVStoreSync`] at the -/// same time. -pub trait SyncAndAsyncKVStore: KVStore + KVStoreSync {} - -impl SyncAndAsyncKVStore for T -where - T: KVStore, - T: KVStoreSync, -{ -} - pub(crate) trait DynStoreTrait: Send + Sync { fn read_async( &self, primary_namespace: &str, secondary_namespace: &str, key: &str, @@ -70,19 +59,6 @@ pub(crate) trait DynStoreTrait: Send + Sync { fn list_async( &self, primary_namespace: &str, secondary_namespace: &str, ) -> Pin, bitcoin::io::Error>> + Send + 'static>>; - - fn read( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, - ) -> Result, bitcoin::io::Error>; - fn write( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, - ) -> Result<(), bitcoin::io::Error>; - fn remove( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, - ) -> Result<(), bitcoin::io::Error>; - fn list( - &self, primary_namespace: &str, secondary_namespace: &str, - ) -> Result, bitcoin::io::Error>; } impl<'a> KVStore for dyn DynStoreTrait + 'a { @@ -111,32 +87,6 @@ impl<'a> KVStore for dyn DynStoreTrait + 'a { } } -impl<'a> KVStoreSync for dyn DynStoreTrait + 'a { - fn read( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, - ) -> Result, bitcoin::io::Error> { - DynStoreTrait::read(self, primary_namespace, secondary_namespace, key) - } - - fn write( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, - ) -> Result<(), bitcoin::io::Error> { - DynStoreTrait::write(self, primary_namespace, secondary_namespace, key, buf) - } - - fn remove( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, - ) -> Result<(), bitcoin::io::Error> { - DynStoreTrait::remove(self, primary_namespace, secondary_namespace, key, lazy) - } - - fn list( - &self, primary_namespace: &str, secondary_namespace: &str, - ) -> Result, bitcoin::io::Error> { - DynStoreTrait::list(self, primary_namespace, secondary_namespace) - } -} - pub(crate) type DynStore = dyn DynStoreTrait; // Newtype wrapper that implements `KVStore` for `Arc`. This is needed because `KVStore` @@ -172,9 +122,9 @@ impl KVStore for DynStoreRef { } } -pub(crate) struct DynStoreWrapper(pub(crate) T); +pub(crate) struct DynStoreWrapper(pub(crate) T); -impl DynStoreTrait for DynStoreWrapper { +impl DynStoreTrait for DynStoreWrapper { fn read_async( &self, primary_namespace: &str, secondary_namespace: &str, key: &str, ) -> Pin, bitcoin::io::Error>> + Send + 'static>> { @@ -198,30 +148,6 @@ impl DynStoreTrait for DynStoreWrapper ) -> Pin, bitcoin::io::Error>> + Send + 'static>> { Box::pin(KVStore::list(&self.0, primary_namespace, secondary_namespace)) } - - fn read( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, - ) -> Result, bitcoin::io::Error> { - KVStoreSync::read(&self.0, primary_namespace, secondary_namespace, key) - } - - fn write( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, - ) -> Result<(), bitcoin::io::Error> { - KVStoreSync::write(&self.0, primary_namespace, secondary_namespace, key, buf) - } - - fn remove( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, - ) -> Result<(), bitcoin::io::Error> { - KVStoreSync::remove(&self.0, primary_namespace, secondary_namespace, key, lazy) - } - - fn list( - &self, primary_namespace: &str, secondary_namespace: &str, - ) -> Result, bitcoin::io::Error> { - KVStoreSync::list(&self.0, primary_namespace, secondary_namespace) - } } pub(crate) type AsyncPersister = MonitorUpdatingPersisterAsync< diff --git a/tests/common/mod.rs b/tests/common/mod.rs index f2e7191ae4..d7775e67b3 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -50,7 +50,7 @@ use ldk_node::{ use lightning::io; use lightning::ln::msgs::SocketAddress; use lightning::routing::gossip::NodeAlias; -use lightning::util::persist::{KVStore, KVStoreSync}; +use lightning::util::persist::KVStore; use lightning_invoice::{Bolt11InvoiceDescription, Description}; use lightning_persister::fs_store::v1::FilesystemStore; use lightning_types::payment::{PaymentHash, PaymentPreimage}; @@ -1700,32 +1700,6 @@ impl KVStore for TestSyncStore { } } -impl KVStoreSync for TestSyncStore { - fn read( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, - ) -> lightning::io::Result> { - self.inner.read_internal(primary_namespace, secondary_namespace, key) - } - - fn write( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, - ) -> lightning::io::Result<()> { - self.inner.write_internal(primary_namespace, secondary_namespace, key, buf) - } - - fn remove( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, - ) -> lightning::io::Result<()> { - self.inner.remove_internal(primary_namespace, secondary_namespace, key, lazy) - } - - fn list( - &self, primary_namespace: &str, secondary_namespace: &str, - ) -> lightning::io::Result> { - self.inner.list_internal(primary_namespace, secondary_namespace) - } -} - struct TestSyncStoreInner { serializer: tokio::sync::RwLock<()>, test_store: InMemoryStore, @@ -1751,36 +1725,6 @@ impl TestSyncStoreInner { Self { serializer, fs_store, sqlite_store, test_store } } - fn do_list( - &self, primary_namespace: &str, secondary_namespace: &str, - ) -> lightning::io::Result> { - let fs_res = KVStoreSync::list(&self.fs_store, primary_namespace, secondary_namespace); - let sqlite_res = - KVStoreSync::list(&self.sqlite_store, primary_namespace, secondary_namespace); - let test_res = KVStoreSync::list(&self.test_store, primary_namespace, secondary_namespace); - - match fs_res { - Ok(mut list) => { - list.sort(); - - let mut sqlite_list = sqlite_res.unwrap(); - sqlite_list.sort(); - assert_eq!(list, sqlite_list); - - let mut test_list = test_res.unwrap(); - test_list.sort(); - assert_eq!(list, test_list); - - Ok(list) - }, - Err(e) => { - assert!(sqlite_res.is_err()); - assert!(test_res.is_err()); - Err(e) - }, - } - } - async fn do_list_async( &self, primary_namespace: &str, secondary_namespace: &str, ) -> lightning::io::Result> { @@ -1929,123 +1873,4 @@ impl TestSyncStoreInner { }, } } - - fn read_internal( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, - ) -> lightning::io::Result> { - let _guard = self.serializer.blocking_read(); - - let fs_res = KVStoreSync::read(&self.fs_store, primary_namespace, secondary_namespace, key); - let sqlite_res = - KVStoreSync::read(&self.sqlite_store, primary_namespace, secondary_namespace, key); - let test_res = - KVStoreSync::read(&self.test_store, primary_namespace, secondary_namespace, key); - - match fs_res { - Ok(read) => { - assert_eq!(read, sqlite_res.unwrap()); - assert_eq!(read, test_res.unwrap()); - Ok(read) - }, - Err(e) => { - assert!(sqlite_res.is_err()); - assert_eq!(e.kind(), unsafe { sqlite_res.unwrap_err_unchecked().kind() }); - assert!(test_res.is_err()); - assert_eq!(e.kind(), unsafe { test_res.unwrap_err_unchecked().kind() }); - Err(e) - }, - } - } - - fn write_internal( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, - ) -> lightning::io::Result<()> { - let _guard = self.serializer.blocking_write(); - let fs_res = KVStoreSync::write( - &self.fs_store, - primary_namespace, - secondary_namespace, - key, - buf.clone(), - ); - let sqlite_res = KVStoreSync::write( - &self.sqlite_store, - primary_namespace, - secondary_namespace, - key, - buf.clone(), - ); - let test_res = KVStoreSync::write( - &self.test_store, - primary_namespace, - secondary_namespace, - key, - buf.clone(), - ); - - assert!(self - .do_list(primary_namespace, secondary_namespace) - .unwrap() - .contains(&key.to_string())); - - match fs_res { - Ok(()) => { - assert!(sqlite_res.is_ok()); - assert!(test_res.is_ok()); - Ok(()) - }, - Err(e) => { - assert!(sqlite_res.is_err()); - assert!(test_res.is_err()); - Err(e) - }, - } - } - - fn remove_internal( - &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, - ) -> lightning::io::Result<()> { - let _guard = self.serializer.blocking_write(); - let fs_res = - KVStoreSync::remove(&self.fs_store, primary_namespace, secondary_namespace, key, lazy); - let sqlite_res = KVStoreSync::remove( - &self.sqlite_store, - primary_namespace, - secondary_namespace, - key, - lazy, - ); - let test_res = KVStoreSync::remove( - &self.test_store, - primary_namespace, - secondary_namespace, - key, - lazy, - ); - - assert!(!self - .do_list(primary_namespace, secondary_namespace) - .unwrap() - .contains(&key.to_string())); - - match fs_res { - Ok(()) => { - assert!(sqlite_res.is_ok()); - assert!(test_res.is_ok()); - Ok(()) - }, - Err(e) => { - assert!(sqlite_res.is_err()); - assert!(test_res.is_err()); - Err(e) - }, - } - } - - fn list_internal( - &self, primary_namespace: &str, secondary_namespace: &str, - ) -> lightning::io::Result> { - let _guard = self.serializer.blocking_read(); - self.do_list(primary_namespace, secondary_namespace) - } } From 55e0d0328bcf9b8bb89b5a92cc5f77077c1da171 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Wed, 3 Jun 2026 17:31:19 +0200 Subject: [PATCH 17/21] f - Await async VSS store test helper Compile the VSS persistence tests after the shared KV store helper moved to async persistence. Co-Authored-By: HAL 9000 --- src/io/vss_store.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/io/vss_store.rs b/src/io/vss_store.rs index e7e8d031e3..e747639a7d 100644 --- a/src/io/vss_store.rs +++ b/src/io/vss_store.rs @@ -930,7 +930,7 @@ mod tests { VssStoreBuilder::new(entropy, vss_base_url, rand_store_id, Network::Testnet) .build_with_sigs_auth(HashMap::new()) .unwrap(); - do_read_write_remove_list_persist(&vss_store); + do_read_write_remove_list_persist(&vss_store).await; } #[tokio::test(flavor = "multi_thread", worker_threads = 1)] @@ -946,7 +946,7 @@ mod tests { .build_with_sigs_auth(HashMap::new()) .unwrap(); - do_read_write_remove_list_persist(&vss_store); + do_read_write_remove_list_persist(&vss_store).await; drop(vss_store) } } From 393f078df3bbe688f09c6ed5d1adb67a25c77ffa Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Fri, 5 Jun 2026 09:40:59 +0200 Subject: [PATCH 18/21] Add shared store runtime wrapper Add a crate-local runtime wrapper for store backends that need to keep their I/O isolated while shutting down safely from async contexts. Co-Authored-By: HAL 9000 --- src/runtime.rs | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/src/runtime.rs b/src/runtime.rs index 1d8eb32b0a..ab9fc9dad5 100644 --- a/src/runtime.rs +++ b/src/runtime.rs @@ -6,6 +6,8 @@ // accordance with one or both of these licenses. use std::future::Future; +use std::io; +use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; use std::time::Duration; @@ -223,6 +225,60 @@ enum RuntimeMode { Handle(tokio::runtime::Handle), } +pub(crate) struct StoreRuntime { + runtime: Option, +} + +impl StoreRuntime { + pub(crate) fn new( + thread_name_prefix: &'static str, worker_threads: usize, runtime_name: &'static str, + ) -> io::Result { + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .thread_name_fn(move || { + static ATOMIC_ID: AtomicUsize = AtomicUsize::new(0); + let id = ATOMIC_ID.fetch_add(1, Ordering::SeqCst); + format!("{}-{}", thread_name_prefix, id) + }) + .worker_threads(worker_threads) + .max_blocking_threads(worker_threads) + .build() + .map_err(|e| { + io::Error::new( + io::ErrorKind::Other, + format!("Failed to build {runtime_name} runtime: {e}"), + ) + })?; + Ok(Self { runtime: Some(runtime) }) + } + + pub(crate) fn handle(&self) -> &tokio::runtime::Handle { + self.runtime.as_ref().expect("store runtime must be available").handle() + } + + pub(crate) fn spawn(&self, future: F) -> JoinHandle + where + F: Future + Send + 'static, + F::Output: Send + 'static, + { + self.handle().spawn(future) + } + + pub(crate) fn shutdown_background(mut self) { + if let Some(runtime) = self.runtime.take() { + runtime.shutdown_background(); + } + } +} + +impl Drop for StoreRuntime { + fn drop(&mut self) { + if let Some(runtime) = self.runtime.take() { + runtime.shutdown_background(); + } + } +} + pub(crate) struct RuntimeSpawner { runtime: Arc, } From b454e5a65c4d0599fdbbf91fe0d5b792dbb78d79 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Thu, 4 Jun 2026 17:55:51 +0200 Subject: [PATCH 19/21] Isolate VSS persistence from the node runtime Co-Authored-By: HAL 9000 --- src/io/vss_store.rs | 138 ++++++++++++++++++++++++++------------------ 1 file changed, 82 insertions(+), 56 deletions(-) diff --git a/src/io/vss_store.rs b/src/io/vss_store.rs index e747639a7d..4b8cca754b 100644 --- a/src/io/vss_store.rs +++ b/src/io/vss_store.rs @@ -13,7 +13,7 @@ use std::fmt; use std::future::Future; #[cfg(test)] use std::panic::RefUnwindSafe; -use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; use std::time::Duration; @@ -45,6 +45,7 @@ use vss_client::util::storable_builder::{EntropySource, StorableBuilder}; use crate::entropy::NodeEntropy; use crate::io::utils::check_namespace_key_validity; use crate::lnurl_auth::LNURL_AUTH_HARDENED_CHILD_INDEX; +use crate::runtime::StoreRuntime; type CustomRetryPolicy = FilteredRetryPolicy< JitteredRetryPolicy< @@ -77,6 +78,18 @@ const VSS_SCHEMA_VERSION_KEY: &str = "vss_schema_version"; // would hit a blocking case const INTERNAL_RUNTIME_WORKERS: usize = 2; +async fn run_on_internal_runtime( + runtime: Arc, future: impl Future> + Send + 'static, +) -> io::Result +where + T: Send + 'static, +{ + let task = runtime.spawn(future); + task.await.map_err(|e| { + io::Error::new(io::ErrorKind::Other, format!("VSS runtime task failed: {}", e)) + })? +} + /// A [`KVStore`] implementation that writes to and reads from a [VSS] backend. /// /// [VSS]: https://github.com/lightningdevkit/vss-server/blob/main/README.md @@ -85,13 +98,8 @@ pub struct VssStore { // Version counter to ensure that writes are applied in the correct order. It is assumed that read and list // operations aren't sensitive to the order of execution. next_version: AtomicU64, - // A VSS-internal runtime we use to avoid any deadlocks we could hit when waiting on a spawned - // blocking task to finish while the blocked thread had acquired the reactor. In particular, - // this works around a previously-hit case where a concurrent call to - // `PeerManager::process_pending_events` -> `ChannelManager::get_and_clear_pending_msg_events` - // would deadlock when trying to acquire sync `Mutex` locks that are held by the thread - // currently being blocked waiting on the VSS operation to finish. - internal_runtime: Option, + // A VSS-internal runtime that drives VSS I/O independently from the node runtime. + internal_runtime: Option>, } impl VssStore { @@ -100,52 +108,46 @@ impl VssStore { header_provider: Arc, ) -> io::Result { let next_version = AtomicU64::new(1); - let internal_runtime = tokio::runtime::Builder::new_multi_thread() - .enable_all() - .thread_name_fn(|| { - static ATOMIC_ID: AtomicUsize = AtomicUsize::new(0); - let id = ATOMIC_ID.fetch_add(1, Ordering::SeqCst); - format!("ldk-node-vss-runtime-{}", id) - }) - .worker_threads(INTERNAL_RUNTIME_WORKERS) - .max_blocking_threads(INTERNAL_RUNTIME_WORKERS) - .build() - .map_err(|e| { - io::Error::new(io::ErrorKind::Other, format!("Failed to build VSS runtime: {}", e)) - })?; + let internal_runtime = + Arc::new(StoreRuntime::new("ldk-node-vss-runtime", INTERNAL_RUNTIME_WORKERS, "VSS")?); let (data_encryption_key, obfuscation_master_key) = derive_data_encryption_and_obfuscation_keys(&vss_seed); let key_obfuscator = KeyObfuscator::new(obfuscation_master_key); + let setup_key_obfuscator = KeyObfuscator::new(obfuscation_master_key); let mut entropy_seed = [0u8; 32]; getrandom::fill(&mut entropy_seed).expect("Failed to generate random bytes"); let entropy_source = RandomBytes::new(entropy_seed); + let setup_entropy_source = RandomBytes::new(entropy_seed); - let sync_retry_policy = retry_policy(); - let blocking_client = VssClient::new_with_headers( + let setup_retry_policy = retry_policy(); + let setup_client = VssClient::new_with_headers( base_url.clone(), - sync_retry_policy, - header_provider.clone(), + setup_retry_policy, + Arc::clone(&header_provider), ); - let runtime_handle = internal_runtime.handle(); - let schema_version = tokio::task::block_in_place(|| { + let async_retry_policy = retry_policy(); + let async_client = + VssClient::new_with_headers(base_url, async_retry_policy, header_provider); + + let setup_store_id = store_id.clone(); + let runtime_handle = internal_runtime.handle().clone(); + let schema_version = std::thread::spawn(move || { runtime_handle.block_on(async { determine_and_write_schema_version( - &blocking_client, - &store_id, + &setup_client, + &setup_store_id, data_encryption_key, - &key_obfuscator, - &entropy_source, + &setup_key_obfuscator, + &setup_entropy_source, ) .await }) - })?; - - let async_retry_policy = retry_policy(); - let async_client = - VssClient::new_with_headers(base_url, async_retry_policy, header_provider); + }) + .join() + .map_err(|_| io::Error::new(io::ErrorKind::Other, "VSS schema setup task panicked"))??; let inner = Arc::new(VssStoreInner::new( schema_version, @@ -158,6 +160,10 @@ impl VssStore { Ok(Self { inner, next_version, internal_runtime: Some(internal_runtime) }) } + + fn internal_runtime(&self) -> Arc { + Arc::clone(self.internal_runtime.as_ref().expect("VSS runtime must be available")) + } /// Returns a [`VssStoreBuilder`] allowing to build a [`VssStore`]. pub fn builder( node_entropy: NodeEntropy, vss_url: String, store_id: String, network: Network, @@ -200,10 +206,14 @@ impl KVStore for VssStore { let secondary_namespace = secondary_namespace.to_string(); let key = key.to_string(); let inner = Arc::clone(&self.inner); + let runtime = self.internal_runtime(); async move { - inner - .read_internal(&inner.async_client, primary_namespace, secondary_namespace, key) - .await + run_on_internal_runtime(runtime, async move { + inner + .read_internal(&inner.async_client, primary_namespace, secondary_namespace, key) + .await + }) + .await } } fn write( @@ -215,19 +225,23 @@ impl KVStore for VssStore { let secondary_namespace = secondary_namespace.to_string(); let key = key.to_string(); let inner = Arc::clone(&self.inner); + let runtime = self.internal_runtime(); async move { - inner - .write_internal( - &inner.async_client, - inner_lock_ref, - locking_key, - version, - primary_namespace, - secondary_namespace, - key, - buf, - ) - .await + run_on_internal_runtime(runtime, async move { + inner + .write_internal( + &inner.async_client, + inner_lock_ref, + locking_key, + version, + primary_namespace, + secondary_namespace, + key, + buf, + ) + .await + }) + .await } } fn remove( @@ -239,6 +253,7 @@ impl KVStore for VssStore { let secondary_namespace = secondary_namespace.to_string(); let key = key.to_string(); let inner = Arc::clone(&self.inner); + let runtime = self.internal_runtime(); let fut = async move { inner .remove_internal( @@ -254,10 +269,12 @@ impl KVStore for VssStore { }; async move { if lazy { - tokio::task::spawn(async move { fut.await }); + runtime.spawn(async move { + let _ = fut.await; + }); Ok(()) } else { - fut.await + run_on_internal_runtime(runtime, fut).await } } } @@ -267,16 +284,25 @@ impl KVStore for VssStore { let primary_namespace = primary_namespace.to_string(); let secondary_namespace = secondary_namespace.to_string(); let inner = Arc::clone(&self.inner); + let runtime = self.internal_runtime(); async move { - inner.list_internal(&inner.async_client, primary_namespace, secondary_namespace).await + run_on_internal_runtime(runtime, async move { + inner + .list_internal(&inner.async_client, primary_namespace, secondary_namespace) + .await + }) + .await } } } impl Drop for VssStore { fn drop(&mut self) { - let internal_runtime = self.internal_runtime.take(); - tokio::task::block_in_place(move || drop(internal_runtime)); + if let Some(runtime) = self.internal_runtime.take() { + if let Ok(runtime) = Arc::try_unwrap(runtime) { + runtime.shutdown_background(); + } + } } } From 88eb0fb4ce0d49b137dc38e8223a6569b979106c Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Thu, 4 Jun 2026 17:58:56 +0200 Subject: [PATCH 20/21] Isolate PostgreSQL persistence from the node runtime Co-Authored-By: HAL 9000 --- src/io/postgres_store/mod.rs | 144 ++++++++++++++++++++--------------- 1 file changed, 82 insertions(+), 62 deletions(-) diff --git a/src/io/postgres_store/mod.rs b/src/io/postgres_store/mod.rs index c54cbdef12..f62a171b78 100644 --- a/src/io/postgres_store/mod.rs +++ b/src/io/postgres_store/mod.rs @@ -8,7 +8,7 @@ //! Objects related to [`PostgresStore`] live here. use std::collections::HashMap; use std::future::Future; -use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; use lightning::io; @@ -22,6 +22,7 @@ use tokio_postgres::{Config, Error as PgError}; use self::pool::{make_config_connection, ClientConnection, PgTlsConnector, SmallPool}; use crate::io::utils::check_namespace_key_validity; use crate::logger::{log_debug, log_info, LdkLogger, Logger}; +use crate::runtime::StoreRuntime; mod migrations; mod pool; @@ -41,6 +42,18 @@ const PAGE_SIZE: usize = 50; // Keep this small while still allowing progress if one runtime worker blocks on sync store access. const INTERNAL_RUNTIME_WORKERS: usize = 2; +async fn run_on_internal_runtime( + runtime: Arc, future: impl Future> + Send + 'static, +) -> io::Result +where + T: Send + 'static, +{ + let task = runtime.spawn(future); + task.await.map_err(|e| { + io::Error::new(io::ErrorKind::Other, format!("PostgreSQL runtime task failed: {}", e)) + })? +} + fn sql_identifier(identifier: &str) -> io::Result { if identifier.is_empty() || identifier.contains('\0') { return Err(io::Error::new( @@ -101,8 +114,8 @@ pub struct PostgresStore { // operations aren't sensitive to the order of execution. next_write_version: AtomicU64, - // A store-internal runtime used for setup and connection driver tasks. - internal_runtime: Option, + // A store-internal runtime that drives PostgreSQL I/O independently from the node runtime. + internal_runtime: Option>, } // tokio::sync::Mutex (used for the DB client) contains UnsafeCell which opts out of @@ -145,30 +158,16 @@ impl PostgresStore { connection_string: String, db_name: Option, kv_table_name: Option, certificate_pem: Option, logger: Option>, ) -> io::Result { - let internal_runtime = tokio::runtime::Builder::new_multi_thread() - .enable_all() - .thread_name_fn(|| { - static ATOMIC_ID: AtomicUsize = AtomicUsize::new(0); - let id = ATOMIC_ID.fetch_add(1, Ordering::SeqCst); - format!("ldk-node-postgres-runtime-{}", id) - }) - .worker_threads(INTERNAL_RUNTIME_WORKERS) - .max_blocking_threads(INTERNAL_RUNTIME_WORKERS) - .build() - .map_err(|e| { - io::Error::new( - io::ErrorKind::Other, - format!("Failed to build PostgreSQL runtime: {e}"), - ) - })?; + let internal_runtime = Arc::new(StoreRuntime::new( + "ldk-node-postgres-runtime", + INTERNAL_RUNTIME_WORKERS, + "PostgreSQL", + )?); let tls = Self::build_tls_connector(certificate_pem)?; - let runtime_handle = internal_runtime.handle(); - let inner = tokio::task::block_in_place(|| { - runtime_handle.block_on(async { - PostgresStoreInner::new(connection_string, db_name, kv_table_name, tls, logger) - .await - }) - })?; + let inner = run_on_internal_runtime(Arc::clone(&internal_runtime), async move { + PostgresStoreInner::new(connection_string, db_name, kv_table_name, tls, logger).await + }) + .await?; let inner = Arc::new(inner); let next_write_version = AtomicU64::new(1); Ok(Self { inner, next_write_version, internal_runtime: Some(internal_runtime) }) @@ -214,12 +213,18 @@ impl PostgresStore { (inner_lock_ref, version) } + + fn internal_runtime(&self) -> Arc { + Arc::clone(self.internal_runtime.as_ref().expect("PostgreSQL runtime must be available")) + } } impl Drop for PostgresStore { fn drop(&mut self) { if let Some(internal_runtime) = self.internal_runtime.take() { - internal_runtime.shutdown_background(); + if let Ok(internal_runtime) = Arc::try_unwrap(internal_runtime) { + internal_runtime.shutdown_background(); + } } } } @@ -232,7 +237,13 @@ impl KVStore for PostgresStore { let secondary_namespace = secondary_namespace.to_string(); let key = key.to_string(); let inner = Arc::clone(&self.inner); - async move { inner.read_internal(&primary_namespace, &secondary_namespace, &key).await } + let runtime = self.internal_runtime(); + async move { + run_on_internal_runtime(runtime, async move { + inner.read_internal(&primary_namespace, &secondary_namespace, &key).await + }) + .await + } } fn write( @@ -244,18 +255,22 @@ impl KVStore for PostgresStore { let secondary_namespace = secondary_namespace.to_string(); let key = key.to_string(); let inner = Arc::clone(&self.inner); + let runtime = self.internal_runtime(); async move { - inner - .write_internal( - inner_lock_ref, - locking_key, - version, - &primary_namespace, - &secondary_namespace, - &key, - buf, - ) - .await + run_on_internal_runtime(runtime, async move { + inner + .write_internal( + inner_lock_ref, + locking_key, + version, + &primary_namespace, + &secondary_namespace, + &key, + buf, + ) + .await + }) + .await } } @@ -268,17 +283,21 @@ impl KVStore for PostgresStore { let secondary_namespace = secondary_namespace.to_string(); let key = key.to_string(); let inner = Arc::clone(&self.inner); + let runtime = self.internal_runtime(); async move { - inner - .remove_internal( - inner_lock_ref, - locking_key, - version, - &primary_namespace, - &secondary_namespace, - &key, - ) - .await + run_on_internal_runtime(runtime, async move { + inner + .remove_internal( + inner_lock_ref, + locking_key, + version, + &primary_namespace, + &secondary_namespace, + &key, + ) + .await + }) + .await } } @@ -288,16 +307,13 @@ impl KVStore for PostgresStore { let primary_namespace = primary_namespace.to_string(); let secondary_namespace = secondary_namespace.to_string(); let inner = Arc::clone(&self.inner); - async move { inner.list_internal(&primary_namespace, &secondary_namespace).await } - } -} - -impl PostgresStore { - fn internal_runtime(&self) -> io::Result<&tokio::runtime::Runtime> { - self.internal_runtime.as_ref().ok_or_else(|| { - debug_assert!(false, "Failed to access internal PostgreSQL runtime"); - io::Error::new(io::ErrorKind::Other, "Failed to access internal PostgreSQL runtime") - }) + let runtime = self.internal_runtime(); + async move { + run_on_internal_runtime(runtime, async move { + inner.list_internal(&primary_namespace, &secondary_namespace).await + }) + .await + } } } @@ -308,10 +324,14 @@ impl PaginatedKVStore for PostgresStore { let primary_namespace = primary_namespace.to_string(); let secondary_namespace = secondary_namespace.to_string(); let inner = Arc::clone(&self.inner); + let runtime = self.internal_runtime(); async move { - inner - .list_paginated_internal(&primary_namespace, &secondary_namespace, page_token) - .await + run_on_internal_runtime(runtime, async move { + inner + .list_paginated_internal(&primary_namespace, &secondary_namespace, page_token) + .await + }) + .await } } } From be2b74d170028c1acf7c2c316559c29f30046aeb Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Fri, 5 Jun 2026 10:22:27 +0200 Subject: [PATCH 21/21] Enable Tokio eager handoff for node runtimes Enable Tokio's eager driver handoff when building with tokio_unstable so node-owned runtimes can use the dedicated driver handoff path where available. Build binding artifacts and selected CI coverage with tokio_unstable so the cfg-gated runtime path remains exercised. Co-Authored-By: HAL 9000 --- .github/workflows/benchmarks.yml | 2 +- .github/workflows/rust.yml | 4 ++-- scripts/uniffi_bindgen_generate_kotlin.sh | 5 +++++ scripts/uniffi_bindgen_generate_kotlin_android.sh | 11 ++++++++--- scripts/uniffi_bindgen_generate_python.sh | 5 +++++ scripts/uniffi_bindgen_generate_swift.sh | 5 +++++ src/runtime.rs | 6 +++++- 7 files changed, 31 insertions(+), 7 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 6d0056e9aa..cd3980b9af 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -43,4 +43,4 @@ jobs: echo "ELECTRS_EXE=$( pwd )/bin/electrs-${{ runner.os }}-${{ runner.arch }}" >> "$GITHUB_ENV" - name: Run benchmarks run: | - cargo bench + RUSTFLAGS="--cfg tokio_unstable" cargo bench diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index b2575aca1f..16064fa45c 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -84,7 +84,7 @@ jobs: - name: Test on Rust ${{ matrix.toolchain }} if: "matrix.platform != 'windows-latest'" run: | - RUSTFLAGS="--cfg no_download --cfg cycle_tests" cargo test + RUSTFLAGS="--cfg no_download --cfg cycle_tests --cfg tokio_unstable" cargo test - name: Test with UniFFI support on Rust ${{ matrix.toolchain }} if: "matrix.platform != 'windows-latest' && matrix.build-uniffi" run: | @@ -114,4 +114,4 @@ jobs: - uses: actions/checkout@v6 - uses: dtolnay/rust-toolchain@nightly - uses: dtolnay/install@cargo-docs-rs - - run: cargo docs-rs \ No newline at end of file + - run: cargo docs-rs diff --git a/scripts/uniffi_bindgen_generate_kotlin.sh b/scripts/uniffi_bindgen_generate_kotlin.sh index dc0237ba68..f82d5c0d0d 100755 --- a/scripts/uniffi_bindgen_generate_kotlin.sh +++ b/scripts/uniffi_bindgen_generate_kotlin.sh @@ -5,6 +5,11 @@ PROJECT_DIR="ldk-node-jvm" PACKAGE_DIR="org/lightningdevkit/ldknode" UNIFFI_BINDGEN_BIN="cargo run --manifest-path bindings/uniffi-bindgen/Cargo.toml" +case " ${RUSTFLAGS:-} " in + *" --cfg tokio_unstable "*|*" --cfg=tokio_unstable "*) ;; + *) export RUSTFLAGS="${RUSTFLAGS:+$RUSTFLAGS }--cfg tokio_unstable" ;; +esac + if [[ "$OSTYPE" == "linux-gnu"* ]]; then rustup target add x86_64-unknown-linux-gnu || exit 1 cargo build --release --target x86_64-unknown-linux-gnu --features uniffi || exit 1 diff --git a/scripts/uniffi_bindgen_generate_kotlin_android.sh b/scripts/uniffi_bindgen_generate_kotlin_android.sh index 161292857c..d0eb8654d3 100755 --- a/scripts/uniffi_bindgen_generate_kotlin_android.sh +++ b/scripts/uniffi_bindgen_generate_kotlin_android.sh @@ -5,6 +5,11 @@ TARGET_DIR="target" PROJECT_DIR="ldk-node-android" UNIFFI_BINDGEN_BIN="cargo run --manifest-path bindings/uniffi-bindgen/Cargo.toml" +case " ${RUSTFLAGS:-} " in + *" --cfg tokio_unstable "*|*" --cfg=tokio_unstable "*) RUSTFLAGS_WITH_TOKIO_UNSTABLE="${RUSTFLAGS:-}" ;; + *) RUSTFLAGS_WITH_TOKIO_UNSTABLE="${RUSTFLAGS:+$RUSTFLAGS }--cfg tokio_unstable" ;; +esac + export_variable_if_not_present() { local name="$1" local value="$2" @@ -35,9 +40,9 @@ case "$OSTYPE" in PATH="$ANDROID_NDK_ROOT/toolchains/llvm/prebuilt/$LLVM_ARCH_PATH/bin:$PATH" rustup target add x86_64-linux-android aarch64-linux-android armv7-linux-androideabi -RUSTFLAGS="-C link-args=-Wl,-z,max-page-size=16384,-z,common-page-size=16384" CFLAGS="-D__ANDROID_MIN_SDK_VERSION__=21" AR=llvm-ar CARGO_TARGET_X86_64_LINUX_ANDROID_LINKER="x86_64-linux-android21-clang" CC="x86_64-linux-android21-clang" cargo build --profile release-smaller --features uniffi --target x86_64-linux-android || exit 1 -RUSTFLAGS="-C link-args=-Wl,-z,max-page-size=16384,-z,common-page-size=16384" CFLAGS="-D__ANDROID_MIN_SDK_VERSION__=21" AR=llvm-ar CARGO_TARGET_ARMV7_LINUX_ANDROIDEABI_LINKER="armv7a-linux-androideabi21-clang" CC="armv7a-linux-androideabi21-clang" cargo build --profile release-smaller --features uniffi --target armv7-linux-androideabi || exit 1 -RUSTFLAGS="-C link-args=-Wl,-z,max-page-size=16384,-z,common-page-size=16384" CFLAGS="-D__ANDROID_MIN_SDK_VERSION__=21" AR=llvm-ar CARGO_TARGET_AARCH64_LINUX_ANDROID_LINKER="aarch64-linux-android21-clang" CC="aarch64-linux-android21-clang" cargo build --profile release-smaller --features uniffi --target aarch64-linux-android || exit 1 +RUSTFLAGS="$RUSTFLAGS_WITH_TOKIO_UNSTABLE -C link-args=-Wl,-z,max-page-size=16384,-z,common-page-size=16384" CFLAGS="-D__ANDROID_MIN_SDK_VERSION__=21" AR=llvm-ar CARGO_TARGET_X86_64_LINUX_ANDROID_LINKER="x86_64-linux-android21-clang" CC="x86_64-linux-android21-clang" cargo build --profile release-smaller --features uniffi --target x86_64-linux-android || exit 1 +RUSTFLAGS="$RUSTFLAGS_WITH_TOKIO_UNSTABLE -C link-args=-Wl,-z,max-page-size=16384,-z,common-page-size=16384" CFLAGS="-D__ANDROID_MIN_SDK_VERSION__=21" AR=llvm-ar CARGO_TARGET_ARMV7_LINUX_ANDROIDEABI_LINKER="armv7a-linux-androideabi21-clang" CC="armv7a-linux-androideabi21-clang" cargo build --profile release-smaller --features uniffi --target armv7-linux-androideabi || exit 1 +RUSTFLAGS="$RUSTFLAGS_WITH_TOKIO_UNSTABLE -C link-args=-Wl,-z,max-page-size=16384,-z,common-page-size=16384" CFLAGS="-D__ANDROID_MIN_SDK_VERSION__=21" AR=llvm-ar CARGO_TARGET_AARCH64_LINUX_ANDROID_LINKER="aarch64-linux-android21-clang" CC="aarch64-linux-android21-clang" cargo build --profile release-smaller --features uniffi --target aarch64-linux-android || exit 1 $UNIFFI_BINDGEN_BIN generate bindings/ldk_node.udl --lib-file "$TARGET_DIR"/x86_64-linux-android/release-smaller/libldk_node.so --language kotlin --config uniffi-android.toml -o "$BINDINGS_DIR"/"$PROJECT_DIR"/lib/src/main/kotlin || exit 1 JNI_LIB_DIR="$BINDINGS_DIR"/"$PROJECT_DIR"/lib/src/main/jniLibs/ diff --git a/scripts/uniffi_bindgen_generate_python.sh b/scripts/uniffi_bindgen_generate_python.sh index 50ba450b76..abc4088ffa 100755 --- a/scripts/uniffi_bindgen_generate_python.sh +++ b/scripts/uniffi_bindgen_generate_python.sh @@ -2,6 +2,11 @@ BINDINGS_DIR="./bindings/python/src/ldk_node" UNIFFI_BINDGEN_BIN="cargo run --manifest-path bindings/uniffi-bindgen/Cargo.toml" +case " ${RUSTFLAGS:-} " in + *" --cfg tokio_unstable "*|*" --cfg=tokio_unstable "*) ;; + *) export RUSTFLAGS="${RUSTFLAGS:+$RUSTFLAGS }--cfg tokio_unstable" ;; +esac + if [[ "$OSTYPE" == "linux-gnu"* ]]; then DYNAMIC_LIB_PATH="./target/release-smaller/libldk_node.so" else diff --git a/scripts/uniffi_bindgen_generate_swift.sh b/scripts/uniffi_bindgen_generate_swift.sh index d4c900e405..d69ac1fbea 100755 --- a/scripts/uniffi_bindgen_generate_swift.sh +++ b/scripts/uniffi_bindgen_generate_swift.sh @@ -4,6 +4,11 @@ set -eox pipefail BINDINGS_DIR="./bindings/swift" UNIFFI_BINDGEN_BIN="cargo run --manifest-path bindings/uniffi-bindgen/Cargo.toml" +case " ${RUSTFLAGS:-} " in + *" --cfg tokio_unstable "*|*" --cfg=tokio_unstable "*) ;; + *) export RUSTFLAGS="${RUSTFLAGS:+$RUSTFLAGS }--cfg tokio_unstable" ;; +esac + mkdir -p $BINDINGS_DIR # Install rust target toolchains diff --git a/src/runtime.rs b/src/runtime.rs index ab9fc9dad5..f2fa0c982d 100644 --- a/src/runtime.rs +++ b/src/runtime.rs @@ -32,7 +32,11 @@ impl Runtime { let mode = match tokio::runtime::Handle::try_current() { Ok(handle) => RuntimeMode::Handle(handle), Err(_) => { - let rt = tokio::runtime::Builder::new_multi_thread().enable_all().build()?; + let mut runtime_builder = tokio::runtime::Builder::new_multi_thread(); + runtime_builder.enable_all(); + #[cfg(tokio_unstable)] + runtime_builder.enable_eager_driver_handoff(); + let rt = runtime_builder.build()?; RuntimeMode::Owned(rt) }, };