From 95c8b2423302105a5e359ac4ece6b6cd41f2f457 Mon Sep 17 00:00:00 2001 From: benthecarman Date: Tue, 28 Apr 2026 18:03:53 -0700 Subject: [PATCH 1/3] Bump rust-lightning to commit 2313bd5 Update rust-lightning to commit `2313bd584d2c46a50d67b8266f488c07516e0b3c` and bump `bitcoin-payment-instructions` to a fork commit that targets the same rust-lightning revision. Generated with the help of Claude Opus 4.7. --- Cargo.toml | 28 ++++---- bindings/ldk_node.udl | 1 - src/builder.rs | 14 ++-- src/chain/bitcoind.rs | 145 +++++++++++++++++------------------------ src/event.rs | 6 +- src/lib.rs | 30 +++++---- src/liquidity.rs | 8 ++- src/payment/unified.rs | 5 ++ src/wallet/mod.rs | 2 +- 9 files changed, 118 insertions(+), 121 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index e8364c909..3d975abec 100755 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,18 +40,18 @@ default = [] #lightning-macros = { version = "0.2.0" } #lightning-dns-resolver = { version = "0.3.0" } -lightning = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "38a62c32454d3eac22578144c479dbf9a6d9bff6", features = ["std"] } -lightning-types = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "38a62c32454d3eac22578144c479dbf9a6d9bff6" } -lightning-invoice = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "38a62c32454d3eac22578144c479dbf9a6d9bff6", features = ["std"] } -lightning-net-tokio = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "38a62c32454d3eac22578144c479dbf9a6d9bff6" } -lightning-persister = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "38a62c32454d3eac22578144c479dbf9a6d9bff6", features = ["tokio"] } -lightning-background-processor = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "38a62c32454d3eac22578144c479dbf9a6d9bff6" } -lightning-rapid-gossip-sync = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "38a62c32454d3eac22578144c479dbf9a6d9bff6" } -lightning-block-sync = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "38a62c32454d3eac22578144c479dbf9a6d9bff6", features = ["rest-client", "rpc-client", "tokio"] } -lightning-transaction-sync = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "38a62c32454d3eac22578144c479dbf9a6d9bff6", features = ["esplora-async-https", "time", "electrum-rustls-ring"] } -lightning-liquidity = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "38a62c32454d3eac22578144c479dbf9a6d9bff6", features = ["std"] } -lightning-macros = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "38a62c32454d3eac22578144c479dbf9a6d9bff6" } -lightning-dns-resolver = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "38a62c32454d3eac22578144c479dbf9a6d9bff6" } +lightning = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "2313bd584d2c46a50d67b8266f488c07516e0b3c", features = ["std"] } +lightning-types = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "2313bd584d2c46a50d67b8266f488c07516e0b3c" } +lightning-invoice = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "2313bd584d2c46a50d67b8266f488c07516e0b3c", features = ["std"] } +lightning-net-tokio = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "2313bd584d2c46a50d67b8266f488c07516e0b3c" } +lightning-persister = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "2313bd584d2c46a50d67b8266f488c07516e0b3c", features = ["tokio"] } +lightning-background-processor = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "2313bd584d2c46a50d67b8266f488c07516e0b3c" } +lightning-rapid-gossip-sync = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "2313bd584d2c46a50d67b8266f488c07516e0b3c" } +lightning-block-sync = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "2313bd584d2c46a50d67b8266f488c07516e0b3c", features = ["rest-client", "rpc-client", "tokio"] } +lightning-transaction-sync = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "2313bd584d2c46a50d67b8266f488c07516e0b3c", features = ["esplora-async-https", "time", "electrum-rustls-ring"] } +lightning-liquidity = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "2313bd584d2c46a50d67b8266f488c07516e0b3c", features = ["std"] } +lightning-macros = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "2313bd584d2c46a50d67b8266f488c07516e0b3c" } +lightning-dns-resolver = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "2313bd584d2c46a50d67b8266f488c07516e0b3c" } bdk_chain = { version = "0.23.0", default-features = false, features = ["std"] } bdk_esplora = { version = "0.22.0", default-features = false, features = ["async-https-rustls", "tokio"]} @@ -81,13 +81,13 @@ async-trait = { version = "0.1", default-features = false } vss-client = { package = "vss-client-ng", version = "0.5" } prost = { version = "0.11.6", default-features = false} #bitcoin-payment-instructions = { version = "0.6" } -bitcoin-payment-instructions = { git = "https://github.com/jkczyz/bitcoin-payment-instructions", rev = "a7b32d5fded9bb45f73bf82e6d7187adf705171c" } +bitcoin-payment-instructions = { git = "https://github.com/benthecarman/bitcoin-payment-instructions", rev = "23bb47b2d568571c3191d59881ff048d21537ebd" } [target.'cfg(windows)'.dependencies] winapi = { version = "0.3", features = ["winbase"] } [dev-dependencies] -lightning = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "38a62c32454d3eac22578144c479dbf9a6d9bff6", features = ["std", "_test_utils"] } +lightning = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "2313bd584d2c46a50d67b8266f488c07516e0b3c", features = ["std", "_test_utils"] } rand = { version = "0.9.2", default-features = false, features = ["std", "thread_rng", "os_rng"] } proptest = "1.0.0" regex = "1.5.6" diff --git a/bindings/ldk_node.udl b/bindings/ldk_node.udl index c32604708..8551327e8 100644 --- a/bindings/ldk_node.udl +++ b/bindings/ldk_node.udl @@ -233,7 +233,6 @@ enum NodeError { typedef dictionary NodeStatus; -[Remote] dictionary BestBlock { BlockHash block_hash; u32 height; diff --git a/src/builder.rs b/src/builder.rs index b0ff1d03b..6e5076480 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -19,7 +19,7 @@ use bdk_wallet::{KeychainKind, Wallet as BdkWallet}; use bitcoin::bip32::{ChildNumber, Xpriv}; use bitcoin::key::Secp256k1; use bitcoin::secp256k1::PublicKey; -use bitcoin::{BlockHash, Network}; +use bitcoin::Network; use bitcoin_payment_instructions::dns_resolver::DNSHrnResolver; use bitcoin_payment_instructions::onion_message_resolver::LDKOnionMessageDNSSECHrnResolver; use lightning::chain::{chainmonitor, BestBlock}; @@ -1660,8 +1660,12 @@ fn build_with_store_internal( // If we act as an LSPS2 service, set the HTLC-value-in-flight to 100% of the channel value // to ensure we can forward the initial payment. - user_config.channel_handshake_config.max_inbound_htlc_value_in_flight_percent_of_channel = - 100; + user_config + .channel_handshake_config + .announced_channel_max_inbound_htlc_value_in_flight_percentage = 100; + user_config + .channel_handshake_config + .unannounced_channel_max_inbound_htlc_value_in_flight_percentage = 100; } if let Some(role) = async_payments_role { @@ -1695,8 +1699,8 @@ fn build_with_store_internal( user_config, channel_monitor_references, ); - let (_hash, channel_manager) = - <(BlockHash, ChannelManager)>::read(&mut &*reader, read_args).map_err(|e| { + let (_best_block, channel_manager) = + <(BestBlock, ChannelManager)>::read(&mut &*reader, read_args).map_err(|e| { log_error!(logger, "Failed to read channel manager from store: {}", e); BuildError::ReadFailed })?; diff --git a/src/chain/bitcoind.rs b/src/chain/bitcoind.rs index 2bf059f4e..b414a290e 100644 --- a/src/chain/bitcoind.rs +++ b/src/chain/bitcoind.rs @@ -5,7 +5,7 @@ // http://opensource.org/licenses/MIT>, at your option. You may not use this file except in // accordance with one or both of these licenses. -use std::collections::{HashMap, VecDeque}; +use std::collections::HashMap; use std::fmt; use std::future::Future; use std::sync::atomic::{AtomicU64, Ordering}; @@ -25,8 +25,7 @@ use lightning_block_sync::poll::{ChainPoller, ChainTip, ValidatedBlockHeader}; use lightning_block_sync::rest::RestClient; use lightning_block_sync::rpc::{RpcClient, RpcClientError}; use lightning_block_sync::{ - BlockData, BlockHeaderData, BlockSource, BlockSourceError, BlockSourceErrorKind, Cache, - SpvClient, + BlockData, BlockHeaderData, BlockSource, BlockSourceError, BlockSourceErrorKind, SpvClient, }; use serde::Serialize; @@ -47,9 +46,12 @@ use crate::{Error, NodeMetrics}; const CHAIN_POLLING_INTERVAL_SECS: u64 = 2; const CHAIN_POLLING_TIMEOUT_SECS: u64 = 10; +type LongLivedSpvClient = + SpvClient, BitcoindClient>, Arc>; + pub(super) struct BitcoindChainSource { api_client: Arc, - header_cache: tokio::sync::Mutex, + spv_client: tokio::sync::Mutex>, latest_chain_tip: RwLock>, wallet_polling_status: Mutex, fee_estimator: Arc, @@ -72,12 +74,12 @@ impl BitcoindChainSource { rpc_password.clone(), )); - let header_cache = tokio::sync::Mutex::new(BoundedHeaderCache::new()); + let spv_client = tokio::sync::Mutex::new(None); let latest_chain_tip = RwLock::new(None); let wallet_polling_status = Mutex::new(WalletSyncStatus::Completed); Self { api_client, - header_cache, + spv_client, latest_chain_tip, wallet_polling_status, fee_estimator, @@ -103,13 +105,13 @@ impl BitcoindChainSource { rpc_password, )); - let header_cache = tokio::sync::Mutex::new(BoundedHeaderCache::new()); + let spv_client = tokio::sync::Mutex::new(None); let latest_chain_tip = RwLock::new(None); let wallet_polling_status = Mutex::new(WalletSyncStatus::Completed); Self { api_client, - header_cache, + spv_client, latest_chain_tip, wallet_polling_status, fee_estimator, @@ -153,14 +155,14 @@ impl BitcoindChainSource { return; } - let channel_manager_best_block_hash = channel_manager.current_best_block().block_hash; - let sweeper_best_block_hash = output_sweeper.current_best_block().block_hash; - let onchain_wallet_best_block_hash = onchain_wallet.current_best_block().block_hash; + let channel_manager_best_block = channel_manager.current_best_block(); + let sweeper_best_block = output_sweeper.current_best_block(); + let onchain_wallet_best_block = onchain_wallet.current_best_block(); let mut chain_listeners = vec![ - (onchain_wallet_best_block_hash, &*onchain_wallet as &(dyn Listen + Send + Sync)), - (channel_manager_best_block_hash, &*channel_manager as &(dyn Listen + Send + Sync)), - (sweeper_best_block_hash, &*output_sweeper as &(dyn Listen + Send + Sync)), + (onchain_wallet_best_block, &*onchain_wallet as &(dyn Listen + Send + Sync)), + (channel_manager_best_block, &*channel_manager as &(dyn Listen + Send + Sync)), + (sweeper_best_block, &*output_sweeper as &(dyn Listen + Send + Sync)), ]; // TODO: Eventually we might want to see if we can synchronize `ChannelMonitor`s @@ -168,31 +170,28 @@ impl BitcoindChainSource { // trivial as we load them on initialization (in the `Builder`) and only gain // network access during `start`. For now, we just make sure we get the worst known // block hash and sychronize them via `ChainMonitor`. - if let Some(worst_channel_monitor_block_hash) = chain_monitor + if let Some(worst_channel_monitor_best_block) = chain_monitor .list_monitors() .iter() .flat_map(|channel_id| chain_monitor.get_monitor(*channel_id)) .map(|m| m.current_best_block()) .min_by_key(|b| b.height) - .map(|b| b.block_hash) { chain_listeners.push(( - worst_channel_monitor_block_hash, + worst_channel_monitor_best_block, &*chain_monitor as &(dyn Listen + Send + Sync), )); } - let mut locked_header_cache = self.header_cache.lock().await; let now = SystemTime::now(); match synchronize_listeners( self.api_client.as_ref(), self.config.network, - &mut *locked_header_cache, chain_listeners.clone(), ) .await { - Ok(chain_tip) => { + Ok((header_cache, chain_tip)) => { { let elapsed_ms = now.elapsed().map(|d| d.as_millis()).unwrap_or(0); log_info!( @@ -201,6 +200,23 @@ impl BitcoindChainSource { elapsed_ms, ); *self.latest_chain_tip.write().expect("lock") = Some(chain_tip); + + let chain_listener = Arc::new(ChainListener { + onchain_wallet: Arc::clone(&onchain_wallet), + channel_manager: Arc::clone(&channel_manager), + chain_monitor: Arc::clone(&chain_monitor), + output_sweeper: Arc::clone(&output_sweeper), + }); + let chain_poller = + ChainPoller::new(Arc::clone(&self.api_client), self.config.network); + let mut locked_spv_client = self.spv_client.lock().await; + *locked_spv_client = Some(SpvClient::new( + chain_tip, + chain_poller, + header_cache, + chain_listener, + )); + let unix_time_secs_opt = SystemTime::now().duration_since(UNIX_EPOCH).ok().map(|d| d.as_secs()); update_and_persist_node_metrics( @@ -394,36 +410,31 @@ impl BitcoindChainSource { async fn poll_and_update_listeners_inner( &self, onchain_wallet: Arc, channel_manager: Arc, - chain_monitor: Arc, output_sweeper: Arc, + _chain_monitor: Arc, _output_sweeper: Arc, ) -> Result<(), Error> { - let latest_chain_tip_opt = self.latest_chain_tip.read().expect("lock").clone(); - let chain_tip = - if let Some(tip) = latest_chain_tip_opt { tip } else { self.poll_chain_tip().await? }; - - let mut locked_header_cache = self.header_cache.lock().await; - let chain_poller = ChainPoller::new(Arc::clone(&self.api_client), self.config.network); - let chain_listener = ChainListener { - onchain_wallet: Arc::clone(&onchain_wallet), - channel_manager: Arc::clone(&channel_manager), - chain_monitor: Arc::clone(&chain_monitor), - output_sweeper, - }; - let mut spv_client = - SpvClient::new(chain_tip, chain_poller, &mut *locked_header_cache, &chain_listener); + // Ensure `latest_chain_tip` is populated even if the polling loop runs before + // `synchronize_listeners` has completed. + if self.latest_chain_tip.read().expect("lock").is_none() { + self.poll_chain_tip().await?; + } - let now = SystemTime::now(); - match spv_client.poll_best_tip().await { - Ok((ChainTip::Better(tip), true)) => { - let elapsed_ms = now.elapsed().map(|d| d.as_millis()).unwrap_or(0); - log_trace!(self.logger, "Finished polling best tip in {}ms", elapsed_ms); - *self.latest_chain_tip.write().expect("lock") = Some(tip); - }, - Ok(_) => {}, - Err(e) => { - log_error!(self.logger, "Failed to poll for chain data: {:?}", e); - return Err(Error::TxSyncFailed); - }, + let mut locked_spv_client = self.spv_client.lock().await; + if let Some(spv_client) = locked_spv_client.as_mut() { + let now = SystemTime::now(); + match spv_client.poll_best_tip().await { + Ok((ChainTip::Better(tip), true)) => { + let elapsed_ms = now.elapsed().map(|d| d.as_millis()).unwrap_or(0); + log_trace!(self.logger, "Finished polling best tip in {}ms", elapsed_ms); + *self.latest_chain_tip.write().expect("lock") = Some(tip); + }, + Ok(_) => {}, + Err(e) => { + log_error!(self.logger, "Failed to poll for chain data: {:?}", e); + return Err(Error::TxSyncFailed); + }, + } } + drop(locked_spv_client); let cur_height = channel_manager.current_best_block().height; @@ -1350,46 +1361,6 @@ pub(crate) enum FeeRateEstimationMode { Conservative, } -const MAX_HEADER_CACHE_ENTRIES: usize = 100; - -pub(crate) struct BoundedHeaderCache { - header_map: HashMap, - recently_seen: VecDeque, -} - -impl BoundedHeaderCache { - pub(crate) fn new() -> Self { - let header_map = HashMap::new(); - let recently_seen = VecDeque::new(); - Self { header_map, recently_seen } - } -} - -impl Cache for BoundedHeaderCache { - fn look_up(&self, block_hash: &BlockHash) -> Option<&ValidatedBlockHeader> { - self.header_map.get(block_hash) - } - - fn block_connected(&mut self, block_hash: BlockHash, block_header: ValidatedBlockHeader) { - self.recently_seen.push_back(block_hash); - self.header_map.insert(block_hash, block_header); - - if self.header_map.len() >= MAX_HEADER_CACHE_ENTRIES { - // Keep dropping old entries until we've actually removed a header entry. - while let Some(oldest_entry) = self.recently_seen.pop_front() { - if self.header_map.remove(&oldest_entry).is_some() { - break; - } - } - } - } - - fn block_disconnected(&mut self, block_hash: &BlockHash) -> Option { - self.recently_seen.retain(|e| e != block_hash); - self.header_map.remove(block_hash) - } -} - pub(crate) struct ChainListener { pub(crate) onchain_wallet: Arc, pub(crate) channel_manager: Arc, diff --git a/src/event.rs b/src/event.rs index 3161daa2a..0fe735847 100644 --- a/src/event.rs +++ b/src/event.rs @@ -1279,7 +1279,11 @@ where // now we just bump the default for all channels. channel_override_config = Some(ChannelConfigOverrides { handshake_overrides: Some(ChannelHandshakeConfigUpdate { - max_inbound_htlc_value_in_flight_percent_of_channel: Some(100), + announced_channel_max_inbound_htlc_value_in_flight_percentage: Some( + 100, + ), + unannounced_channel_max_inbound_htlc_value_in_flight_percentage: + Some(100), ..Default::default() }), update_overrides: Some(ChannelConfigUpdate { diff --git a/src/lib.rs b/src/lib.rs index faeb6d339..cc0b2398c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -145,7 +145,7 @@ use gossip::GossipSource; use graph::NetworkGraph; use io::utils::update_and_persist_node_metrics; pub use lightning; -use lightning::chain::BestBlock; +use lightning::chain::BestBlock as LdkBestBlock; use lightning::impl_writeable_tlv_based; use lightning::ln::chan_utils::FUNDING_TRANSACTION_WITNESS_WEIGHT; use lightning::ln::channel_state::ChannelDetails as LdkChannelDetails; @@ -1200,7 +1200,7 @@ impl Node { if !announce_for_forwarding { user_config .channel_handshake_config - .max_inbound_htlc_value_in_flight_percent_of_channel = 100; + .unannounced_channel_max_inbound_htlc_value_in_flight_percentage = 100; } let push_msat = push_to_counterparty_msat.unwrap_or(0); @@ -1694,15 +1694,8 @@ impl Node { value: Amount::from_sat(splice_amount_sats), script_pubkey: address.script_pubkey(), }]; - let contribution = self - .runtime - .block_on(funding_template.splice_out( - outputs, - min_feerate, - max_feerate, - Arc::clone(&self.wallet), - )) - .map_err(|e| { + let contribution = + funding_template.splice_out(outputs, min_feerate, max_feerate).map_err(|e| { log_error!(self.logger, "Failed to splice channel: {}", e); Error::ChannelSplicingFailed })?; @@ -2056,6 +2049,21 @@ impl Drop for Node { } } +/// The best block to which our wallets are currently synced. +#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)] +pub struct BestBlock { + /// The block's hash. + pub block_hash: bitcoin::BlockHash, + /// The height at which the block was confirmed. + pub height: u32, +} + +impl From for BestBlock { + fn from(bb: LdkBestBlock) -> Self { + Self { block_hash: bb.block_hash, height: bb.height } + } +} + /// Represents the status of the [`Node`]. #[derive(Clone, Debug, PartialEq, Eq)] #[cfg_attr(feature = "uniffi", derive(uniffi::Record))] diff --git a/src/liquidity.rs b/src/liquidity.rs index 9f02af886..f31c02f6b 100644 --- a/src/liquidity.rs +++ b/src/liquidity.rs @@ -785,7 +785,13 @@ where debug_assert_eq!( config .channel_handshake_config - .max_inbound_htlc_value_in_flight_percent_of_channel, + .announced_channel_max_inbound_htlc_value_in_flight_percentage, + 100 + ); + debug_assert_eq!( + config + .channel_handshake_config + .unannounced_channel_max_inbound_htlc_value_in_flight_percentage, 100 ); debug_assert!(config.accept_forwards_to_priv_channels); diff --git a/src/payment/unified.rs b/src/payment/unified.rs index 9352ee974..83975c41e 100644 --- a/src/payment/unified.rs +++ b/src/payment/unified.rs @@ -256,6 +256,7 @@ impl UnifiedPayment { PaymentMethod::LightningBolt12(_) => 0, PaymentMethod::LightningBolt11(_) => 1, PaymentMethod::OnChain(_) => 2, + PaymentMethod::Cashu(_) => 3, }); for method in sorted_payment_methods { @@ -331,6 +332,10 @@ impl UnifiedPayment { let txid = self.onchain_payment.send_to_address(&address, amt_sats, None)?; return Ok(UnifiedPaymentResult::Onchain { txid }); }, + PaymentMethod::Cashu(_) => { + log_error!(self.logger, "Cashu payment methods are not supported. Skipping."); + continue; + }, } } diff --git a/src/wallet/mod.rs b/src/wallet/mod.rs index cb982e303..69e806b85 100644 --- a/src/wallet/mod.rs +++ b/src/wallet/mod.rs @@ -138,7 +138,7 @@ impl Wallet { pub(crate) fn current_best_block(&self) -> BestBlock { let checkpoint = self.inner.lock().expect("lock").latest_checkpoint(); - BestBlock { block_hash: checkpoint.hash(), height: checkpoint.height() } + BestBlock::new(checkpoint.hash(), checkpoint.height()) } pub(crate) fn apply_update(&self, update: impl Into) -> Result<(), Error> { From 3fd0194f0b5f45fe475eb7edeb73f55d3d3985d3 Mon Sep 17 00:00:00 2001 From: benthecarman Date: Mon, 13 Apr 2026 12:36:05 -0500 Subject: [PATCH 2/3] Allow building 0.7.0 node with different store --- tests/common/mod.rs | 4 ++++ tests/integration_tests_rust.rs | 26 +++++++++++++------------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 850c6f22c..107c846e4 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -334,6 +334,7 @@ pub(crate) enum TestChainSource<'a> { pub(crate) enum TestStoreType { TestSyncStore, Sqlite, + FilesystemStore, } impl Default for TestStoreType { @@ -507,6 +508,9 @@ pub(crate) fn setup_node(chain_source: &TestChainSource, config: TestConfig) -> builder.build_with_store(config.node_entropy.into(), kv_store).unwrap() }, TestStoreType::Sqlite => builder.build(config.node_entropy.into()).unwrap(), + TestStoreType::FilesystemStore => { + builder.build_with_fs_store(config.node_entropy.into()).unwrap() + }, }; if config.recovery_mode { diff --git a/tests/integration_tests_rust.rs b/tests/integration_tests_rust.rs index d2c057a16..969e740ec 100644 --- a/tests/integration_tests_rust.rs +++ b/tests/integration_tests_rust.rs @@ -24,7 +24,7 @@ use common::{ generate_listening_addresses, open_channel, open_channel_push_amt, open_channel_with_all, premine_and_distribute_funds, premine_blocks, prepare_rbf, random_chain_source, random_config, setup_bitcoind_and_electrsd, setup_builder, setup_node, setup_two_nodes, splice_in_with_all, - wait_for_tx, TestChainSource, TestStoreType, TestSyncStore, + wait_for_tx, TestChainSource, TestConfig, TestStoreType, TestSyncStore, }; use electrsd::corepc_node::Node as BitcoinD; use electrsd::ElectrsD; @@ -2541,15 +2541,19 @@ async fn build_0_6_2_node( } async fn build_0_7_0_node( - bitcoind: &BitcoinD, electrsd: &ElectrsD, storage_path: String, esplora_url: String, - seed_bytes: [u8; 64], + bitcoind: &BitcoinD, electrsd: &ElectrsD, esplora_url: String, seed_bytes: [u8; 64], + config: &TestConfig, ) -> (u64, bitcoin::secp256k1::PublicKey) { let mut builder_old = ldk_node_070::Builder::new(); builder_old.set_network(bitcoin::Network::Regtest); - builder_old.set_storage_dir_path(storage_path); + builder_old.set_storage_dir_path(config.node_config.storage_dir_path.clone()); builder_old.set_entropy_seed_bytes(seed_bytes); builder_old.set_chain_source_esplora(esplora_url, None); - let node_old = builder_old.build().unwrap(); + let node_old = match config.store_type { + TestStoreType::FilesystemStore => builder_old.build_with_fs_store().unwrap(), + TestStoreType::Sqlite => builder_old.build().unwrap(), + TestStoreType::TestSyncStore => panic!("TestSyncStore not supported in v0.7.0 builder"), + }; node_old.start().unwrap(); let addr_old = node_old.onchain_payment().new_address().unwrap(); @@ -2590,14 +2594,10 @@ async fn do_persistence_backwards_compatibility(version: OldLdkVersion) { .await }, OldLdkVersion::V0_7_0 => { - build_0_7_0_node( - &bitcoind, - &electrsd, - storage_path.clone(), - esplora_url.clone(), - seed_bytes, - ) - .await + let mut config = TestConfig::default(); + config.store_type = TestStoreType::Sqlite; + config.node_config.storage_dir_path = storage_path.clone(); + build_0_7_0_node(&bitcoind, &electrsd, esplora_url.clone(), seed_bytes, &config).await }, }; From 75ca8d70fb3910f5fab6fedbe07a442386e6a5b3 Mon Sep 17 00:00:00 2001 From: benthecarman Date: Mon, 13 Apr 2026 12:36:29 -0500 Subject: [PATCH 3/3] Safely migrate to FileSystemStoreV2 Before moving to PaginatedKVStore everywhere we need to use FileSystemStoreV2 instead of FileSystemStoreV1. This will safely migrate over to it on first start up. Also adds a test to make sure we handle it properly. --- src/builder.rs | 20 +++++++----- src/io/utils.rs | 54 ++++++++++++++++++++++++++++----- tests/integration_tests_rust.rs | 43 ++++++++++++++++++++++++++ 3 files changed, 102 insertions(+), 15 deletions(-) diff --git a/src/builder.rs b/src/builder.rs index 6e5076480..be709d91b 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -43,7 +43,6 @@ use lightning::util::persist::{ use lightning::util::ser::ReadableArgs; use lightning::util::sweep::OutputSweeper; use lightning_dns_resolver::OMDomainResolver; -use lightning_persister::fs_store::v1::FilesystemStore; use vss_client::headers::VssHeaderProvider; use crate::chain::ChainSource; @@ -59,9 +58,9 @@ use crate::fee_estimator::OnchainFeeEstimator; use crate::gossip::GossipSource; use crate::io::sqlite_store::SqliteStore; use crate::io::utils::{ - read_event_queue, read_external_pathfinding_scores_from_cache, read_network_graph, - read_node_metrics, read_output_sweeper, read_payments, read_peer_info, read_pending_payments, - read_scorer, + open_or_migrate_fs_store, read_event_queue, read_external_pathfinding_scores_from_cache, + read_network_graph, read_node_metrics, read_output_sweeper, read_payments, read_peer_info, + read_pending_payments, read_scorer, }; use crate::io::vss_store::VssStoreBuilder; use crate::io::{ @@ -641,15 +640,22 @@ impl NodeBuilder { self.build_with_store(node_entropy, kv_store) } - /// Builds a [`Node`] instance with a [`FilesystemStore`] backend and according to the options + /// Builds a [`Node`] instance with a [`FilesystemStoreV2`] backend and according to the options /// previously configured. + /// + /// If the storage directory contains data from a v1 filesystem store, it will be + /// automatically migrated to the v2 format. + /// + /// [`FilesystemStoreV2`]: lightning_persister::fs_store::v2::FilesystemStoreV2 pub fn build_with_fs_store(&self, node_entropy: NodeEntropy) -> Result { let mut storage_dir_path: PathBuf = self.config.storage_dir_path.clone().into(); storage_dir_path.push("fs_store"); fs::create_dir_all(storage_dir_path.clone()) .map_err(|_| BuildError::StoragePathAccessFailed)?; - let kv_store = FilesystemStore::new(storage_dir_path); + + let kv_store = open_or_migrate_fs_store(storage_dir_path)?; + self.build_with_store(node_entropy, kv_store) } @@ -1103,7 +1109,7 @@ impl ArcedNodeBuilder { self.inner.read().expect("lock").build(*node_entropy).map(Arc::new) } - /// Builds a [`Node`] instance with a [`FilesystemStore`] backend and according to the options + /// Builds a [`Node`] instance with a [`FilesystemStoreV2`] backend and according to the options /// previously configured. pub fn build_with_fs_store( &self, node_entropy: Arc, diff --git a/src/io/utils.rs b/src/io/utils.rs index ff78c7e91..9bc257359 100644 --- a/src/io/utils.rs +++ b/src/io/utils.rs @@ -10,7 +10,7 @@ use std::io::Write; use std::ops::Deref; #[cfg(unix)] use std::os::unix::fs::OpenOptionsExt; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::sync::{Arc, RwLock}; use bdk_chain::indexer::keychain_txout::ChangeSet as BdkIndexerChangeSet; @@ -26,14 +26,16 @@ use lightning::routing::scoring::{ ChannelLiquidities, ProbabilisticScorer, ProbabilisticScoringDecayParameters, }; use lightning::util::persist::{ - KVStore, KVStoreSync, KVSTORE_NAMESPACE_KEY_ALPHABET, KVSTORE_NAMESPACE_KEY_MAX_LEN, - NETWORK_GRAPH_PERSISTENCE_KEY, NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, - NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, OUTPUT_SWEEPER_PERSISTENCE_KEY, - OUTPUT_SWEEPER_PERSISTENCE_PRIMARY_NAMESPACE, OUTPUT_SWEEPER_PERSISTENCE_SECONDARY_NAMESPACE, - SCORER_PERSISTENCE_KEY, SCORER_PERSISTENCE_PRIMARY_NAMESPACE, - SCORER_PERSISTENCE_SECONDARY_NAMESPACE, + migrate_kv_store_data, KVStore, KVStoreSync, KVSTORE_NAMESPACE_KEY_ALPHABET, + KVSTORE_NAMESPACE_KEY_MAX_LEN, NETWORK_GRAPH_PERSISTENCE_KEY, + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + OUTPUT_SWEEPER_PERSISTENCE_KEY, OUTPUT_SWEEPER_PERSISTENCE_PRIMARY_NAMESPACE, + OUTPUT_SWEEPER_PERSISTENCE_SECONDARY_NAMESPACE, SCORER_PERSISTENCE_KEY, + SCORER_PERSISTENCE_PRIMARY_NAMESPACE, SCORER_PERSISTENCE_SECONDARY_NAMESPACE, }; use lightning::util::ser::{Readable, ReadableArgs, Writeable}; +use lightning_persister::fs_store::v1::FilesystemStore; +use lightning_persister::fs_store::v2::{FilesystemStoreV2, FilesystemStoreV2Error}; use lightning_types::string::PrintableString; use super::*; @@ -48,7 +50,7 @@ use crate::payment::PendingPaymentDetails; use crate::peer_store::PeerStore; use crate::types::{Broadcaster, DynStore, KeysManager, Sweeper}; use crate::wallet::ser::{ChangeSetDeserWrapper, ChangeSetSerWrapper}; -use crate::{Error, EventQueue, NodeMetrics, PaymentDetails}; +use crate::{BuildError, Error, EventQueue, NodeMetrics, PaymentDetails}; pub const EXTERNAL_PATHFINDING_SCORES_CACHE_KEY: &str = "external_pathfinding_scores_cache"; @@ -709,6 +711,42 @@ where Ok(res) } +/// Opens a [`FilesystemStoreV2`], automatically migrating from v1 format if necessary. +/// +/// If the directory contains v1 data (files at the top level), the data is migrated to v2 format +/// in a temporary directory, the original is renamed to `fs_store_v1_backup`, and the migrated +/// directory is moved into place. +pub(crate) fn open_or_migrate_fs_store( + storage_dir_path: PathBuf, +) -> Result { + match FilesystemStoreV2::new(storage_dir_path.clone()) { + Ok(store) => Ok(store), + Err(FilesystemStoreV2Error::V1DataDetected(_)) => { + // The directory contains v1 data, migrate to v2. + let mut v1_store = FilesystemStore::new(storage_dir_path.clone()); + + let mut v2_dir = storage_dir_path.clone(); + v2_dir.set_file_name("fs_store_v2_migrating"); + fs::create_dir_all(v2_dir.clone()).map_err(|_| BuildError::StoragePathAccessFailed)?; + let mut v2_store = FilesystemStoreV2::new(v2_dir.clone()) + .map_err(|_| BuildError::KVStoreSetupFailed)?; + + migrate_kv_store_data(&mut v1_store, &mut v2_store) + .map_err(|_| BuildError::KVStoreSetupFailed)?; + + // Swap directories: rename v1 out of the way, move v2 into place. + let mut backup_dir = storage_dir_path.clone(); + backup_dir.set_file_name("fs_store_v1_backup"); + fs::rename(&storage_dir_path, &backup_dir) + .map_err(|_| BuildError::KVStoreSetupFailed)?; + fs::rename(&v2_dir, &storage_dir_path).map_err(|_| BuildError::KVStoreSetupFailed)?; + + FilesystemStoreV2::new(storage_dir_path).map_err(|_| BuildError::KVStoreSetupFailed) + }, + Err(_) => Err(BuildError::KVStoreSetupFailed), + } +} + #[cfg(test)] mod tests { use super::read_or_generate_seed_file; diff --git a/tests/integration_tests_rust.rs b/tests/integration_tests_rust.rs index 969e740ec..490f5516e 100644 --- a/tests/integration_tests_rust.rs +++ b/tests/integration_tests_rust.rs @@ -2634,6 +2634,49 @@ async fn persistence_backwards_compatibility() { do_persistence_backwards_compatibility(OldLdkVersion::V0_7_0).await; } +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn fs_store_persistence_backwards_compatibility() { + let (bitcoind, electrsd) = common::setup_bitcoind_and_electrsd(); + let esplora_url = format!("http://{}", electrsd.esplora_url.as_ref().unwrap()); + + let storage_path = common::random_storage_path().to_str().unwrap().to_owned(); + let seed_bytes = [42u8; 64]; + + // Build a node using v0.7.0's build_with_fs_store (FilesystemStore v1). + let mut config = TestConfig::default(); + config.node_config.storage_dir_path = storage_path.clone(); + config.store_type = TestStoreType::FilesystemStore; + let (old_balance, old_node_id) = + build_0_7_0_node(&bitcoind, &electrsd, esplora_url.clone(), seed_bytes, &config).await; + + // Now reopen with current code's build_with_fs_store, which should + // auto-migrate from FilesystemStore v1 to FilesystemStoreV2. + #[cfg(feature = "uniffi")] + let builder_new = Builder::new(); + #[cfg(not(feature = "uniffi"))] + let mut builder_new = Builder::new(); + builder_new.set_network(bitcoin::Network::Regtest); + builder_new.set_storage_dir_path(storage_path); + builder_new.set_chain_source_esplora(esplora_url, None); + + #[cfg(feature = "uniffi")] + let node_entropy = NodeEntropy::from_seed_bytes(seed_bytes.to_vec()).unwrap(); + #[cfg(not(feature = "uniffi"))] + let node_entropy = NodeEntropy::from_seed_bytes(seed_bytes); + let node_new = builder_new.build_with_fs_store(node_entropy.into()).unwrap(); + + node_new.start().unwrap(); + node_new.sync_wallets().unwrap(); + + let new_balance = node_new.list_balances().spendable_onchain_balance_sats; + let new_node_id = node_new.node_id(); + + assert_eq!(old_node_id, new_node_id); + assert_eq!(old_balance, new_balance); + + node_new.stop().unwrap(); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 1)] async fn onchain_fee_bump_rbf() { let (bitcoind, electrsd) = setup_bitcoind_and_electrsd();