From 2433495289d17616ed78d256bb886eba0017ba61 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Tue, 15 Sep 2026 11:51:22 +0000 Subject: [PATCH 01/30] fix(wallet): make incompatible-data guidance and upgrade backups safe - WalletDataIncompatible no longer tells users to remove local wallet data; it asks them to write down recovery phrases and imported keys, move only the .sqlite files aside and keep the secrets folder, whose imported and masternode keys no recovery phrase can restore. - The compatibility bridge backs up only after the staged copy validates, right before the rebuild, and keeps at most one backup per database. - Deterministic upgrade failures are terminal storage-open errors, so they surface instead of being silently retried; busy, disk-full and I/O failures stay retryable. - Removing a wallet or identity, or clearing a network's data, deletes the upgrade backups of det-app.sqlite and the network wallet database. - Upgrade and funding-persist errors name their actual cause (another instance, full disk, or neither) without disk-space or "account" jargon where it does not apply; the two identity-roster variants share one message. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 13 ++ .../2026-09-10-platform-pin/upgrade-notes.md | 5 +- src/backend_task/error.rs | 44 ++++-- src/backend_task/identity/remove_identity.rs | 26 ++++ src/backend_task/mod.rs | 34 ++++- src/context/identity_db.rs | 7 + src/context/wallet_lifecycle/removal.rs | 25 ++++ src/context/wallet_lifecycle/spv.rs | 5 + src/context/wallet_lifecycle/tests.rs | 56 ++++++- src/wallet_backend/identity_ops.rs | 7 +- .../platform_compatibility/engine.rs | 130 +++++++++++++--- .../platform_compatibility/engine/tests.rs | 139 +++++++++++++++++- .../platform_compatibility/mod.rs | 4 + 13 files changed, 452 insertions(+), 43 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a9e232bd..f84376e5b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,19 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). Allowing 3.x needs an upstream change in `dashpay/rust-dashcore` first. A TODO in `Cargo.toml` marks the re-check. +- **Safer guidance for wallet data this version cannot open**: the message no + longer tells you to remove your local wallet data, which could have deleted + the `secrets` folder along with keys no recovery phrase can restore. It now + asks you to write down your recovery phrases and imported keys in the version + you used before, then set aside only the `.sqlite` files and keep the + `secrets` folder. + +- **Upgrade backups no longer pile up or outlive deleted data**: a failed + database upgrade no longer leaves a new backup on every attempt. Each + database keeps at most one upgrade backup, and it is deleted when you remove + a wallet or identity or clear a network's data. An upgrade that can never + succeed now shows its message instead of being retried silently. + ### Added - **Keys saved on this device but not on the identity's key lists are now diff --git a/docs/ai-design/2026-09-10-platform-pin/upgrade-notes.md b/docs/ai-design/2026-09-10-platform-pin/upgrade-notes.md index 106152079..20f68b8df 100644 --- a/docs/ai-design/2026-09-10-platform-pin/upgrade-notes.md +++ b/docs/ai-design/2026-09-10-platform-pin/upgrade-notes.md @@ -60,7 +60,10 @@ typed identity rows remain in the backup, while their opaque local metadata is preserved. An ambiguous or malformed saved identity roster fails closed. The encrypted seed vault is not migrated or rewritten. -Keep the retained backups. Downgrading does not automatically reverse the +The bridge takes its backup only after the converted copy validates, just before +the rebuild, and keeps at most one backup per database. Deleting a wallet or an +identity, or clearing a network's data, also deletes these backups, because they +copy wallet and identity history. Downgrading does not automatically reverse the database conversion; recovery requires the corresponding backup and the prior application version. Validation uses synthetic upstream fixtures, not a user's real profile. diff --git a/src/backend_task/error.rs b/src/backend_task/error.rs index b5e17062d..63f93e3ce 100644 --- a/src/backend_task/error.rs +++ b/src/backend_task/error.rs @@ -188,15 +188,27 @@ pub enum TaskError { /// An identity-funding account was derived but could not be saved. Saving /// it is what lets a restart find the account again, so the operation is /// stopped rather than left able to strand a funding lock the app could no - /// longer spend. The technical cause lives in `Debug` and the logs. + /// longer spend. Covers non-retryable store failures (constraint, fatal); + /// retryable ones use [`Self::IdentityFundingAccountPersistBusy`]. #[error( - "Your wallet could not save the account this payment needs. Check that your disk is not full, then try again." + "Your wallet could not save the information it needs for this payment. Restart the application and try again." )] IdentityFundingAccountPersistFailed { #[source] source: Box, }, + /// Like [`Self::IdentityFundingAccountPersistFailed`], but the store kept + /// reporting a retryable condition until the retry budget ran out. Upstream + /// classifies a busy database and a full disk alike, so the copy names both. + #[error( + "Your wallet could not save the information it needs for this payment because its data is busy or the disk is full. Close any other copy of Dash Evo Tool, make sure there is free disk space, and try again." + )] + IdentityFundingAccountPersistBusy { + #[source] + source: Box, + }, + /// Single-key wallets are not supported in this version. Their data is /// preserved; HD (recovery-phrase) wallets remain fully functional. #[error( @@ -473,11 +485,11 @@ pub enum TaskError { /// /// The migration diagnostic is preserved through the `#[source]` chain for /// logs and the `Debug` view; it is kept out of the user-facing `Display` - /// copy. On the active development branch the storage layout changed in an - /// incompatible way, so the practical action is to remove the local wallet - /// data and let the app recreate it (see `docs/kv-keys.md`). + /// copy. The copy names only the rebuildable `.sqlite` stores and tells the + /// user to keep the `secrets` folder: the vault holds imported and + /// masternode keys that no recovery phrase can re-derive. #[error( - "Your wallet data is not compatible with this version of the app and cannot be opened. Remove the local wallet data so the app can create it fresh, then restart." + "Your wallet data was saved by an app version that this version cannot open, so it was left unchanged. Your recovery phrases and keys are still stored on this device. First open the app version you used before and write down the recovery phrase of every wallet and every key you imported. Then move the files ending in .sqlite out of the app data folder, leave the secrets folder where it is, and start this app again." )] WalletDataIncompatible { #[source] @@ -2829,8 +2841,9 @@ impl TaskError { /// - A divergent migration history (e.g. a database written under an /// earlier, incompatible storage layout that this build's migrations /// cannot reconcile) is surfaced as [`Self::WalletDataIncompatible`] so - /// the banner tells the user to remove the local wallet data — freeing - /// disk space or restarting never resolves a structural mismatch. + /// the banner tells the user to back up keys and set the databases aside + /// while keeping the vault — freeing disk space or restarting never + /// resolves a structural mismatch. /// - Every other storage failure keeps the generic disk/IO copy via /// [`Self::WalletStorage`]. /// @@ -5799,8 +5812,8 @@ mod tests { /// A divergent migration history (database written under an /// incompatible storage layout) maps to the dedicated - /// `WalletDataIncompatible` variant. Its `Display` tells the user to - /// remove the local wallet data, NOT the misleading "free disk space" copy. + /// `WalletDataIncompatible` variant. Its `Display` must never lead the user + /// to destroy the secrets vault, and must not blame disk space. #[test] fn migration_error_maps_to_wallet_data_incompatible() { let upstream = @@ -5813,8 +5826,15 @@ mod tests { let msg = err.to_string(); assert!( - msg.contains("not compatible") && msg.contains("Remove"), - "Expected incompatibility guidance, got: {msg}" + msg.contains("write down the recovery phrase") + && msg.contains(".sqlite") + && msg.contains("leave the secrets folder where it is"), + "Expected scoped, vault-preserving guidance, got: {msg}" + ); + let lower = msg.to_lowercase(); + assert!( + !lower.contains("remove") && !lower.contains("delete"), + "Incompatible-data guidance must not tell the user to delete anything, got: {msg}" ); assert!( !msg.contains("disk space"), diff --git a/src/backend_task/identity/remove_identity.rs b/src/backend_task/identity/remove_identity.rs index 80ed6d423..70c2a5870 100644 --- a/src/backend_task/identity/remove_identity.rs +++ b/src/backend_task/identity/remove_identity.rs @@ -149,6 +149,32 @@ mod tests { .any(|key_id| view.get(&MAIN, *key_id).unwrap().is_some()) } + /// Upgrade backups copy the network database with every identity in it, so a + /// removed identity must not survive in them. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn removing_an_identity_deletes_upgrade_backups() { + let staged = stage_identity_with_vaulted_keys(HIGH, MEDIUM).await; + let database = crate::wallet_backend::wallet_database_path( + staged.ctx.data_dir(), + staged.ctx.network(), + ); + let backup = database.with_file_name(format!( + "{}.platform-67d4ef3-backup-fixture.sqlite", + database.file_name().unwrap().to_string_lossy() + )); + std::fs::write(&backup, b"old identity rows").unwrap(); + + staged + .ctx + .remove_identity(staged.id) + .expect("the identity must be removed"); + + assert!( + !backup.exists(), + "the upgrade backup must be deleted together with the identity" + ); + } + /// A removal that fails before the identity is delisted did not happen: /// the identity is still on every screen, its keys are still on the /// device, and the button that started this still works. Reporting it as a diff --git a/src/backend_task/mod.rs b/src/backend_task/mod.rs index d9f372e47..bc64a582b 100644 --- a/src/backend_task/mod.rs +++ b/src/backend_task/mod.rs @@ -230,14 +230,16 @@ fn identity_load_ticket(task: &BackendTask) -> Option<(Identifier, IdentityLoadT } /// Whether a wallet-backend build error is terminal (storage written by a -/// newer/incompatible app build). These must surface their actionable -/// message instead of being logged-and-discarded as a transient deferral -/// (F50); every other init error is retried by the cold-boot bridge. +/// newer/incompatible app build, or a compatibility upgrade that fails the +/// same way on every attempt). These must surface their actionable message +/// instead of being logged-and-discarded as a transient deferral (F50); every +/// other init error is retried by the cold-boot bridge. pub(crate) fn is_terminal_storage_open_error(error: &TaskError) -> bool { - matches!( - error, - TaskError::WalletDataTooNew { .. } | TaskError::WalletDataIncompatible { .. } - ) + match error { + TaskError::WalletDataTooNew { .. } | TaskError::WalletDataIncompatible { .. } => true, + TaskError::PlatformDatabaseUpgrade { source } => !source.is_retryable(), + _ => false, + } } /// Information about fees for a platform state transition. @@ -2331,6 +2333,24 @@ mod tests { )), } )); + use crate::wallet_backend::platform_compatibility::UpgradeError; + for source in [ + UpgradeError::Unrecognized, + UpgradeError::IdentityRoster { source: None }, + UpgradeError::Verification, + UpgradeError::TypedValidation(Box::new(std::io::Error::other("fixture"))), + ] { + assert!( + is_terminal_storage_open_error(&TaskError::PlatformDatabaseUpgrade { source }), + "a deterministic upgrade failure must surface instead of re-running" + ); + } + // A user-clearable upgrade failure keeps offering a retry. + assert!(!is_terminal_storage_open_error( + &TaskError::PlatformDatabaseUpgrade { + source: UpgradeError::from(std::io::Error::from(std::io::ErrorKind::StorageFull)), + } + )); // A transient pre-wire state must NOT be treated as terminal. assert!(!is_terminal_storage_open_error( &TaskError::WalletBackendNotYetWired diff --git a/src/context/identity_db.rs b/src/context/identity_db.rs index 3375c75e6..30d133894 100644 --- a/src/context/identity_db.rs +++ b/src/context/identity_db.rs @@ -1737,6 +1737,13 @@ impl AppContext { "Deleted identity still registered with the wallet store" ); } + if let Err(error) = self.remove_upgrade_backups() { + tracing::warn!( + identity_id = %identifier, + ?error, + "Upgrade backups holding a deleted identity could not be removed" + ); + } Ok(sidecar_cleanup) } diff --git a/src/context/wallet_lifecycle/removal.rs b/src/context/wallet_lifecycle/removal.rs index 802ce9ec7..3babd26e5 100644 --- a/src/context/wallet_lifecycle/removal.rs +++ b/src/context/wallet_lifecycle/removal.rs @@ -81,10 +81,35 @@ impl AppContext { ); } } + + if let Err(error) = self.remove_upgrade_backups() { + tracing::warn!(?error, "Failed to delete wallet upgrade backups on removal"); + show_wallet_data_removal_warning(self.egui_ctx(), error); + } } Ok(()) } + + /// Delete the retained compatibility-upgrade backups of the app and this network's + /// wallet databases, which copy wallet and identity history a deletion must not leave behind. + /// + /// Call only with the wallet backend wired: this process then holds both databases open + /// and already upgraded, so no other instance can be writing a backup of them. + pub(crate) fn remove_upgrade_backups(&self) -> Result<(), TaskError> { + let mut first_error = None; + for database in [ + self.data_dir().join("det-app.sqlite"), + crate::wallet_backend::wallet_database_path(self.data_dir(), self.network), + ] { + if let Err(source) = + crate::wallet_backend::platform_compatibility::remove_backups(&database) + { + first_error.get_or_insert(TaskError::FileSystem { source }); + } + } + first_error.map_or(Ok(()), Err) + } } #[cfg(test)] diff --git a/src/context/wallet_lifecycle/spv.rs b/src/context/wallet_lifecycle/spv.rs index eac04cbae..1c1a0a9d3 100644 --- a/src/context/wallet_lifecycle/spv.rs +++ b/src/context/wallet_lifecycle/spv.rs @@ -113,6 +113,11 @@ impl AppContext { failures.push(error); } + if let Err(error) = self.remove_upgrade_backups() { + tracing::warn!(?error, "Upgrade backup removal failed during clear"); + failures.push(error); + } + if let Ok(mut wallets) = self.wallets.write() { wallets.clear(); } diff --git a/src/context/wallet_lifecycle/tests.rs b/src/context/wallet_lifecycle/tests.rs index fb4e3e73a..f380e84b1 100644 --- a/src/context/wallet_lifecycle/tests.rs +++ b/src/context/wallet_lifecycle/tests.rs @@ -1718,6 +1718,53 @@ async fn remove_wallet_wipes_seed_envelope() { backend.shutdown().await; } +/// Upgrade backups copy whole wallet databases (xpubs, history, identities), so +/// removing a wallet must not leave them behind. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn remove_wallet_deletes_upgrade_backups() { + let (ctx, sender, _tmp) = offline_testnet_context(); + ctx.ensure_wallet_backend(sender) + .await + .expect("ensure_wallet_backend should succeed offline"); + + let seed = [0xA7u8; 64]; + let wallet = crate::model::wallet::Wallet::new_from_seed(seed, Network::Testnet, None, None) + .expect("build wallet"); + let seed_hash = wallet.seed_hash(); + ctx.register_wallet(wallet, &seed, WalletOrigin::Fresh) + .expect("register wallet"); + let backend = ctx.wallet_backend().expect("backend wired"); + + let databases = [ + ctx.data_dir().join("det-app.sqlite"), + crate::wallet_backend::wallet_database_path(ctx.data_dir(), Network::Testnet), + ]; + let backups = databases.each_ref().map(|database| { + let name = database.file_name().expect("database file name"); + let backup = database.with_file_name(format!( + "{}.platform-67d4ef3-backup-fixture.sqlite", + name.to_string_lossy() + )); + std::fs::write(&backup, b"old wallet history").expect("write backup fixture"); + backup + }); + + ctx.remove_wallet(&seed_hash).expect("remove wallet"); + + for backup in &backups { + assert!( + !backup.exists(), + "{} must be deleted together with the wallet", + backup.display() + ); + } + for database in &databases { + assert!(database.exists(), "{} must be kept", database.display()); + } + + backend.shutdown().await; +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn remove_wallet_warns_when_local_secret_wipe_fails() { let (ctx, sender, _tmp) = offline_testnet_context(); @@ -6091,8 +6138,8 @@ async fn an_exhausted_transient_persist_budget_keeps_memory_and_the_next_attempt .await .expect_err("an exhausted retry budget must surface as a typed error"); assert!( - matches!(error, TaskError::IdentityFundingAccountPersistFailed { .. }), - "the persist failure must keep its dedicated variant, got {error:?}", + matches!(error, TaskError::IdentityFundingAccountPersistBusy { .. }), + "a still-retryable failure must use the busy variant, got {error:?}", ); let view = backend @@ -6364,6 +6411,11 @@ async fn a_constraint_registration_persist_failure_rolls_back_the_in_memory_acco matches!(error, TaskError::IdentityFundingAccountPersistFailed { .. }), "the persist failure must keep its dedicated variant, got {error:?}", ); + let text = error.to_string(); + assert!( + !text.contains("disk") && !text.contains("account"), + "a constraint violation has nothing to do with disk space: {text}", + ); assert!( backend .identity_funding_account(&seed_hash, fault_topup_account_type()) diff --git a/src/wallet_backend/identity_ops.rs b/src/wallet_backend/identity_ops.rs index 3f0e06132..df2ae2903 100644 --- a/src/wallet_backend/identity_ops.rs +++ b/src/wallet_backend/identity_ops.rs @@ -189,8 +189,11 @@ enum RegistrationPersistFailure { impl RegistrationPersistFailure { fn into_task_error(self) -> TaskError { let (Self::Staged(source) | Self::Discarded(source)) = self; - TaskError::IdentityFundingAccountPersistFailed { - source: Box::new(source), + let source = Box::new(source); + if source.is_transient() { + TaskError::IdentityFundingAccountPersistBusy { source } + } else { + TaskError::IdentityFundingAccountPersistFailed { source } } } } diff --git a/src/wallet_backend/platform_compatibility/engine.rs b/src/wallet_backend/platform_compatibility/engine.rs index 6c8b3f9e1..a2418f7e9 100644 --- a/src/wallet_backend/platform_compatibility/engine.rs +++ b/src/wallet_backend/platform_compatibility/engine.rs @@ -11,26 +11,35 @@ const IDENTITY_INDEX_KEY: &str = "det:identity_index:v1"; /// A compatibility upgrade stopped before committing changes to the original database. #[derive(Debug, thiserror::Error)] pub enum UpgradeError { + /// Another process holds the database (`SQLITE_BUSY` / `SQLITE_LOCKED`). #[error( - "Could not upgrade wallet data. Check available disk space and restart the application." + "Your wallet data is open in another Dash Evo Tool window or command-line session. Close it and try again." )] - Sqlite(#[from] rusqlite::Error), + InUse(#[source] rusqlite::Error), #[error( - "Could not back up wallet data. Check available disk space and restart the application." + "Could not upgrade wallet data because the disk is full. Free up disk space and try again." )] - Io(#[from] std::io::Error), + StorageFull(#[source] Box), #[error( - "Wallet data does not match a supported upgrade. Keep your data folder and reopen the previous application version." + "Could not upgrade wallet data, and your data was not changed. Restart the application to try again, or keep your data folder and reopen the previous application version." )] - Unrecognized, + Sqlite(#[source] rusqlite::Error), #[error( - "The saved identity list could not be read. Keep your data folder and reopen the previous application version." + "Could not back up wallet data. Check that the app data folder can be written to and try again." + )] + Io(#[source] std::io::Error), + #[error( + "Wallet data does not match a supported upgrade. Keep your data folder and reopen the previous application version." )] - IdentityRoster, + Unrecognized, + /// The saved roster is missing, ambiguous, or does not decode. #[error( "The saved identity list could not be read. Keep your data folder and reopen the previous application version." )] - IdentityRosterDecode(#[from] bincode::error::DecodeError), + IdentityRoster { + #[source] + source: Option, + }, #[error( "Wallet data verification failed. Keep your data folder and reopen the previous application version." )] @@ -41,6 +50,44 @@ pub enum UpgradeError { TypedValidation(#[source] Box), } +impl UpgradeError { + /// Whether the user can clear the cause (close another instance, free space) and retry; + /// every other failure repeats identically on the same data. + pub fn is_retryable(&self) -> bool { + matches!(self, Self::InUse(_) | Self::StorageFull(_) | Self::Io(_)) + } +} + +impl From for UpgradeError { + fn from(error: rusqlite::Error) -> Self { + match error.sqlite_error_code() { + Some(rusqlite::ErrorCode::DatabaseBusy | rusqlite::ErrorCode::DatabaseLocked) => { + Self::InUse(error) + } + Some(rusqlite::ErrorCode::DiskFull) => Self::StorageFull(Box::new(error)), + _ => Self::Sqlite(error), + } + } +} + +impl From for UpgradeError { + fn from(error: std::io::Error) -> Self { + if error.kind() == std::io::ErrorKind::StorageFull { + Self::StorageFull(Box::new(error)) + } else { + Self::Io(error) + } + } +} + +impl From for UpgradeError { + fn from(source: bincode::error::DecodeError) -> Self { + Self::IdentityRoster { + source: Some(source), + } + } +} + #[derive(Debug, PartialEq, Eq)] struct Object { kind: String, @@ -111,19 +158,19 @@ fn active_identities(conn: &Connection) -> Result>, UpgradeErro .optional()?; let Some(value) = value else { if conn.prepare("SELECT 1 FROM identities i JOIN meta_identity m ON i.identity_id = m.identity_id WHERE i.tombstoned = 1 AND m.key = 'det:identity:v1'")?.exists([])? { - return Err(UpgradeError::IdentityRoster); + return Err(UpgradeError::IdentityRoster { source: None }); } return Ok(BTreeSet::new()); }; let Some((&1, body)) = value.split_first() else { - return Err(UpgradeError::IdentityRoster); + return Err(UpgradeError::IdentityRoster { source: None }); }; let (ids, consumed): (Vec<[u8; 32]>, usize) = bincode::serde::decode_from_slice( body, bincode::config::standard().with_limit::<16777216>(), )?; if consumed != body.len() { - return Err(UpgradeError::IdentityRoster); + return Err(UpgradeError::IdentityRoster { source: None }); } Ok(ids.into_iter().map(Vec::from).collect()) } @@ -163,10 +210,51 @@ fn selected_rows(table: &str) -> String { .unwrap_or_default() } +fn backup_prefix(path: &Path) -> Option { + Some(format!( + "{}.platform-67d4ef3-backup-", + path.file_name()?.to_string_lossy() + )) +} + +/// Retained upgrade backups of the database at `path`. +fn backups(path: &Path) -> std::io::Result> { + let (Some(parent), Some(prefix)) = (path.parent(), backup_prefix(path)) else { + return Ok(Vec::new()); + }; + let entries = match std::fs::read_dir(parent) { + Ok(entries) => entries, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(error) => return Err(error), + }; + let mut found = Vec::new(); + for entry in entries { + let entry = entry?; + let name = entry.file_name(); + let name = name.to_string_lossy(); + if name.starts_with(&prefix) && name.ends_with(".sqlite") { + found.push(entry.path()); + } + } + Ok(found) +} + +/// Delete every retained upgrade backup of the database at `path`. +/// +/// Backups never contain vault secrets. Attempts every file and returns the first failure. +pub(crate) fn remove_backups(path: &Path) -> std::io::Result<()> { + let mut first_error = None; + for backup in backups(path)? { + if let Err(error) = std::fs::remove_file(&backup) { + first_error.get_or_insert(error); + } + } + first_error.map_or(Ok(()), Err) +} + fn backup(path: &Path) -> Result { let parent = path.parent().ok_or(UpgradeError::Unrecognized)?; - let filename = path.file_name().ok_or(UpgradeError::Unrecognized)?; - let prefix = format!("{}.platform-67d4ef3-backup-", filename.to_string_lossy()); + let prefix = backup_prefix(path).ok_or(UpgradeError::Unrecognized)?; let file = tempfile::Builder::new() .prefix(&prefix) .suffix(".sqlite") @@ -185,10 +273,16 @@ fn backup(path: &Path) -> Result { verify(&dest)?; drop(dest); file.as_file().sync_all()?; - let (_, path) = file.keep().map_err(|e| e.error)?; + let (_, kept) = file.keep().map_err(|e| e.error)?; #[cfg(unix)] std::fs::File::open(parent)?.sync_all()?; - Ok(path) + // Only an unconverted original reaches this point, so older backups are superseded snapshots. + for old in backups(path)?.into_iter().filter(|old| *old != kept) { + if let Err(error) = std::fs::remove_file(&old) { + tracing::warn!(backup = %old.display(), %error, "Could not remove a superseded wallet upgrade backup"); + } + } + Ok(kept) } fn copy_rows( @@ -316,7 +410,6 @@ fn upgrade_with_hook( } verify(&source)?; let retired = retired_identities(&source)?; - let backup = backup(path)?; let mut target = Connection::open_with_flags( target_path, OpenFlags::SQLITE_OPEN_READ_WRITE | OpenFlags::SQLITE_OPEN_NOFOLLOW, @@ -396,6 +489,9 @@ fn upgrade_with_hook( target_tx.commit()?; drop(target); validate(target_path)?; + // The source transaction has written only temp tables, so a separate reader still + // sees the committed original; earlier failures roll back and need no backup. + let backup = backup(path)?; let target = Connection::open_with_flags( target_path, OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NOFOLLOW, diff --git a/src/wallet_backend/platform_compatibility/engine/tests.rs b/src/wallet_backend/platform_compatibility/engine/tests.rs index fb99701a6..074b4cf84 100644 --- a/src/wallet_backend/platform_compatibility/engine/tests.rs +++ b/src/wallet_backend/platform_compatibility/engine/tests.rs @@ -216,7 +216,142 @@ fn platform_compatibility_typed_validation_failure_never_rewrites_original() { .unwrap_err(); assert!(matches!(error, UpgradeError::TypedValidation(_))); assert_eq!(snapshot(&path), before); - assert_eq!(snapshot(&backup_files(dir.path())[0]), before); + assert!( + backup_files(dir.path()).is_empty(), + "A failure before the rebuild leaves the original untouched, so no backup may remain." + ); +} + +#[test] +fn platform_compatibility_repeated_failed_rebuilds_keep_one_backup() { + let (dir, path, _target) = fixture(); + let before = snapshot(&path); + for attempt in 0..3 { + let stage = dir.path().join(format!("stage-{attempt}.sqlite")); + Connection::open(&stage) + .unwrap() + .execute_batch(TARGET_SCHEMA) + .unwrap(); + let error = upgrade_with_hook( + &path, + &stage, + |_| Ok(()), + || Err(UpgradeError::Verification), + ) + .unwrap_err(); + assert!(matches!(error, UpgradeError::Verification)); + } + let backups = backup_files(dir.path()); + assert_eq!( + backups.len(), + 1, + "Retried upgrades must not accumulate backups." + ); + assert_eq!(snapshot(&backups[0]), before); +} + +#[test] +fn platform_compatibility_backup_removal_only_touches_the_named_database() { + let dir = tempfile::tempdir().unwrap(); + let names = [ + "det-app.sqlite.platform-67d4ef3-backup-a1.sqlite", + "det-testnet.sqlite.platform-67d4ef3-backup-b2.sqlite", + "det-mainnet.sqlite.platform-67d4ef3-backup-c3.sqlite", + "det-testnet.sqlite", + "det-testnet-shielded.sqlite", + ]; + for name in names { + std::fs::write(dir.path().join(name), b"x").unwrap(); + } + remove_backups(&dir.path().join("det-testnet.sqlite")).unwrap(); + let mut left = std::fs::read_dir(dir.path()) + .unwrap() + .map(|e| e.unwrap().file_name().to_string_lossy().into_owned()) + .collect::>(); + left.sort(); + assert_eq!( + left, + [ + "det-app.sqlite.platform-67d4ef3-backup-a1.sqlite", + "det-mainnet.sqlite.platform-67d4ef3-backup-c3.sqlite", + "det-testnet-shielded.sqlite", + "det-testnet.sqlite", + ] + ); + remove_backups(&dir.path().join("absent.sqlite")).unwrap(); +} + +fn sqlite_failure(code: std::ffi::c_int) -> rusqlite::Error { + rusqlite::Error::SqliteFailure(rusqlite::ffi::Error::new(code), None) +} + +#[test] +fn platform_compatibility_errors_name_their_actual_cause() { + let in_use = [ + UpgradeError::from(sqlite_failure(rusqlite::ffi::SQLITE_BUSY)), + UpgradeError::from(sqlite_failure(rusqlite::ffi::SQLITE_LOCKED)), + ]; + for error in &in_use { + assert!(matches!(error, UpgradeError::InUse(_)), "{error:?}"); + assert!(error.is_retryable()); + let text = error.to_string(); + assert!(text.contains("another Dash Evo Tool"), "{text}"); + assert!(!text.contains("disk"), "{text}"); + } + let full = [ + UpgradeError::from(sqlite_failure(rusqlite::ffi::SQLITE_FULL)), + UpgradeError::from(std::io::Error::from(std::io::ErrorKind::StorageFull)), + ]; + for error in &full { + assert!(matches!(error, UpgradeError::StorageFull(_)), "{error:?}"); + assert!(error.is_retryable()); + assert!(error.to_string().contains("disk")); + } + let io = UpgradeError::from(std::io::Error::from(std::io::ErrorKind::PermissionDenied)); + assert!(matches!(io, UpgradeError::Io(_)) && io.is_retryable()); + assert!(!io.to_string().contains("disk"), "{io}"); + + let generic = UpgradeError::from(sqlite_failure(rusqlite::ffi::SQLITE_CORRUPT)); + assert!(matches!(generic, UpgradeError::Sqlite(_))); + assert!(!generic.is_retryable()); + assert!(!generic.to_string().contains("disk"), "{generic}"); + for error in [ + std::error::Error::source(&in_use[0]), + std::error::Error::source(&full[1]), + std::error::Error::source(&generic), + ] { + assert!( + error.is_some(), + "The technical cause must stay in the source chain." + ); + } + + for error in [ + UpgradeError::Unrecognized, + UpgradeError::IdentityRoster { source: None }, + UpgradeError::Verification, + UpgradeError::TypedValidation(Box::new(std::io::Error::other("fixture"))), + ] { + assert!(!error.is_retryable(), "{error:?}"); + } +} + +#[test] +fn platform_compatibility_malformed_roster_reports_one_message() { + let (_dir, path, target) = fixture(); + Connection::open(&path) + .unwrap() + .execute_batch("INSERT INTO meta_global VALUES ('det:identity_index:v1', X'01ff', 0)") + .unwrap(); + let decode = upgrade(&path, &target, |_| Ok(())).unwrap_err(); + assert!( + matches!(decode, UpgradeError::IdentityRoster { source: Some(_) }), + "{decode:?}" + ); + assert_eq!( + decode.to_string(), + UpgradeError::IdentityRoster { source: None }.to_string() + ); } #[test] @@ -286,7 +421,7 @@ fn platform_compatibility_missing_roster_with_identity_sidecar_is_ambiguous() { let before = snapshot(&path); assert!(matches!( upgrade(&path, &target, |_| Ok(())), - Err(UpgradeError::IdentityRoster) + Err(UpgradeError::IdentityRoster { source: None }) )); assert_eq!(snapshot(&path), before); assert!(backup_files(dir.path()).is_empty()); diff --git a/src/wallet_backend/platform_compatibility/mod.rs b/src/wallet_backend/platform_compatibility/mod.rs index 657d05bdb..8edbca047 100644 --- a/src/wallet_backend/platform_compatibility/mod.rs +++ b/src/wallet_backend/platform_compatibility/mod.rs @@ -2,6 +2,7 @@ mod engine; pub use engine::UpgradeError; +pub(crate) use engine::remove_backups; use platform_wallet::changeset::PlatformWalletPersistence; use platform_wallet_storage::{SqlitePersister, SqlitePersisterConfig, WalletStorageError}; @@ -9,6 +10,9 @@ use platform_wallet_storage::{SqlitePersister, SqlitePersisterConfig, WalletStor use crate::backend_task::error::TaskError; pub(crate) fn open(config: SqlitePersisterConfig) -> Result { + // TODO: each failed open of an old-lineage database leaves another upstream + // `backups/auto/pre-migration-*` copy; bound them once a terminal open failure is + // remembered per process, or upstream offers per-database retention. let original = match SqlitePersister::open(config.clone()) { Ok(persister) => return Ok(persister), Err(error @ WalletStorageError::Migration(_)) => error, From c2b7e459cbe5222ac35420acf0df1ad543e44994 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Wed, 16 Sep 2026 14:59:40 +0000 Subject: [PATCH 02/30] fix(wallet): address upgrade backup retention and cleanup reviews MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keep upstream migration snapshots bounded, validate backup candidates, and preserve pending identity cleanup until backup removal succeeds. Clarify private-key recovery and SQLite sidecar handling. Co-Authored-By: Codex 🤖 Co-authored by [Claudius the Magnificent](https://github.com/lklimek/claudius) AI Agent --- CHANGELOG.md | 8 +- .../2026-09-10-platform-pin/upgrade-notes.md | 9 +- src/backend_task/error.rs | 6 +- src/context/identity_db.rs | 46 +++++- src/ui/identity/mod.rs | 12 +- .../platform_compatibility/engine.rs | 139 +++++++++++++++--- .../platform_compatibility/engine/tests.rs | 56 +++++++ .../platform_compatibility/mod.rs | 55 ++++++- 8 files changed, 285 insertions(+), 46 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f84376e5b..2e19baba0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,14 +27,18 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). longer tells you to remove your local wallet data, which could have deleted the `secrets` folder along with keys no recovery phrase can restore. It now asks you to write down your recovery phrases and imported keys in the version - you used before, then set aside only the `.sqlite` files and keep the - `secrets` folder. + you used before, close every running instance, then set aside the `.sqlite` + files together with their matching `-wal` and `-shm` files and keep the + `secrets` folder. Imported private keys are listed separately from recovery phrases. - **Upgrade backups no longer pile up or outlive deleted data**: a failed database upgrade no longer leaves a new backup on every attempt. Each database keeps at most one upgrade backup, and it is deleted when you remove a wallet or identity or clear a network's data. An upgrade that can never succeed now shows its message instead of being retried silently. + Retention and deletion cover upstream pre-migration snapshots too. Cleanup + rejects links and non-files, reports pruning failures, and retries incomplete + identity backup deletion at the next startup. ### Added diff --git a/docs/ai-design/2026-09-10-platform-pin/upgrade-notes.md b/docs/ai-design/2026-09-10-platform-pin/upgrade-notes.md index 20f68b8df..2ba0c275a 100644 --- a/docs/ai-design/2026-09-10-platform-pin/upgrade-notes.md +++ b/docs/ai-design/2026-09-10-platform-pin/upgrade-notes.md @@ -61,8 +61,13 @@ preserved. An ambiguous or malformed saved identity roster fails closed. The encrypted seed vault is not migrated or rewritten. The bridge takes its backup only after the converted copy validates, just before -the rebuild, and keeps at most one backup per database. Deleting a wallet or an -identity, or clearing a network's data, also deletes these backups, because they +the rebuild, and keeps at most one backup per database. Retention includes upstream +`backups/auto/pre-migration--*.db` snapshots and runs before and after +storage opens, including failed opens. Pruning errors stop the operation; backup +candidates must have the exact database-specific name format and be regular files, +not symbolic links (or hard links on Unix). Identity cleanup retains its pending +manifest until backup deletion succeeds, so startup retries incomplete cleanup. +Deleting a wallet or an identity, or clearing a network's data, also deletes these backups, because they copy wallet and identity history. Downgrading does not automatically reverse the database conversion; recovery requires the corresponding backup and the prior application version. Validation uses synthetic upstream fixtures, not a user's diff --git a/src/backend_task/error.rs b/src/backend_task/error.rs index 63f93e3ce..fd0812378 100644 --- a/src/backend_task/error.rs +++ b/src/backend_task/error.rs @@ -489,7 +489,7 @@ pub enum TaskError { /// user to keep the `secrets` folder: the vault holds imported and /// masternode keys that no recovery phrase can re-derive. #[error( - "Your wallet data was saved by an app version that this version cannot open, so it was left unchanged. Your recovery phrases and keys are still stored on this device. First open the app version you used before and write down the recovery phrase of every wallet and every key you imported. Then move the files ending in .sqlite out of the app data folder, leave the secrets folder where it is, and start this app again." + "Your wallet data was saved by an app version that this version cannot open, so it was left unchanged. Your recovery phrases and keys are still stored on this device. First open the app version you used before and write down the recovery phrase of every wallet. Separately, write down each imported private key. Close all Dash Evo Tool windows and command-line sessions. Then move each .sqlite file together with its matching .sqlite-wal and .sqlite-shm files, if present, out of the app data folder, leave the secrets folder where it is, and start this app again." )] WalletDataIncompatible { #[source] @@ -1252,7 +1252,7 @@ pub enum TaskError { /// The identity and its private keys are gone, but at least one optional /// owner-scoped sidecar could not be removed. #[error( - "The identity was removed, but some DashPay or token-list data may still be stored on this device. The app will keep trying to clear this local data automatically." + "The identity was removed, but some local identity data or upgrade backups may still be stored on this device. The app will keep trying to clear this local data automatically." )] IdentitySidecarCleanupIncomplete, @@ -5825,6 +5825,8 @@ mod tests { ); let msg = err.to_string(); + assert!(msg.contains(".sqlite-wal") && msg.contains(".sqlite-shm")); + assert!(msg.contains("Close all") && msg.contains("write down each imported private key")); assert!( msg.contains("write down the recovery phrase") && msg.contains(".sqlite") diff --git a/src/context/identity_db.rs b/src/context/identity_db.rs index 30d133894..36f9b6349 100644 --- a/src/context/identity_db.rs +++ b/src/context/identity_db.rs @@ -1737,13 +1737,7 @@ impl AppContext { "Deleted identity still registered with the wallet store" ); } - if let Err(error) = self.remove_upgrade_backups() { - tracing::warn!( - identity_id = %identifier, - ?error, - "Upgrade backups holding a deleted identity could not be removed" - ); - } + Ok(sidecar_cleanup) } @@ -1762,7 +1756,12 @@ impl AppContext { .wallet_backend() .and_then(|backend| backend.dashpay_clear_owner_overlays(&identifier)); let token_cleanup = super::contract_token_db::forget_identity_token_state(kv, &identifier); - let sidecar_cleanup = sidecar_cleanup_outcome(dashpay_cleanup, token_cleanup, &identifier); + let mut sidecar_cleanup = + sidecar_cleanup_outcome(dashpay_cleanup, token_cleanup, &identifier); + if let Err(error) = self.remove_upgrade_backups() { + sidecar_cleanup = IdentitySidecarCleanup::Incomplete; + tracing::warn!(identity_id = %identifier, ?error, "Upgrade backup removal remains pending after identity deletion"); + } if !sidecar_cleanup.is_incomplete() { clear_vault_cleanup_manifest(kv, id); } @@ -5372,4 +5371,35 @@ mod tests { ); staged.ctx.wallet_backend().unwrap().shutdown().await; } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn removal_recovery_retains_manifest_until_backups_are_removed() { + let staged = stage_identity_with_vaulted_keys([0x33; 32], [0x44; 32]).await; + let kv = staged.ctx.det_kv().unwrap(); + let blocked = staged + .ctx + .data_dir() + .join("det-app.sqlite.platform-67d4ef3-backup-blocked.sqlite"); + std::fs::create_dir(&blocked).unwrap(); + assert!(matches!( + staged.ctx.delete_local_qualified_identity(&staged.id), + Err(TaskError::IdentitySidecarCleanupIncomplete) + )); + staged.ctx.resume_pending_vault_cleanups(); + assert!( + !kv.list(DetScope::Global, Some(VAULT_CLEANUP_PENDING_PREFIX)) + .unwrap() + .is_empty() + ); + std::fs::remove_dir(&blocked).unwrap(); + std::fs::write(&blocked, b"backup").unwrap(); + staged.ctx.resume_pending_vault_cleanups(); + assert!(!blocked.exists()); + assert!( + kv.list(DetScope::Global, Some(VAULT_CLEANUP_PENDING_PREFIX)) + .unwrap() + .is_empty() + ); + staged.ctx.wallet_backend().unwrap().shutdown().await; + } } diff --git a/src/ui/identity/mod.rs b/src/ui/identity/mod.rs index 3bc35d894..9167752c8 100644 --- a/src/ui/identity/mod.rs +++ b/src/ui/identity/mod.rs @@ -106,18 +106,18 @@ pub const IDENTITY_REMOVED_VOTER_LEFT_AND_CLEANUP_PENDING: &str = "The identity /// Shown when the identity and its keys are gone but secondary local data /// could not be fully cleared. -pub const IDENTITY_REMOVED_LOCAL_DATA_CLEANUP_FAILED: &str = "The identity and its private keys were removed, but some DashPay or token-list data may still be stored on this device. The app will keep trying to clear this local data automatically."; +pub const IDENTITY_REMOVED_LOCAL_DATA_CLEANUP_FAILED: &str = "The identity and its private keys were removed, but some local identity data or upgrade backups may still be stored on this device. The app will keep trying to clear this local data automatically."; /// Shown when the voter identity remains and secondary local data cleanup also /// failed for at least one identity. -pub const IDENTITY_REMOVED_VOTER_LEFT_AND_LOCAL_DATA_CLEANUP_FAILED: &str = "The identity was removed, but its associated voter identity could not be removed. Some DashPay or token-list data may also remain on this device. Retry the voter removal after restarting the app. The app will keep trying to clear this local data automatically."; +pub const IDENTITY_REMOVED_VOTER_LEFT_AND_LOCAL_DATA_CLEANUP_FAILED: &str = "The identity was removed, but its associated voter identity could not be removed. Some local identity data or upgrade backups may also remain on this device. Retry the voter removal after restarting the app. The app will keep trying to clear this local data automatically."; /// Shown when both vault cleanup and secondary local-data cleanup are /// incomplete for an identity that is already delisted. -pub const IDENTITY_REMOVED_ALL_CLEANUP_PENDING: &str = "The identity was removed, but its private keys may still be stored here, and some DashPay or token-list data may remain. The app will keep trying to clear the keys automatically. Until then, treat this device as if it still holds them. The app will keep trying to clear this local data automatically."; +pub const IDENTITY_REMOVED_ALL_CLEANUP_PENDING: &str = "The identity was removed, but its private keys may still be stored here, and some local identity data or upgrade backups may remain. The app will keep trying to clear the keys automatically. Until then, treat this device as if it still holds them. The app will keep trying to clear this local data automatically."; /// Shown when the voter remains and both cleanup categories are incomplete. -pub const IDENTITY_REMOVED_VOTER_LEFT_AND_ALL_CLEANUP_PENDING: &str = "The identity was removed, but its associated voter identity could not be removed. Private keys for one or both identities may still be stored here, and some DashPay or token-list data may remain. The app will keep trying to clear the keys automatically. Until then, treat this device as if it still holds them. Retry the voter removal after restarting the app. The app will keep trying to clear this local data automatically."; +pub const IDENTITY_REMOVED_VOTER_LEFT_AND_ALL_CLEANUP_PENDING: &str = "The identity was removed, but its associated voter identity could not be removed. Private keys for one or both identities may still be stored here, and some local identity data or upgrade backups may remain. The app will keep trying to clear the keys automatically. Until then, treat this device as if it still holds them. Retry the voter removal after restarting the app. The app will keep trying to clear this local data automatically."; /// Shown when a removal is refused because the storage update is still running. pub const IDENTITY_REMOVAL_BLOCKED_BY_STORAGE_UPDATE: &str = @@ -345,7 +345,7 @@ mod tests { fn removed_identities_banner_reports_incomplete_local_data_cleanup() { let (message, message_type) = removed_identities_banner(false, false, true); assert_eq!(message, IDENTITY_REMOVED_LOCAL_DATA_CLEANUP_FAILED); - assert!(message.contains("DashPay or token-list data")); + assert!(message.contains("local identity data or upgrade backups")); assert!(message.contains("keep trying to clear this local data automatically")); assert_eq!(message_type, MessageType::Warning); } @@ -355,7 +355,7 @@ mod tests { let (message, message_type) = removed_identities_banner(false, true, true); assert_eq!(message, IDENTITY_REMOVED_ALL_CLEANUP_PENDING); assert!(message.contains("private keys")); - assert!(message.contains("DashPay or token-list data")); + assert!(message.contains("local identity data or upgrade backups")); assert_eq!(message_type, MessageType::Warning); } diff --git a/src/wallet_backend/platform_compatibility/engine.rs b/src/wallet_backend/platform_compatibility/engine.rs index a2418f7e9..46fadb2b4 100644 --- a/src/wallet_backend/platform_compatibility/engine.rs +++ b/src/wallet_backend/platform_compatibility/engine.rs @@ -219,33 +219,133 @@ fn backup_prefix(path: &Path) -> Option { /// Retained upgrade backups of the database at `path`. fn backups(path: &Path) -> std::io::Result> { + backups_in( + path, + Some(&platform_wallet_storage::default_auto_backup_dir(path)), + ) +} + +fn backups_in(path: &Path, auto_dir: Option<&Path>) -> std::io::Result> { let (Some(parent), Some(prefix)) = (path.parent(), backup_prefix(path)) else { return Ok(Vec::new()); }; - let entries = match std::fs::read_dir(parent) { - Ok(entries) => entries, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), - Err(error) => return Err(error), - }; let mut found = Vec::new(); - for entry in entries { - let entry = entry?; - let name = entry.file_name(); - let name = name.to_string_lossy(); - if name.starts_with(&prefix) && name.ends_with(".sqlite") { - found.push(entry.path()); + for directory in [Some(parent), auto_dir].into_iter().flatten() { + let metadata = match std::fs::symlink_metadata(directory) { + Ok(metadata) => metadata, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, + Err(error) => return Err(error), + }; + if !metadata.is_dir() { + return Err(std::io::Error::other( + "Backup directory is not a regular directory", + )); + } + for entry in std::fs::read_dir(directory)? { + let entry = entry?; + let name = entry.file_name(); + let Some(name) = name.to_str() else { continue }; + let bridge = directory == parent + && name + .strip_prefix(&prefix) + .and_then(|suffix| suffix.strip_suffix(".sqlite")) + .is_some_and(|suffix| { + !suffix.is_empty() && suffix.bytes().all(|b| b.is_ascii_alphanumeric()) + }); + let upstream = Some(directory) == auto_dir && upstream_backup_name(path, name); + if bridge || upstream { + validate_backup_file(&entry.path(), path)?; + found.push(entry.path()); + } } } + found.sort(); + found.dedup(); Ok(found) } +fn upstream_backup_name(path: &Path, name: &str) -> bool { + let Some(stem) = path.file_stem().and_then(|stem| stem.to_str()) else { + return false; + }; + // DET database names are already valid upstream stems; reject lossy/ambiguous names. + if stem.is_empty() + || stem.len() > 32 + || !stem + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_') + { + return false; + } + let Some(suffix) = name + .strip_prefix(&format!("pre-migration-{stem}-")) + .and_then(|suffix| suffix.strip_suffix(".db")) + else { + return false; + }; + let Some((versions, timestamp)) = suffix.rsplit_once('-') else { + return false; + }; + let Some((from, to)) = versions.split_once("-to-") else { + return false; + }; + from.parse::().is_ok() + && to.parse::().is_ok() + && chrono::NaiveDateTime::parse_from_str(timestamp, "%Y%m%dT%H%M%SZ").is_ok() +} + +fn validate_backup_file(backup: &Path, database: &Path) -> std::io::Result<()> { + let metadata = std::fs::symlink_metadata(backup)?; + if !metadata.is_file() || backup == database { + return Err(std::io::Error::other( + "Refusing to remove a non-regular backup file", + )); + } + if let Ok(live_path) = database.canonicalize() + && backup.canonicalize()? == live_path + { + return Err(std::io::Error::other( + "Refusing to remove the live database", + )); + } + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + if metadata.nlink() != 1 { + return Err(std::io::Error::other( + "Refusing to remove a hard-linked backup file", + )); + } + } + Ok(()) +} + +fn remove_backup(backup: &Path, database: &Path) -> std::io::Result<()> { + validate_backup_file(backup, database)?; + std::fs::remove_file(backup) +} + +/// Retain the newest snapshot across both upgrade-backup formats. +pub(super) fn retain_one_backup(path: &Path, auto_dir: Option<&Path>) -> std::io::Result<()> { + let mut snapshots = backups_in(path, auto_dir)? + .into_iter() + .map(|backup| Ok((std::fs::metadata(&backup)?.modified()?, backup))) + .collect::>>()?; + snapshots.sort(); + snapshots.pop(); + for (_, backup) in snapshots { + remove_backup(&backup, path)?; + } + Ok(()) +} + /// Delete every retained upgrade backup of the database at `path`. /// /// Backups never contain vault secrets. Attempts every file and returns the first failure. pub(crate) fn remove_backups(path: &Path) -> std::io::Result<()> { let mut first_error = None; for backup in backups(path)? { - if let Err(error) = std::fs::remove_file(&backup) { + if let Err(error) = remove_backup(&backup, path) { first_error.get_or_insert(error); } } @@ -257,7 +357,7 @@ fn backup(path: &Path) -> Result { let prefix = backup_prefix(path).ok_or(UpgradeError::Unrecognized)?; let file = tempfile::Builder::new() .prefix(&prefix) - .suffix(".sqlite") + .suffix(".pending") .tempfile_in(parent)?; let source = Connection::open_with_flags( path, @@ -273,15 +373,14 @@ fn backup(path: &Path) -> Result { verify(&dest)?; drop(dest); file.as_file().sync_all()?; - let (_, kept) = file.keep().map_err(|e| e.error)?; + // A verified snapshot supersedes old backups only while the original is still untouched. + for old in backups(path)? { + remove_backup(&old, path)?; + } + let kept = file.path().with_extension("sqlite"); + file.persist_noclobber(&kept).map_err(|e| e.error)?; #[cfg(unix)] std::fs::File::open(parent)?.sync_all()?; - // Only an unconverted original reaches this point, so older backups are superseded snapshots. - for old in backups(path)?.into_iter().filter(|old| *old != kept) { - if let Err(error) = std::fs::remove_file(&old) { - tracing::warn!(backup = %old.display(), %error, "Could not remove a superseded wallet upgrade backup"); - } - } Ok(kept) } diff --git a/src/wallet_backend/platform_compatibility/engine/tests.rs b/src/wallet_backend/platform_compatibility/engine/tests.rs index 074b4cf84..d91ecc04d 100644 --- a/src/wallet_backend/platform_compatibility/engine/tests.rs +++ b/src/wallet_backend/platform_compatibility/engine/tests.rs @@ -281,6 +281,62 @@ fn platform_compatibility_backup_removal_only_touches_the_named_database() { remove_backups(&dir.path().join("absent.sqlite")).unwrap(); } +#[test] +fn platform_compatibility_removes_only_matching_upstream_backups() { + let dir = tempfile::tempdir().unwrap(); + let auto = dir.path().join("backups/auto"); + std::fs::create_dir_all(&auto).unwrap(); + let removed = auto.join("pre-migration-det-testnet-1-to-2-20260915T120000Z.db"); + let kept = auto.join("pre-migration-det-testnet-other-1-to-2-20260915T120000Z.db"); + for path in [&removed, &kept] { + std::fs::write(path, b"backup").unwrap(); + } + remove_backups(&dir.path().join("det-testnet.sqlite")).unwrap(); + assert!(!removed.exists()); + assert!(kept.exists()); +} + +#[test] +fn platform_compatibility_prune_failure_preserves_original() { + let (dir, path, target) = fixture(); + let before = snapshot(&path); + std::fs::create_dir( + dir.path() + .join("wallet.sqlite.platform-67d4ef3-backup-blocked.sqlite"), + ) + .unwrap(); + assert!(upgrade(&path, &target, |_| Ok(())).is_err()); + assert_eq!(snapshot(&path), before); +} + +#[cfg(unix)] +#[test] +fn platform_compatibility_rejects_backup_links_to_live_databases() { + let (dir, path, target) = fixture(); + let candidate = dir + .path() + .join("wallet.sqlite.platform-67d4ef3-backup-link.sqlite"); + for live in [&path, &target] { + std::os::unix::fs::symlink(live, &candidate).unwrap(); + assert!(remove_backups(&path).is_err()); + assert!(live.exists()); + assert!(candidate.symlink_metadata().is_ok()); + std::fs::remove_file(&candidate).unwrap(); + std::fs::hard_link(live, &candidate).unwrap(); + assert!(remove_backups(&path).is_err()); + assert!(live.exists()); + std::fs::remove_file(&candidate).unwrap(); + } + let live_alias = dir.path().join("alias.sqlite"); + let live_target = dir + .path() + .join("alias.sqlite.platform-67d4ef3-backup-live.sqlite"); + std::fs::write(&live_target, b"live database").unwrap(); + std::os::unix::fs::symlink(&live_target, &live_alias).unwrap(); + assert!(remove_backups(&live_alias).is_err()); + assert!(live_target.exists()); +} + fn sqlite_failure(code: std::ffi::c_int) -> rusqlite::Error { rusqlite::Error::SqliteFailure(rusqlite::ffi::Error::new(code), None) } diff --git a/src/wallet_backend/platform_compatibility/mod.rs b/src/wallet_backend/platform_compatibility/mod.rs index 8edbca047..1c09142e9 100644 --- a/src/wallet_backend/platform_compatibility/mod.rs +++ b/src/wallet_backend/platform_compatibility/mod.rs @@ -10,20 +10,28 @@ use platform_wallet_storage::{SqlitePersister, SqlitePersisterConfig, WalletStor use crate::backend_task::error::TaskError; pub(crate) fn open(config: SqlitePersisterConfig) -> Result { - // TODO: each failed open of an old-lineage database leaves another upstream - // `backups/auto/pre-migration-*` copy; bound them once a terminal open failure is - // remembered per process, or upstream offers per-database retention. + let prune = || { + engine::retain_one_backup(&config.path, config.auto_backup_dir.as_deref()) + .map_err(|source| TaskError::FileSystem { source }) + }; + // Check retention before another attempt can create a snapshot, including after a restart. + prune()?; + let result = open_inner(&config); + prune()?; + result +} + +fn open_inner(config: &SqlitePersisterConfig) -> Result { let original = match SqlitePersister::open(config.clone()) { Ok(persister) => return Ok(persister), Err(error @ WalletStorageError::Migration(_)) => error, Err(error) => return Err(TaskError::from_wallet_storage_open_error(error)), }; - let result = - upgrade(&config).map_err(|source| TaskError::PlatformDatabaseUpgrade { source })?; + let result = upgrade(config).map_err(|source| TaskError::PlatformDatabaseUpgrade { source })?; if !result { return Err(TaskError::from_wallet_storage_open_error(original)); } - SqlitePersister::open(config).map_err(TaskError::from_wallet_storage_open_error) + SqlitePersister::open(config.clone()).map_err(TaskError::from_wallet_storage_open_error) } fn upgrade(config: &SqlitePersisterConfig) -> Result { @@ -65,6 +73,39 @@ mod tests { use super::*; use dash_sdk::dpp::identity::accessors::IdentityGettersV0; + #[test] + fn platform_compatibility_bounds_upstream_backups_after_failed_opens() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("wallet.sqlite"); + rusqlite::Connection::open(&path) + .unwrap() + .execute_batch(include_str!("fixtures/67d4ef3.sql")) + .unwrap(); + rusqlite::Connection::open(&path) + .unwrap() + .execute_batch("ALTER TABLE wallets ADD COLUMN unsupported INTEGER;") + .unwrap(); + let auto = platform_wallet_storage::default_auto_backup_dir(&path); + std::fs::create_dir_all(&auto).unwrap(); + for attempt in 1..=3 { + let old = auto.join(format!( + "pre-migration-wallet-1-to-2-20260915T12000{attempt}Z.db" + )); + std::fs::write(&old, b"snapshot from an earlier attempt").unwrap(); + assert!(open(SqlitePersisterConfig::new(&path)).is_err()); + assert_eq!(std::fs::read_dir(&auto).unwrap().count(), 1); + let retained = std::fs::read_dir(&auto) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + if retained != old { + std::fs::rename(retained, old).unwrap(); + } + } + } + #[test] fn platform_compatibility_normal_open_upgrades_old_profile() { let dir = tempfile::tempdir().unwrap(); @@ -116,5 +157,7 @@ mod tests { ); drop(persister); open(SqlitePersisterConfig::new(&path)).unwrap(); + let upstream = platform_wallet_storage::default_auto_backup_dir(&path); + assert_eq!(std::fs::read_dir(upstream).unwrap().count(), 0); } } From 095f8c8a1723c4c5280cd5a2b67fd2441de2ad4e Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Thu, 17 Sep 2026 07:03:18 +0000 Subject: [PATCH 03/30] fix(wallet): clean crash snapshots and classify upgrade retries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove pending upgrade snapshots during retention and deletion without pruning an active backup or replacing a published recovery copy. Clean backups when removing a wallet even if its backend is unavailable. Preserve transient storage and memory failures while surfacing deterministic access errors. Co-Authored-By: Codex 🤖 Co-authored by [Claudius the Magnificent](https://github.com/lklimek/claudius) AI Agent --- CHANGELOG.md | 3 + src/context/wallet_lifecycle/removal.rs | 11 +- src/context/wallet_lifecycle/tests.rs | 52 +++++++++ .../platform_compatibility/engine.rs | 70 +++++++++--- .../platform_compatibility/engine/tests.rs | 104 +++++++++++++++--- 5 files changed, 207 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e19baba0..dd4fb069a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). Retention and deletion cover upstream pre-migration snapshots too. Cleanup rejects links and non-files, reports pruning failures, and retries incomplete identity backup deletion at the next startup. + Cleanup also removes unfinished snapshots left by a crash and runs when + removing a wallet without an available backend. Temporary storage and memory + failures remain retryable; permission and invalid-input failures surface directly. ### Added diff --git a/src/context/wallet_lifecycle/removal.rs b/src/context/wallet_lifecycle/removal.rs index 3babd26e5..ee0ecba2a 100644 --- a/src/context/wallet_lifecycle/removal.rs +++ b/src/context/wallet_lifecycle/removal.rs @@ -81,11 +81,11 @@ impl AppContext { ); } } + } - if let Err(error) = self.remove_upgrade_backups() { - tracing::warn!(?error, "Failed to delete wallet upgrade backups on removal"); - show_wallet_data_removal_warning(self.egui_ctx(), error); - } + if let Err(error) = self.remove_upgrade_backups() { + tracing::warn!(?error, "Failed to delete wallet upgrade backups on removal"); + show_wallet_data_removal_warning(self.egui_ctx(), error); } Ok(()) @@ -94,8 +94,7 @@ impl AppContext { /// Delete the retained compatibility-upgrade backups of the app and this network's /// wallet databases, which copy wallet and identity history a deletion must not leave behind. /// - /// Call only with the wallet backend wired: this process then holds both databases open - /// and already upgraded, so no other instance can be writing a backup of them. + /// Filesystem cleanup is available even when the wallet backend could not be opened. pub(crate) fn remove_upgrade_backups(&self) -> Result<(), TaskError> { let mut first_error = None; for database in [ diff --git a/src/context/wallet_lifecycle/tests.rs b/src/context/wallet_lifecycle/tests.rs index f380e84b1..189763668 100644 --- a/src/context/wallet_lifecycle/tests.rs +++ b/src/context/wallet_lifecycle/tests.rs @@ -6907,3 +6907,55 @@ async fn reconcile_managed_identities_skips_identities_linked_to_another_wallet( backend.shutdown().await; } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn remove_wallet_deletes_upgrade_backups_without_backend() { + let (ctx, _sender, _tmp) = offline_testnet_context(); + assert!(ctx.wallet_backend().is_err()); + let wallet = + crate::model::wallet::Wallet::new_from_seed([0xA7; 64], Network::Testnet, None, None) + .expect("build wallet"); + let seed_hash = wallet.seed_hash(); + ctx.wallets + .write() + .unwrap() + .insert(seed_hash, Arc::new(RwLock::new(wallet))); + let backups = [ + "det-app.sqlite.platform-67d4ef3-backup-fixture.sqlite", + "det-testnet.sqlite.platform-67d4ef3-backup-fixture.pending", + "backups/auto/pre-migration-det-testnet-1-to-2-20260915T120001Z.db", + ] + .map(|name| ctx.data_dir().join(name)); + std::fs::create_dir_all(backups[2].parent().unwrap()).unwrap(); + for backup in &backups { + std::fs::write(backup, b"old wallet history").unwrap(); + } + ctx.remove_wallet(&seed_hash).unwrap(); + for backup in &backups { + assert!(!backup.exists(), "{}", backup.display()); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn remove_wallet_warns_when_backup_cleanup_fails_without_backend() { + let (ctx, _sender, _tmp) = offline_testnet_context(); + assert!(ctx.wallet_backend().is_err()); + let wallet = + crate::model::wallet::Wallet::new_from_seed([0xA7; 64], Network::Testnet, None, None) + .expect("build wallet"); + let seed_hash = wallet.seed_hash(); + ctx.wallets + .write() + .unwrap() + .insert(seed_hash, Arc::new(RwLock::new(wallet))); + let backup = ctx + .data_dir() + .join("det-app.sqlite.platform-67d4ef3-backup-blocked.pending"); + std::fs::create_dir(&backup).unwrap(); + crate::ui::components::MessageBanner::clear_all_global(ctx.egui_ctx()); + ctx.remove_wallet(&seed_hash).unwrap(); + assert!(backup.is_dir()); + assert!(crate::ui::components::MessageBanner::has_global( + ctx.egui_ctx() + )); +} diff --git a/src/wallet_backend/platform_compatibility/engine.rs b/src/wallet_backend/platform_compatibility/engine.rs index 46fadb2b4..48da0d035 100644 --- a/src/wallet_backend/platform_compatibility/engine.rs +++ b/src/wallet_backend/platform_compatibility/engine.rs @@ -20,6 +20,18 @@ pub enum UpgradeError { "Could not upgrade wallet data because the disk is full. Free up disk space and try again." )] StorageFull(#[source] Box), + #[error( + "Could not access wallet storage. Check that the drive is connected and available, then try again." + )] + StorageUnavailable(#[source] Box), + #[error( + "Could not upgrade wallet data because memory is exhausted. Close other applications and try again." + )] + OutOfMemory(#[source] Box), + #[error( + "Could not write wallet data. Allow write access to the app data folder, then restart the application." + )] + AccessDenied(#[source] Box), #[error( "Could not upgrade wallet data, and your data was not changed. Restart the application to try again, or keep your data folder and reopen the previous application version." )] @@ -51,10 +63,15 @@ pub enum UpgradeError { } impl UpgradeError { - /// Whether the user can clear the cause (close another instance, free space) and retry; - /// every other failure repeats identically on the same data. + /// Whether retrying can recover from contention or temporary resource exhaustion. pub fn is_retryable(&self) -> bool { - matches!(self, Self::InUse(_) | Self::StorageFull(_) | Self::Io(_)) + matches!( + self, + Self::InUse(_) + | Self::StorageFull(_) + | Self::StorageUnavailable(_) + | Self::OutOfMemory(_) + ) } } @@ -65,6 +82,11 @@ impl From for UpgradeError { Self::InUse(error) } Some(rusqlite::ErrorCode::DiskFull) => Self::StorageFull(Box::new(error)), + Some(rusqlite::ErrorCode::SystemIoFailure) => Self::StorageUnavailable(Box::new(error)), + Some(rusqlite::ErrorCode::OutOfMemory) => Self::OutOfMemory(Box::new(error)), + Some(rusqlite::ErrorCode::PermissionDenied | rusqlite::ErrorCode::ReadOnly) => { + Self::AccessDenied(Box::new(error)) + } _ => Self::Sqlite(error), } } @@ -72,10 +94,18 @@ impl From for UpgradeError { impl From for UpgradeError { fn from(error: std::io::Error) -> Self { - if error.kind() == std::io::ErrorKind::StorageFull { - Self::StorageFull(Box::new(error)) - } else { - Self::Io(error) + use std::io::ErrorKind; + match error.kind() { + ErrorKind::StorageFull => Self::StorageFull(Box::new(error)), + ErrorKind::OutOfMemory => Self::OutOfMemory(Box::new(error)), + ErrorKind::Interrupted + | ErrorKind::WouldBlock + | ErrorKind::TimedOut + | ErrorKind::ResourceBusy => Self::StorageUnavailable(Box::new(error)), + ErrorKind::PermissionDenied | ErrorKind::ReadOnlyFilesystem => { + Self::AccessDenied(Box::new(error)) + } + _ => Self::Io(error), } } } @@ -248,7 +278,11 @@ fn backups_in(path: &Path, auto_dir: Option<&Path>) -> std::io::Result std::io::Result<()> { /// Retain the newest snapshot across both upgrade-backup formats. pub(super) fn retain_one_backup(path: &Path, auto_dir: Option<&Path>) -> std::io::Result<()> { - let mut snapshots = backups_in(path, auto_dir)? - .into_iter() - .map(|backup| Ok((std::fs::metadata(&backup)?.modified()?, backup))) - .collect::>>()?; + let mut snapshots = Vec::new(); + for backup in backups_in(path, auto_dir)? { + // Unpublished copies may be incomplete and must never supersede a recovery snapshot. + if backup + .extension() + .is_some_and(|extension| extension == "pending") + { + remove_backup(&backup, path)?; + } else { + snapshots.push((std::fs::metadata(&backup)?.modified()?, backup)); + } + } snapshots.sort(); snapshots.pop(); for (_, backup) in snapshots { @@ -375,7 +417,9 @@ fn backup(path: &Path) -> Result { file.as_file().sync_all()?; // A verified snapshot supersedes old backups only while the original is still untouched. for old in backups(path)? { - remove_backup(&old, path)?; + if old != file.path() { + remove_backup(&old, path)?; + } } let kept = file.path().with_extension("sqlite"); file.persist_noclobber(&kept).map_err(|e| e.error)?; diff --git a/src/wallet_backend/platform_compatibility/engine/tests.rs b/src/wallet_backend/platform_compatibility/engine/tests.rs index d91ecc04d..35c42392b 100644 --- a/src/wallet_backend/platform_compatibility/engine/tests.rs +++ b/src/wallet_backend/platform_compatibility/engine/tests.rs @@ -313,19 +313,21 @@ fn platform_compatibility_prune_failure_preserves_original() { #[test] fn platform_compatibility_rejects_backup_links_to_live_databases() { let (dir, path, target) = fixture(); - let candidate = dir - .path() - .join("wallet.sqlite.platform-67d4ef3-backup-link.sqlite"); - for live in [&path, &target] { - std::os::unix::fs::symlink(live, &candidate).unwrap(); - assert!(remove_backups(&path).is_err()); - assert!(live.exists()); - assert!(candidate.symlink_metadata().is_ok()); - std::fs::remove_file(&candidate).unwrap(); - std::fs::hard_link(live, &candidate).unwrap(); - assert!(remove_backups(&path).is_err()); - assert!(live.exists()); - std::fs::remove_file(&candidate).unwrap(); + for extension in ["sqlite", "pending"] { + let candidate = dir.path().join(format!( + "wallet.sqlite.platform-67d4ef3-backup-link.{extension}" + )); + for live in [&path, &target] { + std::os::unix::fs::symlink(live, &candidate).unwrap(); + assert!(remove_backups(&path).is_err()); + assert!(live.exists()); + assert!(candidate.symlink_metadata().is_ok()); + std::fs::remove_file(&candidate).unwrap(); + std::fs::hard_link(live, &candidate).unwrap(); + assert!(remove_backups(&path).is_err()); + assert!(live.exists()); + std::fs::remove_file(&candidate).unwrap(); + } } let live_alias = dir.path().join("alias.sqlite"); let live_target = dir @@ -364,7 +366,7 @@ fn platform_compatibility_errors_name_their_actual_cause() { assert!(error.to_string().contains("disk")); } let io = UpgradeError::from(std::io::Error::from(std::io::ErrorKind::PermissionDenied)); - assert!(matches!(io, UpgradeError::Io(_)) && io.is_retryable()); + assert!(matches!(io, UpgradeError::AccessDenied(_)) && !io.is_retryable()); assert!(!io.to_string().contains("disk"), "{io}"); let generic = UpgradeError::from(sqlite_failure(rusqlite::ffi::SQLITE_CORRUPT)); @@ -521,3 +523,77 @@ fn platform_compatibility_writer_exclusion_covers_staged_validation() { Ok(()) }).unwrap(); } + +#[test] +fn platform_compatibility_pending_cleanup_preserves_published_snapshot() { + let (dir, path, _target) = fixture(); + let published = backup(&path).unwrap(); + let pending = dir + .path() + .join("wallet.sqlite.platform-67d4ef3-backup-crash.pending"); + std::fs::copy(&path, &pending).unwrap(); + retain_one_backup(&path, None).unwrap(); + assert!(!pending.exists()); + assert!(published.exists()); + assert!(path.exists()); +} + +#[test] +fn platform_compatibility_deletion_removes_pending_snapshots() { + let (dir, path, _target) = fixture(); + let pending = dir + .path() + .join("wallet.sqlite.platform-67d4ef3-backup-crash.pending"); + let unrelated = dir + .path() + .join("other.sqlite.platform-67d4ef3-backup-crash.pending"); + std::fs::copy(&path, &pending).unwrap(); + std::fs::copy(&path, &unrelated).unwrap(); + remove_backups(&path).unwrap(); + assert!(!pending.exists()); + assert!(unrelated.exists()); + assert!(path.exists()); +} + +#[test] +fn platform_compatibility_transient_sqlite_errors_can_retry() { + for code in [rusqlite::ffi::SQLITE_IOERR, rusqlite::ffi::SQLITE_NOMEM] { + let source = UpgradeError::from(sqlite_failure(code)); + assert!(source.is_retryable(), "{source:?}"); + assert!(!crate::backend_task::is_terminal_storage_open_error( + &crate::backend_task::error::TaskError::PlatformDatabaseUpgrade { source } + )); + } +} + +#[test] +fn platform_compatibility_deterministic_io_errors_do_not_retry() { + for kind in [ + std::io::ErrorKind::PermissionDenied, + std::io::ErrorKind::InvalidInput, + ] { + let source = UpgradeError::from(std::io::Error::from(kind)); + assert!(!source.is_retryable(), "{source:?}"); + } +} + +#[test] +fn platform_compatibility_transient_io_errors_preserve_causes() { + use std::io::ErrorKind; + for kind in [ + ErrorKind::Interrupted, + ErrorKind::WouldBlock, + ErrorKind::TimedOut, + ErrorKind::ResourceBusy, + ErrorKind::StorageFull, + ErrorKind::OutOfMemory, + ] { + let error = UpgradeError::from(std::io::Error::from(kind)); + assert!(error.is_retryable(), "{error:?}"); + let cause = std::error::Error::source(&error).unwrap(); + assert_eq!(cause.downcast_ref::().unwrap().kind(), kind); + if kind == ErrorKind::OutOfMemory { + assert!(error.to_string().contains("Close other applications")); + } + } +} From 38d872f26dbfa4e6ad9f63ffabf096461eecc9a5 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Fri, 18 Sep 2026 08:58:57 +0000 Subject: [PATCH 04/30] fix(storage): protect active snapshots and preserve validation retries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Serialize snapshot creation and cleanup across processes. Classify staged validation failures from their typed causes so temporary resource errors retain retry guidance while invalid data stays terminal. Co-Authored-By: Codex 🤖 Co-authored by [Claudius the Magnificent](https://github.com/lklimek/claudius) AI Agent --- CHANGELOG.md | 2 + .../platform_compatibility/engine.rs | 48 +++++- .../platform_compatibility/engine/tests.rs | 68 ++++++++ .../platform_compatibility/mod.rs | 155 ++++++++++++++++-- 4 files changed, 259 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dd4fb069a..3840be38b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). Cleanup also removes unfinished snapshots left by a crash and runs when removing a wallet without an available backend. Temporary storage and memory failures remain retryable; permission and invalid-input failures surface directly. + Concurrent sessions now protect active snapshots from cleanup, and staged + validation preserves retry options and guidance for temporary resource failures. ### Added diff --git a/src/wallet_backend/platform_compatibility/engine.rs b/src/wallet_backend/platform_compatibility/engine.rs index 48da0d035..ad5bea1c7 100644 --- a/src/wallet_backend/platform_compatibility/engine.rs +++ b/src/wallet_backend/platform_compatibility/engine.rs @@ -15,7 +15,7 @@ pub enum UpgradeError { #[error( "Your wallet data is open in another Dash Evo Tool window or command-line session. Close it and try again." )] - InUse(#[source] rusqlite::Error), + InUse(#[source] Box), #[error( "Could not upgrade wallet data because the disk is full. Free up disk space and try again." )] @@ -79,7 +79,7 @@ impl From for UpgradeError { fn from(error: rusqlite::Error) -> Self { match error.sqlite_error_code() { Some(rusqlite::ErrorCode::DatabaseBusy | rusqlite::ErrorCode::DatabaseLocked) => { - Self::InUse(error) + Self::InUse(Box::new(error)) } Some(rusqlite::ErrorCode::DiskFull) => Self::StorageFull(Box::new(error)), Some(rusqlite::ErrorCode::SystemIoFailure) => Self::StorageUnavailable(Box::new(error)), @@ -359,8 +359,42 @@ fn remove_backup(backup: &Path, database: &Path) -> std::io::Result<()> { std::fs::remove_file(backup) } +fn backup_lock(path: &Path) -> std::io::Result> { + let Some(name) = path.file_name() else { + return Err(std::io::ErrorKind::InvalidInput.into()); + }; + let mut lock_name = name.to_os_string(); + lock_name.push(".platform-upgrade.lock"); + let lock_path = path.with_file_name(lock_name); + let mut options = std::fs::File::options(); + options.read(true).write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let file = match options.open(&lock_path) { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => { + validate_backup_file(&lock_path, path)?; + std::fs::File::options() + .read(true) + .write(true) + .open(&lock_path)? + } + // A missing parent has no snapshots to clean up. + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error), + }; + validate_backup_file(&lock_path, path)?; + file.try_lock().map_err(std::io::Error::from)?; + // Keep the pathname stable: unlinking it could let contenders lock different files. + Ok(Some(file)) +} + /// Retain the newest snapshot across both upgrade-backup formats. pub(super) fn retain_one_backup(path: &Path, auto_dir: Option<&Path>) -> std::io::Result<()> { + let _guard = backup_lock(path)?; let mut snapshots = Vec::new(); for backup in backups_in(path, auto_dir)? { // Unpublished copies may be incomplete and must never supersede a recovery snapshot. @@ -385,6 +419,7 @@ pub(super) fn retain_one_backup(path: &Path, auto_dir: Option<&Path>) -> std::io /// /// Backups never contain vault secrets. Attempts every file and returns the first failure. pub(crate) fn remove_backups(path: &Path) -> std::io::Result<()> { + let _guard = backup_lock(path)?; let mut first_error = None; for backup in backups(path)? { if let Err(error) = remove_backup(&backup, path) { @@ -395,12 +430,21 @@ pub(crate) fn remove_backups(path: &Path) -> std::io::Result<()> { } fn backup(path: &Path) -> Result { + backup_with_hook(path, |_| Ok(())) +} + +fn backup_with_hook( + path: &Path, + pending_created: impl FnOnce(&Path) -> Result<(), UpgradeError>, +) -> Result { + let _guard = backup_lock(path)?; let parent = path.parent().ok_or(UpgradeError::Unrecognized)?; let prefix = backup_prefix(path).ok_or(UpgradeError::Unrecognized)?; let file = tempfile::Builder::new() .prefix(&prefix) .suffix(".pending") .tempfile_in(parent)?; + pending_created(file.path())?; let source = Connection::open_with_flags( path, OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NOFOLLOW, diff --git a/src/wallet_backend/platform_compatibility/engine/tests.rs b/src/wallet_backend/platform_compatibility/engine/tests.rs index 35c42392b..84430477e 100644 --- a/src/wallet_backend/platform_compatibility/engine/tests.rs +++ b/src/wallet_backend/platform_compatibility/engine/tests.rs @@ -276,6 +276,7 @@ fn platform_compatibility_backup_removal_only_touches_the_named_database() { "det-mainnet.sqlite.platform-67d4ef3-backup-c3.sqlite", "det-testnet-shielded.sqlite", "det-testnet.sqlite", + "det-testnet.sqlite.platform-upgrade.lock", ] ); remove_backups(&dir.path().join("absent.sqlite")).unwrap(); @@ -597,3 +598,70 @@ fn platform_compatibility_transient_io_errors_preserve_causes() { } } } + +#[test] +fn platform_compatibility_active_snapshot_survives_concurrent_cleanup() { + use std::io::{BufRead, Read, Write}; + const CHILD_DIR: &str = "DET_PLATFORM_COMPAT_ACTIVE_FIXTURE_DIR"; + if let Some(dir) = std::env::var_os(CHILD_DIR) { + let path = PathBuf::from(dir).join("wallet.sqlite"); + backup_with_hook(&path, |pending| { + // Pause the real writer after creation, before it copies the database. + println!("PENDING:{}", pending.display()); + std::io::stdout().flush()?; + std::io::stdin().read_exact(&mut [0])?; + Ok(()) + }) + .unwrap(); + return; + } + for crash in [false, true] { + let (dir, path, _target) = fixture(); + let test_name = std::thread::current().name().unwrap().to_owned(); + let mut child = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", &test_name, "--nocapture"]) + .env(CHILD_DIR, dir.path()) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .spawn() + .unwrap(); + let mut reader = std::io::BufReader::new(child.stdout.take().unwrap()); + let pending = loop { + let mut line = String::new(); + assert_ne!( + reader.read_line(&mut line).unwrap(), + 0, + "Writer exited before creating a snapshot" + ); + if let Some((_, path)) = line.trim().split_once("PENDING:") { + break PathBuf::from(path); + } + }; + let retained = retain_one_backup(&path, None); + let removed = remove_backups(&path); + let exists = pending.exists(); + if crash { + child.kill().unwrap(); + } else { + child.stdin.take().unwrap().write_all(&[1]).unwrap(); + } + let status = child.wait().unwrap(); + assert!(exists, "Cleanup deleted another process's active snapshot"); + assert_eq!(retained.unwrap_err().kind(), std::io::ErrorKind::WouldBlock); + assert_eq!(removed.unwrap_err().kind(), std::io::ErrorKind::WouldBlock); + if crash { + assert!(!status.success()); + assert!(pending.exists()); + retain_one_backup(&path, None).unwrap(); + assert!(!pending.exists()); + continue; + } + assert!(status.success()); + let published = pending.with_extension("sqlite"); + assert_eq!(snapshot(&published), snapshot(&path)); + retain_one_backup(&path, None).unwrap(); + assert!(published.exists()); + remove_backups(&path).unwrap(); + assert!(!published.exists()); + } +} diff --git a/src/wallet_backend/platform_compatibility/mod.rs b/src/wallet_backend/platform_compatibility/mod.rs index 1c09142e9..4176bb32c 100644 --- a/src/wallet_backend/platform_compatibility/mod.rs +++ b/src/wallet_backend/platform_compatibility/mod.rs @@ -45,19 +45,12 @@ fn upgrade(config: &SqlitePersisterConfig) -> Result { } let stage_dir = builder.tempdir_in(parent)?; let stage_path = stage_dir.path().join("wallet.sqlite"); - drop( - SqlitePersister::open(SqlitePersisterConfig::new(&stage_path)) - .map_err(|e| UpgradeError::TypedValidation(Box::new(e)))?, - ); + drop(SqlitePersister::open(SqlitePersisterConfig::new(&stage_path)).map_err(validation_error)?); let backup = engine::upgrade(&config.path, &stage_path, |path| { - let staged = SqlitePersister::open(SqlitePersisterConfig::new(path)) - .map_err(|e| UpgradeError::TypedValidation(Box::new(e)))?; - staged - .load() - .map_err(|e| UpgradeError::TypedValidation(Box::new(e)))?; - staged - .load_unowned_identities() - .map_err(|e| UpgradeError::TypedValidation(Box::new(e)))?; + let staged = + SqlitePersister::open(SqlitePersisterConfig::new(path)).map_err(validation_error)?; + staged.load().map_err(validation_error)?; + staged.load_unowned_identities().map_err(validation_error)?; Ok(()) })?; if let Some(backup) = backup { @@ -68,6 +61,62 @@ fn upgrade(config: &SqlitePersisterConfig) -> Result { } } +fn validation_error(error: impl std::error::Error + Send + Sync + 'static) -> UpgradeError { + use platform_wallet::changeset::PersistenceError; + use rusqlite::ErrorCode; + use std::io::ErrorKind; + + type WrapError = fn(Box) -> UpgradeError; + + let mut cause: &dyn std::error::Error = &error; + let mut transient = false; + loop { + let category: Option = + if let Some(sqlite) = cause.downcast_ref::() { + match sqlite.sqlite_error_code() { + Some(ErrorCode::DatabaseBusy | ErrorCode::DatabaseLocked) => { + Some(UpgradeError::InUse) + } + Some(ErrorCode::DiskFull) => Some(UpgradeError::StorageFull), + Some(ErrorCode::SystemIoFailure) => Some(UpgradeError::StorageUnavailable), + Some(ErrorCode::OutOfMemory) => Some(UpgradeError::OutOfMemory), + Some(ErrorCode::PermissionDenied | ErrorCode::ReadOnly) => { + Some(UpgradeError::AccessDenied) + } + _ => None, + } + } else if let Some(io) = cause.downcast_ref::() { + match io.kind() { + ErrorKind::StorageFull => Some(UpgradeError::StorageFull), + ErrorKind::OutOfMemory => Some(UpgradeError::OutOfMemory), + ErrorKind::Interrupted + | ErrorKind::WouldBlock + | ErrorKind::TimedOut + | ErrorKind::ResourceBusy => Some(UpgradeError::StorageUnavailable), + ErrorKind::PermissionDenied | ErrorKind::ReadOnlyFilesystem => { + Some(UpgradeError::AccessDenied) + } + _ => None, + } + } else { + None + }; + if let Some(wrap) = category { + return wrap(Box::new(error)); + } + if let Some(persistence) = cause.downcast_ref::() { + transient |= persistence.is_transient(); + } + let Some(source) = cause.source() else { break }; + cause = source; + } + if transient { + UpgradeError::StorageUnavailable(Box::new(error)) + } else { + UpgradeError::TypedValidation(Box::new(error)) + } +} + #[cfg(test)] mod tests { use super::*; @@ -161,3 +210,85 @@ mod tests { assert_eq!(std::fs::read_dir(upstream).unwrap().count(), 0); } } + +#[cfg(test)] +mod validation_tests { + use super::*; + use platform_wallet::changeset::PersistenceError; + + #[test] + fn platform_compatibility_staged_resource_failures_remain_retryable() { + for code in [ + rusqlite::ffi::SQLITE_BUSY, + rusqlite::ffi::SQLITE_LOCKED, + rusqlite::ffi::SQLITE_FULL, + rusqlite::ffi::SQLITE_IOERR, + rusqlite::ffi::SQLITE_NOMEM, + ] { + for persistence_wrapper in [false, true] { + let storage = WalletStorageError::Sqlite(rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error::new(code), + None, + )); + let source = if persistence_wrapper { + validation_error(PersistenceError::from(storage)) + } else { + validation_error(storage) + }; + assert!(source.is_retryable(), "{source:?}"); + let cause = std::error::Error::source(&source).unwrap(); + if persistence_wrapper { + assert!(cause.is::()); + } else { + assert!(cause.is::()); + } + assert!(!source.to_string().contains("previous application version")); + assert!(!crate::backend_task::is_terminal_storage_open_error( + &TaskError::PlatformDatabaseUpgrade { source } + )); + } + } + let source = validation_error(WalletStorageError::Io(std::io::Error::from( + std::io::ErrorKind::StorageFull, + ))); + assert!(matches!(source, UpgradeError::StorageFull(_))); + } + + #[test] + fn platform_compatibility_staged_migration_disk_full_remains_retryable() { + let mut connection = rusqlite::Connection::open_in_memory().unwrap(); + connection.pragma_update(None, "max_page_count", 1).unwrap(); + let migration = refinery::Migration::unapplied( + "V1__resource_fixture", + "CREATE TABLE fixture (id INTEGER);", + ) + .unwrap(); + let error = refinery::Runner::new(&[migration]) + .run(&mut connection) + .unwrap_err(); + let source = validation_error(WalletStorageError::Migration(error)); + assert!(matches!(source, UpgradeError::StorageFull(_)), "{source:?}"); + assert!(!crate::backend_task::is_terminal_storage_open_error( + &TaskError::PlatformDatabaseUpgrade { source } + )); + } + + #[test] + fn platform_compatibility_staged_invalid_data_remains_terminal() { + for storage in [ + WalletStorageError::SchemaHistoryMissing, + WalletStorageError::BlobDecode { + reason: "invalid fixture", + }, + WalletStorageError::IntegrityCheckFailed { + report: "invalid fixture".into(), + }, + ] { + let source = validation_error(PersistenceError::from(storage)); + assert!(matches!(source, UpgradeError::TypedValidation(_))); + assert!(crate::backend_task::is_terminal_storage_open_error( + &TaskError::PlatformDatabaseUpgrade { source } + )); + } + } +} From 2b9e40745cace6d15160b8fe7f99983df5ea9891 Mon Sep 17 00:00:00 2001 From: lklimek <842586+lklimek@users.noreply.github.com> Date: Fri, 18 Sep 2026 14:01:54 +0200 Subject: [PATCH 05/30] fix(storage): serialize compatibility open lifecycle --- CHANGELOG.md | 3 + src/backend_task/error.rs | 79 ++++++++++++++++++- .../platform_compatibility/engine.rs | 68 ++++++++++++++-- .../platform_compatibility/engine/tests.rs | 13 +++ .../platform_compatibility/mod.rs | 44 +++++++++-- 5 files changed, 191 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3840be38b..a895ca557 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). you used before, close every running instance, then set aside the `.sqlite` files together with their matching `-wal` and `-shm` files and keep the `secrets` folder. Imported private keys are listed separately from recovery phrases. + A database temporarily held by another session now asks you to close that + session and try again instead of reporting incompatible data. - **Upgrade backups no longer pile up or outlive deleted data**: a failed database upgrade no longer leaves a new backup on every attempt. Each @@ -44,6 +46,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). failures remain retryable; permission and invalid-input failures surface directly. Concurrent sessions now protect active snapshots from cleanup, and staged validation preserves retry options and guidance for temporary resource failures. + One lock now covers the complete open, upgrade, backup, and retention cycle. ### Added diff --git a/src/backend_task/error.rs b/src/backend_task/error.rs index fd0812378..07c868305 100644 --- a/src/backend_task/error.rs +++ b/src/backend_task/error.rs @@ -435,6 +435,15 @@ pub enum TaskError { source: platform_wallet_storage::WalletStorageError, }, + /// Another process currently owns the wallet database write lock. + #[error( + "Your wallet data is open in another Dash Evo Tool window or command-line session. Close it and try again." + )] + WalletStorageInUse { + #[source] + source: platform_wallet_storage::WalletStorageError, + }, + /// A pinned-PR wallet database could not be upgraded without losing data. #[error(transparent)] PlatformDatabaseUpgrade { @@ -2831,7 +2840,7 @@ impl TaskError { /// Map a wallet-storage open failure to the right user-facing variant. /// - /// Three storage failures get honest, distinct copy; everything else keeps + /// Four storage failures get honest, distinct copy; everything else keeps /// the generic disk/IO message: /// /// - A forward-version database (written by a newer build, schema beyond @@ -2844,12 +2853,16 @@ impl TaskError { /// the banner tells the user to back up keys and set the databases aside /// while keeping the vault — freeing disk space or restarting never /// resolves a structural mismatch. + /// - A migration blocked by another SQLite writer is surfaced as + /// [`Self::WalletStorageInUse`] so the user can close the competing + /// process and retry instead of following incompatible-data recovery. /// - Every other storage failure keeps the generic disk/IO copy via /// [`Self::WalletStorage`]. /// /// Discrimination is on the typed upstream variant /// (`WalletStorageError::SchemaVersionUnsupported` / - /// `WalletStorageError::Migration`), never on its `Display` text. + /// `WalletStorageError::Migration`) and the typed migration source chain, + /// never on `Display` text. pub fn from_wallet_storage_open_error( source: platform_wallet_storage::WalletStorageError, ) -> Self { @@ -2861,6 +2874,11 @@ impl TaskError { found, max_supported, }, + other @ platform_wallet_storage::WalletStorageError::Migration(_) + if Self::wallet_storage_error_is_in_use(&other) => + { + Self::WalletStorageInUse { source: other } + } other @ platform_wallet_storage::WalletStorageError::Migration(_) => { Self::WalletDataIncompatible { source: other } } @@ -2868,6 +2886,28 @@ impl TaskError { } } + fn wallet_storage_error_is_in_use( + source: &platform_wallet_storage::WalletStorageError, + ) -> bool { + let mut cause: Option<&(dyn std::error::Error + 'static)> = Some(source); + while let Some(error) = cause { + if error + .downcast_ref::() + .and_then(rusqlite::Error::sqlite_error_code) + .is_some_and(|code| { + matches!( + code, + rusqlite::ErrorCode::DatabaseBusy | rusqlite::ErrorCode::DatabaseLocked + ) + }) + { + return true; + } + cause = error.source(); + } + false + } + /// Returns `true` when this is a [`Self::SecretStore`] open failure caused /// specifically by a vault passphrase mismatch /// ([`SecretStoreError::WrongPassphrase`]). @@ -5810,6 +5850,41 @@ mod tests { .expect_err("divergent checksum must abort") } + fn busy_migration_error() -> refinery::Error { + use refinery::{Migration, Runner}; + + let dir = tempfile::tempdir().expect("temporary directory"); + let path = dir.path().join("wallet.sqlite"); + let writer = rusqlite::Connection::open(&path).expect("writer connection"); + writer + .execute_batch("BEGIN IMMEDIATE; CREATE TABLE held (id INTEGER);") + .expect("hold write lock"); + let mut contender = rusqlite::Connection::open(&path).expect("contender connection"); + contender + .busy_timeout(std::time::Duration::from_millis(1)) + .expect("short deterministic timeout"); + let migration = Migration::unapplied("V1__init", "CREATE TABLE data (id INTEGER);") + .expect("valid migration"); + Runner::new(&[migration]) + .run(&mut contender) + .expect_err("held writer must make the migration busy") + } + + #[test] + fn busy_migration_error_maps_to_retryable_wallet_storage_message() { + let upstream = + platform_wallet_storage::WalletStorageError::Migration(busy_migration_error()); + let err = TaskError::from_wallet_storage_open_error(upstream); + assert!( + matches!(err, TaskError::WalletStorageInUse { .. }), + "Expected WalletStorageInUse, got: {err:?}" + ); + let message = err.to_string(); + assert!(message.contains("Close") && message.contains("try again")); + assert!(!message.contains("incompatible")); + assert!(std::error::Error::source(&err).is_some()); + } + /// A divergent migration history (database written under an /// incompatible storage layout) maps to the dedicated /// `WalletDataIncompatible` variant. Its `Display` must never lead the user diff --git a/src/wallet_backend/platform_compatibility/engine.rs b/src/wallet_backend/platform_compatibility/engine.rs index ad5bea1c7..31d6ba521 100644 --- a/src/wallet_backend/platform_compatibility/engine.rs +++ b/src/wallet_backend/platform_compatibility/engine.rs @@ -359,7 +359,12 @@ fn remove_backup(backup: &Path, database: &Path) -> std::io::Result<()> { std::fs::remove_file(backup) } -fn backup_lock(path: &Path) -> std::io::Result> { +pub(super) struct BackupGuard { + path: PathBuf, + _file: Option, +} + +pub(super) fn backup_lock(path: &Path) -> std::io::Result { let Some(name) = path.file_name() else { return Err(std::io::ErrorKind::InvalidInput.into()); }; @@ -383,18 +388,35 @@ fn backup_lock(path: &Path) -> std::io::Result> { .open(&lock_path)? } // A missing parent has no snapshots to clean up. - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return Ok(BackupGuard { + path: path.to_owned(), + _file: None, + }); + } Err(error) => return Err(error), }; validate_backup_file(&lock_path, path)?; file.try_lock().map_err(std::io::Error::from)?; // Keep the pathname stable: unlinking it could let contenders lock different files. - Ok(Some(file)) + Ok(BackupGuard { + path: path.to_owned(), + _file: Some(file), + }) } /// Retain the newest snapshot across both upgrade-backup formats. +#[cfg(test)] pub(super) fn retain_one_backup(path: &Path, auto_dir: Option<&Path>) -> std::io::Result<()> { - let _guard = backup_lock(path)?; + let guard = backup_lock(path)?; + retain_one_backup_locked(&guard, auto_dir) +} + +pub(super) fn retain_one_backup_locked( + guard: &BackupGuard, + auto_dir: Option<&Path>, +) -> std::io::Result<()> { + let path = &guard.path; let mut snapshots = Vec::new(); for backup in backups_in(path, auto_dir)? { // Unpublished copies may be incomplete and must never supersede a recovery snapshot. @@ -429,15 +451,25 @@ pub(crate) fn remove_backups(path: &Path) -> std::io::Result<()> { first_error.map_or(Ok(()), Err) } +#[cfg(test)] fn backup(path: &Path) -> Result { backup_with_hook(path, |_| Ok(())) } +#[cfg(test)] fn backup_with_hook( path: &Path, pending_created: impl FnOnce(&Path) -> Result<(), UpgradeError>, ) -> Result { - let _guard = backup_lock(path)?; + let guard = backup_lock(path)?; + backup_with_hook_locked(&guard, pending_created) +} + +fn backup_with_hook_locked( + guard: &BackupGuard, + pending_created: impl FnOnce(&Path) -> Result<(), UpgradeError>, +) -> Result { + let path = &guard.path; let parent = path.parent().ok_or(UpgradeError::Unrecognized)?; let prefix = backup_prefix(path).ok_or(UpgradeError::Unrecognized)?; let file = tempfile::Builder::new() @@ -559,20 +591,42 @@ fn allowed_columns( } /// Translate only the exact pinned PR schema, preserving the original in a durable backup. +#[cfg(test)] pub(super) fn upgrade( path: &Path, target_path: &Path, validate: impl FnOnce(&Path) -> Result<(), UpgradeError>, ) -> Result, UpgradeError> { - upgrade_with_hook(path, target_path, validate, || Ok(())) + let guard = backup_lock(path)?; + upgrade_locked(&guard, target_path, validate) } +pub(super) fn upgrade_locked( + guard: &BackupGuard, + target_path: &Path, + validate: impl FnOnce(&Path) -> Result<(), UpgradeError>, +) -> Result, UpgradeError> { + upgrade_with_hook_locked(guard, target_path, validate, || Ok(())) +} + +#[cfg(test)] fn upgrade_with_hook( path: &Path, target_path: &Path, validate: impl FnOnce(&Path) -> Result<(), UpgradeError>, before_commit: impl FnOnce() -> Result<(), UpgradeError>, ) -> Result, UpgradeError> { + let guard = backup_lock(path)?; + upgrade_with_hook_locked(&guard, target_path, validate, before_commit) +} + +fn upgrade_with_hook_locked( + guard: &BackupGuard, + target_path: &Path, + validate: impl FnOnce(&Path) -> Result<(), UpgradeError>, + before_commit: impl FnOnce() -> Result<(), UpgradeError>, +) -> Result, UpgradeError> { + let path = &guard.path; let old = old_reference()?; let reference = target_reference()?; let mut source = Connection::open_with_flags( @@ -678,7 +732,7 @@ fn upgrade_with_hook( validate(target_path)?; // The source transaction has written only temp tables, so a separate reader still // sees the committed original; earlier failures roll back and need no backup. - let backup = backup(path)?; + let backup = backup_with_hook_locked(guard, |_| Ok(()))?; let target = Connection::open_with_flags( target_path, OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NOFOLLOW, diff --git a/src/wallet_backend/platform_compatibility/engine/tests.rs b/src/wallet_backend/platform_compatibility/engine/tests.rs index 84430477e..c77e713fd 100644 --- a/src/wallet_backend/platform_compatibility/engine/tests.rs +++ b/src/wallet_backend/platform_compatibility/engine/tests.rs @@ -665,3 +665,16 @@ fn platform_compatibility_active_snapshot_survives_concurrent_cleanup() { assert!(!published.exists()); } } + +#[test] +fn platform_compatibility_one_guard_covers_retention_and_upgrade() { + let (_dir, path, target) = fixture(); + let before = snapshot(&path); + let guard = backup_lock(&path).unwrap(); + retain_one_backup_locked(&guard, None).unwrap(); + let backup = upgrade_locked(&guard, &target, |_| Ok(())) + .unwrap() + .unwrap(); + retain_one_backup_locked(&guard, None).unwrap(); + assert_eq!(snapshot(&backup), before); +} diff --git a/src/wallet_backend/platform_compatibility/mod.rs b/src/wallet_backend/platform_compatibility/mod.rs index 4176bb32c..25286673f 100644 --- a/src/wallet_backend/platform_compatibility/mod.rs +++ b/src/wallet_backend/platform_compatibility/mod.rs @@ -10,31 +10,39 @@ use platform_wallet_storage::{SqlitePersister, SqlitePersisterConfig, WalletStor use crate::backend_task::error::TaskError; pub(crate) fn open(config: SqlitePersisterConfig) -> Result { + let guard = engine::backup_lock(&config.path).map_err(lock_error)?; let prune = || { - engine::retain_one_backup(&config.path, config.auto_backup_dir.as_deref()) - .map_err(|source| TaskError::FileSystem { source }) + engine::retain_one_backup_locked(&guard, config.auto_backup_dir.as_deref()) + .map_err(lock_error) }; // Check retention before another attempt can create a snapshot, including after a restart. prune()?; - let result = open_inner(&config); + let result = open_inner(&config, &guard); prune()?; result } -fn open_inner(config: &SqlitePersisterConfig) -> Result { +fn open_inner( + config: &SqlitePersisterConfig, + guard: &engine::BackupGuard, +) -> Result { let original = match SqlitePersister::open(config.clone()) { Ok(persister) => return Ok(persister), Err(error @ WalletStorageError::Migration(_)) => error, Err(error) => return Err(TaskError::from_wallet_storage_open_error(error)), }; - let result = upgrade(config).map_err(|source| TaskError::PlatformDatabaseUpgrade { source })?; + let result = + upgrade(config, guard).map_err(|source| TaskError::PlatformDatabaseUpgrade { source })?; if !result { return Err(TaskError::from_wallet_storage_open_error(original)); } SqlitePersister::open(config.clone()).map_err(TaskError::from_wallet_storage_open_error) } -fn upgrade(config: &SqlitePersisterConfig) -> Result { +fn upgrade( + config: &SqlitePersisterConfig, + guard: &engine::BackupGuard, +) -> Result { let parent = config.path.parent().ok_or(UpgradeError::Unrecognized)?; let mut builder = tempfile::Builder::new(); builder.prefix("platform-upgrade-"); @@ -46,7 +54,7 @@ fn upgrade(config: &SqlitePersisterConfig) -> Result { let stage_dir = builder.tempdir_in(parent)?; let stage_path = stage_dir.path().join("wallet.sqlite"); drop(SqlitePersister::open(SqlitePersisterConfig::new(&stage_path)).map_err(validation_error)?); - let backup = engine::upgrade(&config.path, &stage_path, |path| { + let backup = engine::upgrade_locked(guard, &stage_path, |path| { let staged = SqlitePersister::open(SqlitePersisterConfig::new(path)).map_err(validation_error)?; staged.load().map_err(validation_error)?; @@ -61,6 +69,14 @@ fn upgrade(config: &SqlitePersisterConfig) -> Result { } } +fn lock_error(source: std::io::Error) -> TaskError { + if source.kind() == std::io::ErrorKind::WouldBlock { + TaskError::WalletStorageNotReady + } else { + TaskError::FileSystem { source } + } +} + fn validation_error(error: impl std::error::Error + Send + Sync + 'static) -> UpgradeError { use platform_wallet::changeset::PersistenceError; use rusqlite::ErrorCode; @@ -155,6 +171,20 @@ mod tests { } } + #[test] + fn platform_compatibility_open_reports_lifecycle_lock_as_retryable() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("wallet.sqlite"); + rusqlite::Connection::open(&path).unwrap(); + let _guard = engine::backup_lock(&path).unwrap(); + let error = match open(SqlitePersisterConfig::new(&path)) { + Ok(_) => panic!("open unexpectedly acquired the held lifecycle lock"), + Err(error) => error, + }; + assert!(matches!(error, TaskError::WalletStorageNotReady)); + assert!(error.to_string().contains("try again")); + } + #[test] fn platform_compatibility_normal_open_upgrades_old_profile() { let dir = tempfile::tempdir().unwrap(); From dcb6d8d847b97e9f144aadc23172db72c8cdf2a5 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Wed, 23 Sep 2026 08:46:21 +0000 Subject: [PATCH 06/30] fix(wallet): keep resource-exhausted migrations retryable A wallet-storage migration that fails on SQLite DiskFull, SystemIoFailure or OutOfMemory is recoverable by freeing the resource and retrying, but it was classified as terminal WalletDataIncompatible, showing move-the- databases-aside guidance and stopping the cold-boot retry bridge. Route those causes to the generic retryable WalletStorage variant; access failures (PermissionDenied, ReadOnly) keep their existing classification. Co-Authored-By: Claude Opus 5.5 --- src/backend_task/error.rs | 106 +++++++++++++++++++++++++++++++++++--- 1 file changed, 98 insertions(+), 8 deletions(-) diff --git a/src/backend_task/error.rs b/src/backend_task/error.rs index b53664514..daa5de94c 100644 --- a/src/backend_task/error.rs +++ b/src/backend_task/error.rs @@ -2870,8 +2870,9 @@ impl TaskError { /// - A migration blocked by another SQLite writer is surfaced as /// [`Self::WalletStorageInUse`] so the user can close the competing /// process and retry instead of following incompatible-data recovery. - /// - Every other storage failure keeps the generic disk/IO copy via - /// [`Self::WalletStorage`]. + /// - A migration that failed on a recoverable resource (disk full, OS I/O + /// failure, out of memory) and every other storage failure keep the + /// generic, retryable disk/IO copy via [`Self::WalletStorage`]. /// /// Discrimination is on the typed upstream variant /// (`WalletStorageError::SchemaVersionUnsupported` / @@ -2893,6 +2894,11 @@ impl TaskError { { Self::WalletStorageInUse { source: other } } + other @ platform_wallet_storage::WalletStorageError::Migration(_) + if Self::wallet_storage_error_is_resource_exhausted(&other) => + { + Self::WalletStorage { source: other } + } other @ platform_wallet_storage::WalletStorageError::Migration(_) => { Self::WalletDataIncompatible { source: other } } @@ -2902,18 +2908,45 @@ impl TaskError { fn wallet_storage_error_is_in_use( source: &platform_wallet_storage::WalletStorageError, + ) -> bool { + Self::wallet_storage_sqlite_cause(source, |code| { + matches!( + code, + rusqlite::ErrorCode::DatabaseBusy | rusqlite::ErrorCode::DatabaseLocked + ) + }) + } + + /// A migration that ran out of disk space, hit an OS I/O failure or ran + /// out of memory is recoverable by freeing the resource and retrying, so + /// it must not get the terminal incompatible-data guidance. Access + /// failures (`PermissionDenied`, `ReadOnly`) are deliberately excluded: + /// retrying never fixes them. + fn wallet_storage_error_is_resource_exhausted( + source: &platform_wallet_storage::WalletStorageError, + ) -> bool { + Self::wallet_storage_sqlite_cause(source, |code| { + matches!( + code, + rusqlite::ErrorCode::DiskFull + | rusqlite::ErrorCode::SystemIoFailure + | rusqlite::ErrorCode::OutOfMemory + ) + }) + } + + /// Whether any `rusqlite::Error` in the typed source chain carries a + /// SQLite code accepted by `is_match`. + fn wallet_storage_sqlite_cause( + source: &platform_wallet_storage::WalletStorageError, + is_match: impl Fn(rusqlite::ErrorCode) -> bool, ) -> bool { let mut cause: Option<&(dyn std::error::Error + 'static)> = Some(source); while let Some(error) = cause { if error .downcast_ref::() .and_then(rusqlite::Error::sqlite_error_code) - .is_some_and(|code| { - matches!( - code, - rusqlite::ErrorCode::DatabaseBusy | rusqlite::ErrorCode::DatabaseLocked - ) - }) + .is_some_and(&is_match) { return true; } @@ -5899,6 +5932,63 @@ mod tests { assert!(std::error::Error::source(&err).is_some()); } + /// Wraps a SQLite failure with `code` in a refinery migration error the + /// same way refinery's rusqlite driver does, so the source chain matches + /// what `SqlitePersister::open` returns for a migration that ran out of a + /// resource. + fn resource_migration_error(code: std::ffi::c_int) -> refinery::Error { + use refinery::error::WrapMigrationError; + + Err::<(), _>(rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error::new(code), + None, + )) + .migration_err("error applying migration", None) + .expect_err("wrapped SQLite failure") + } + + /// Running out of disk space, an OS I/O failure or memory exhaustion + /// during a migration is recoverable by freeing the resource and + /// retrying. It must keep the generic, non-terminal storage copy instead + /// of the terminal move-the-databases-aside guidance. + #[test] + fn resource_migration_errors_map_to_retryable_wallet_storage() { + for code in [ + rusqlite::ffi::SQLITE_FULL, + rusqlite::ffi::SQLITE_IOERR, + rusqlite::ffi::SQLITE_NOMEM, + ] { + let upstream = platform_wallet_storage::WalletStorageError::Migration( + resource_migration_error(code), + ); + let err = TaskError::from_wallet_storage_open_error(upstream); + assert!( + matches!(err, TaskError::WalletStorage { .. }), + "Expected WalletStorage for SQLite code {code}, got: {err:?}" + ); + assert!( + !crate::backend_task::is_terminal_storage_open_error(&err), + "SQLite code {code} must stay retryable" + ); + } + } + + /// Access failures are not fixed by freeing space or retrying, so they + /// stay out of the retryable resource class. + #[test] + fn access_migration_errors_do_not_map_to_retryable_wallet_storage() { + for code in [rusqlite::ffi::SQLITE_PERM, rusqlite::ffi::SQLITE_READONLY] { + let upstream = platform_wallet_storage::WalletStorageError::Migration( + resource_migration_error(code), + ); + let err = TaskError::from_wallet_storage_open_error(upstream); + assert!( + !matches!(err, TaskError::WalletStorage { .. }), + "SQLite code {code} must not be classified as a resource failure, got: {err:?}" + ); + } + } + /// A divergent migration history (database written under an /// incompatible storage layout) maps to the dedicated /// `WalletDataIncompatible` variant. Its `Display` must never lead the user From 14f08fbb1bbfcefa89008d91c6c28dc28007f363 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Wed, 23 Sep 2026 08:46:21 +0000 Subject: [PATCH 07/30] fix(wallet): remove valid upgrade backups despite a rejected candidate remove_backups aborted on the first backup candidate that failed validation (directory, symlink, hard link, live-database alias), leaving every valid snapshot of the removed wallet on disk. Deletion now scans tolerantly, removes all valid snapshots and then reports the first rejection. Retention and publication keep the strict fail-fast scan. Co-Authored-By: Claude Opus 5.5 --- .../platform_compatibility/engine.rs | 52 +++++++++++++++---- .../platform_compatibility/engine/tests.rs | 34 ++++++++++++ 2 files changed, 75 insertions(+), 11 deletions(-) diff --git a/src/wallet_backend/platform_compatibility/engine.rs b/src/wallet_backend/platform_compatibility/engine.rs index 31d6ba521..154b380a8 100644 --- a/src/wallet_backend/platform_compatibility/engine.rs +++ b/src/wallet_backend/platform_compatibility/engine.rs @@ -249,17 +249,39 @@ fn backup_prefix(path: &Path) -> Option { /// Retained upgrade backups of the database at `path`. fn backups(path: &Path) -> std::io::Result> { - backups_in( - path, - Some(&platform_wallet_storage::default_auto_backup_dir(path)), - ) + backups_in(path, Some(&default_auto_dir(path))) +} + +fn default_auto_dir(path: &Path) -> PathBuf { + platform_wallet_storage::default_auto_backup_dir(path) } +/// Backup candidates that passed validation, plus the first rejected one. +/// +/// Directory-level failures still abort the scan; a single rejected +/// candidate does not, so deletion can remove every valid snapshot before +/// reporting the rejection. +struct BackupScan { + found: Vec, + first_rejection: Option, +} + +/// Strict scan: any rejected candidate fails the whole scan. Retention and +/// publication rely on this to never proceed past an unexpected file. fn backups_in(path: &Path, auto_dir: Option<&Path>) -> std::io::Result> { + let scan = scan_backups_in(path, auto_dir)?; + scan.first_rejection.map_or(Ok(scan.found), Err) +} + +fn scan_backups_in(path: &Path, auto_dir: Option<&Path>) -> std::io::Result { + let mut found = Vec::new(); + let mut first_rejection = None; let (Some(parent), Some(prefix)) = (path.parent(), backup_prefix(path)) else { - return Ok(Vec::new()); + return Ok(BackupScan { + found, + first_rejection, + }); }; - let mut found = Vec::new(); for directory in [Some(parent), auto_dir].into_iter().flatten() { let metadata = match std::fs::symlink_metadata(directory) { Ok(metadata) => metadata, @@ -288,14 +310,21 @@ fn backups_in(path: &Path, auto_dir: Option<&Path>) -> std::io::Result found.push(entry.path()), + Err(error) => { + first_rejection.get_or_insert(error); + } + } } } } found.sort(); found.dedup(); - Ok(found) + Ok(BackupScan { + found, + first_rejection, + }) } fn upstream_backup_name(path: &Path, name: &str) -> bool { @@ -442,8 +471,9 @@ pub(super) fn retain_one_backup_locked( /// Backups never contain vault secrets. Attempts every file and returns the first failure. pub(crate) fn remove_backups(path: &Path) -> std::io::Result<()> { let _guard = backup_lock(path)?; - let mut first_error = None; - for backup in backups(path)? { + let scan = scan_backups_in(path, Some(&default_auto_dir(path)))?; + let mut first_error = scan.first_rejection; + for backup in scan.found { if let Err(error) = remove_backup(&backup, path) { first_error.get_or_insert(error); } diff --git a/src/wallet_backend/platform_compatibility/engine/tests.rs b/src/wallet_backend/platform_compatibility/engine/tests.rs index c77e713fd..ec5c6256b 100644 --- a/src/wallet_backend/platform_compatibility/engine/tests.rs +++ b/src/wallet_backend/platform_compatibility/engine/tests.rs @@ -297,6 +297,40 @@ fn platform_compatibility_removes_only_matching_upstream_backups() { assert!(kept.exists()); } +/// One invalid candidate must not shield the valid snapshots from deletion: +/// every valid backup is removed and the rejection is still reported. +#[test] +fn platform_compatibility_invalid_candidate_does_not_block_removal_of_valid_backups() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("det-testnet.sqlite"); + let blocked = dir + .path() + .join("det-testnet.sqlite.platform-67d4ef3-backup-a0.pending"); + std::fs::create_dir(&blocked).unwrap(); + let valid = [ + dir.path() + .join("det-testnet.sqlite.platform-67d4ef3-backup-b1.sqlite"), + dir.path() + .join("det-testnet.sqlite.platform-67d4ef3-backup-c2.pending"), + ]; + for backup in &valid { + std::fs::write(backup, b"backup").unwrap(); + } + let auto = dir.path().join("backups/auto"); + std::fs::create_dir_all(&auto).unwrap(); + let upstream = auto.join("pre-migration-det-testnet-1-to-2-20260915T120000Z.db"); + std::fs::write(&upstream, b"backup").unwrap(); + + assert!(remove_backups(&path).is_err()); + for backup in valid.iter().chain([&upstream]) { + assert!(!backup.exists(), "{} must be removed", backup.display()); + } + assert!( + blocked.is_dir(), + "the rejected candidate must be left alone" + ); +} + #[test] fn platform_compatibility_prune_failure_preserves_original() { let (dir, path, target) = fixture(); From f9081f33c7f70962d3b96142776adf672a2439cf Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Thu, 24 Sep 2026 11:29:22 +0000 Subject: [PATCH 08/30] fix(wallet): sync backup directories before reporting removal done remove_backups returned Ok after plain unlinks, so identity cleanup could retire its retry manifest while a backup deletion was still volatile and could reappear after a power loss. Sync every directory a backup was removed from and fail the removal when a sync fails, keeping the manifest for the next retry. Co-Authored-By: Claude Opus 5.5 --- .../platform_compatibility/engine.rs | 37 +++++++++++++++- .../platform_compatibility/engine/tests.rs | 44 +++++++++++++++++++ 2 files changed, 80 insertions(+), 1 deletion(-) diff --git a/src/wallet_backend/platform_compatibility/engine.rs b/src/wallet_backend/platform_compatibility/engine.rs index 154b380a8..7114dbadf 100644 --- a/src/wallet_backend/platform_compatibility/engine.rs +++ b/src/wallet_backend/platform_compatibility/engine.rs @@ -469,18 +469,53 @@ pub(super) fn retain_one_backup_locked( /// Delete every retained upgrade backup of the database at `path`. /// /// Backups never contain vault secrets. Attempts every file and returns the first failure. +/// `Ok` means the deletions are durable: every directory an entry was removed +/// from is synced, so callers may retire their retry state afterwards. pub(crate) fn remove_backups(path: &Path) -> std::io::Result<()> { + remove_backups_with_sync(path, sync_directory) +} + +fn remove_backups_with_sync( + path: &Path, + mut sync_dir: impl FnMut(&Path) -> std::io::Result<()>, +) -> std::io::Result<()> { let _guard = backup_lock(path)?; let scan = scan_backups_in(path, Some(&default_auto_dir(path)))?; let mut first_error = scan.first_rejection; + let mut touched = std::collections::BTreeSet::new(); for backup in scan.found { - if let Err(error) = remove_backup(&backup, path) { + match remove_backup(&backup, path) { + Ok(()) => { + if let Some(parent) = backup.parent() { + touched.insert(parent.to_owned()); + } + } + Err(error) => { + first_error.get_or_insert(error); + } + } + } + // One sync per directory after all unlinks, still under the guard. + for directory in touched { + if let Err(error) = sync_dir(&directory) { first_error.get_or_insert(error); } } first_error.map_or(Ok(()), Err) } +/// Persist directory-entry changes (unlinks) in `directory`. +/// +/// Windows cannot open a directory as a `File` without extra flags and NTFS +/// journals metadata changes, so this is a no-op there. +fn sync_directory(directory: &Path) -> std::io::Result<()> { + #[cfg(unix)] + std::fs::File::open(directory)?.sync_all()?; + #[cfg(not(unix))] + let _ = directory; + Ok(()) +} + #[cfg(test)] fn backup(path: &Path) -> Result { backup_with_hook(path, |_| Ok(())) diff --git a/src/wallet_backend/platform_compatibility/engine/tests.rs b/src/wallet_backend/platform_compatibility/engine/tests.rs index ec5c6256b..959444823 100644 --- a/src/wallet_backend/platform_compatibility/engine/tests.rs +++ b/src/wallet_backend/platform_compatibility/engine/tests.rs @@ -297,6 +297,50 @@ fn platform_compatibility_removes_only_matching_upstream_backups() { assert!(kept.exists()); } +/// An unlink is only durable once its directory is synced: removal must sync +/// every directory it deleted from, and a failed sync must fail the removal so +/// callers keep their retry state (the identity-cleanup manifest) instead of +/// retiring it while the deletion could still be lost on power failure. +#[test] +fn platform_compatibility_backup_removal_fails_when_directory_sync_fails() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("det-testnet.sqlite"); + let auto = dir.path().join("backups/auto"); + std::fs::create_dir_all(&auto).unwrap(); + let sibling = dir + .path() + .join("det-testnet.sqlite.platform-67d4ef3-backup-a1.sqlite"); + let upstream = auto.join("pre-migration-det-testnet-1-to-2-20260915T120000Z.db"); + for backup in [&sibling, &upstream] { + std::fs::write(backup, b"backup").unwrap(); + } + + let mut synced = Vec::new(); + let error = remove_backups_with_sync(&path, |directory| { + synced.push(directory.to_owned()); + Err(std::io::Error::other("injected directory sync failure")) + }) + .unwrap_err(); + + assert_eq!(error.kind(), std::io::ErrorKind::Other); + assert!(!sibling.exists()); + assert!(!upstream.exists()); + synced.sort(); + let mut expected = vec![dir.path().to_owned(), auto]; + expected.sort(); + assert_eq!(synced, expected, "every touched directory must be synced"); +} + +/// Nothing deleted means nothing to make durable. +#[test] +fn platform_compatibility_backup_removal_skips_sync_without_deletions() { + let dir = tempfile::tempdir().unwrap(); + remove_backups_with_sync(&dir.path().join("det-testnet.sqlite"), |_| { + panic!("no directory changed, so none may be synced") + }) + .unwrap(); +} + /// One invalid candidate must not shield the valid snapshots from deletion: /// every valid backup is removed and the rejection is still reported. #[test] From 9624d011f1ff78a385443c9a1c6d119a53eac9f8 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Thu, 24 Sep 2026 13:17:08 +0000 Subject: [PATCH 09/30] fix(wallet): keep successful wallet open when post-open backup retention fails Post-open backup retention is best-effort housekeeping. A failure there (stray entry, hard link, transient scan error) discarded an already-open persister, aborting app boot or deferring the wallet backend. Warn-log it instead; the pre-open retention check still runs strictly on the next open. Co-Authored-By: Claude Opus 5.5 --- .../platform_compatibility/mod.rs | 62 +++++++++++++++++-- 1 file changed, 56 insertions(+), 6 deletions(-) diff --git a/src/wallet_backend/platform_compatibility/mod.rs b/src/wallet_backend/platform_compatibility/mod.rs index bb0d76489..cd74d93f8 100644 --- a/src/wallet_backend/platform_compatibility/mod.rs +++ b/src/wallet_backend/platform_compatibility/mod.rs @@ -10,15 +10,26 @@ use platform_wallet_storage::{SqlitePersister, SqlitePersisterConfig, WalletStor use crate::backend_task::error::TaskError; pub(crate) fn open(config: SqlitePersisterConfig) -> Result { + open_with_retention(config, engine::retain_one_backup_locked) +} + +fn open_with_retention( + config: SqlitePersisterConfig, + mut retain: impl FnMut(&engine::BackupGuard, Option<&std::path::Path>) -> std::io::Result<()>, +) -> Result { let guard = engine::backup_lock(&config.path).map_err(lock_error)?; - let prune = || { - engine::retain_one_backup_locked(&guard, config.auto_backup_dir.as_deref()) - .map_err(lock_error) - }; + let auto_dir = config.auto_backup_dir.as_deref(); // Check retention before another attempt can create a snapshot, including after a restart. - prune()?; + retain(&guard, auto_dir).map_err(lock_error)?; let result = open_inner(&config, &guard); - prune()?; + // Post-open retention is best-effort housekeeping: it must neither discard a successful + // open nor mask the open's own error. The next open retries it before any new snapshot. + if let Err(error) = retain(&guard, auto_dir) { + tracing::warn!( + ?error, + "Upgrade backup retention failed after opening the wallet database" + ); + } result } @@ -174,6 +185,45 @@ mod tests { } } + #[test] + fn platform_compatibility_post_open_retention_failure_keeps_open_result() { + let dir = tempfile::tempdir().unwrap(); + crate::app_dir::ensure_data_dir_exists(dir.path()).unwrap(); + let path = dir.path().join("wallet.sqlite"); + let mut calls = 0; + let persister = open_with_retention(SqlitePersisterConfig::new(&path), |_, _| { + calls += 1; + if calls == 1 { + Ok(()) + } else { + Err(std::io::Error::other("stray entry rejected by backup scan")) + } + }) + .expect("a post-open retention failure must not discard a successful open"); + assert_eq!(calls, 2); + persister.load().unwrap(); + + let bad = dir.path().join("corrupt.sqlite"); + std::fs::write(&bad, b"not a sqlite database, just candy wrappers").unwrap(); + let mut calls = 0; + let error = match open_with_retention(SqlitePersisterConfig::new(&bad), |_, _| { + calls += 1; + if calls == 1 { + Ok(()) + } else { + Err(std::io::Error::other("retention failure")) + } + }) { + Ok(_) => panic!("open unexpectedly succeeded on a corrupt database"), + Err(error) => error, + }; + assert_eq!(calls, 2); + assert!( + !matches!(&error, TaskError::FileSystem { source } if source.to_string() == "retention failure"), + "the open error must not be masked by retention: {error:?}" + ); + } + #[test] fn platform_compatibility_open_reports_lifecycle_lock_as_retryable() { let dir = tempfile::tempdir().unwrap(); From 8d2d9052d27fdbd6cfc97c05cdf143b8c2ce5175 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Fri, 25 Sep 2026 08:31:56 +0000 Subject: [PATCH 10/30] fix(wallet): keep current data openable and never drop the last upgrade snapshot - A pre-open retention failure no longer blocks a database that needs no snapshot: a probe open with auto-backups disabled proves it is current (upstream refuses pending migrations before mutating), then it is reopened with the full config. Migrations and bridge upgrades still fail closed on the strict retention error. - Publish the replacement bridge snapshot (and sync its directory) before pruning superseded ones, so a failed publication keeps the previous recovery snapshot. The strict scan still runs before publication. - Backup removal syncs every existing backup directory on each call, so a retry settles an earlier failed directory sync before callers retire their cleanup manifest. - Identity removal wipes upgrade backups only for a listed identity or a resumed removal; an unknown id leaves unrelated snapshots alone. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 + src/context/identity_db.rs | 50 ++++++- .../platform_compatibility/engine.rs | 65 ++++++---- .../platform_compatibility/engine/tests.rs | 64 ++++++++- .../platform_compatibility/mod.rs | 122 +++++++++++++++++- 5 files changed, 275 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c8b0a2264..5a2f9b3e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,6 +51,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). Concurrent sessions now protect active snapshots from cleanup, and staged validation preserves retry options and guidance for temporary resource failures. One lock now covers the complete open, upgrade, backup, and retention cycle. + A backup cleanup problem no longer stops up-to-date wallet data from opening; + it blocks only an upgrade that would create a new backup. A new upgrade backup + replaces the previous one only after it is saved. Removing an identity that is + not stored on this device leaves upgrade backups in place. ### Added diff --git a/src/context/identity_db.rs b/src/context/identity_db.rs index e23e3fb91..c18fca56c 100644 --- a/src/context/identity_db.rs +++ b/src/context/identity_db.rs @@ -1717,8 +1717,21 @@ impl AppContext { // re-derive the delete set from. The manifest below is what survives // that: persisted before any mutation runs, retained across every // error, and cleared only once every key and owner sidecar is removed. + // Upgrade backups copy every identity, so they are wiped only for a removal that + // concerns stored data: a listed identity, or a retry of an interrupted removal. + // An unknown id must not delete unrelated wallets' recovery snapshots. + let resumes_removal = kv + .get::>( + DetScope::Global, + &vault_cleanup_pending_key(&id), + ) + .map_err(identity_err)? + .is_some(); let vault_keys = self.pending_vault_key_placements(&kv, &id)?; self.persist_vault_cleanup_manifest(&kv, &id, &vault_keys)?; + // Read the roster only after the manifest is durable, so an unreadable index + // still leaves the retry record behind. + let removes_stored_identity = resumes_removal || identity_is_listed(&kv, &id)?; // Before delisting, so a failure between the two leaves a marker for a // still-listed identity — inert, since discovery only consults it when // storage says the identity is absent. The reverse order would leave a @@ -1726,7 +1739,8 @@ impl AppContext { self.mark_identity_unloaded(&kv, &id)?; index_remove_identity(&kv, &id)?; purge_identity_scope(&kv, &id)?; - let sidecar_cleanup = self.finish_identity_removal_cleanup(&kv, &id, vault_keys)?; + let sidecar_cleanup = + self.finish_identity_removal_cleanup(&kv, &id, vault_keys, removes_stored_identity)?; // Mirror removal into the upstream unowned scope; wallet-owned identities are unaffected. if let Ok(backend) = self.wallet_backend() && let Err(error) = backend.remove_unowned_identity(identifier) @@ -1746,6 +1760,7 @@ impl AppContext { kv: &DetKv, id: &[u8; 32], vault_keys: impl IntoIterator, + remove_upgrade_backups: bool, ) -> std::result::Result { crate::wallet_backend::IdentityKeyView::new(&self.secret_store, *id) .delete_all(vault_keys)?; @@ -1758,7 +1773,7 @@ impl AppContext { let token_cleanup = super::contract_token_db::forget_identity_token_state(kv, &identifier); let mut sidecar_cleanup = sidecar_cleanup_outcome(dashpay_cleanup, token_cleanup, &identifier); - if let Err(error) = self.remove_upgrade_backups() { + if remove_upgrade_backups && let Err(error) = self.remove_upgrade_backups() { sidecar_cleanup = IdentitySidecarCleanup::Incomplete; tracing::warn!(identity_id = %identifier, ?error, "Upgrade backup removal remains pending after identity deletion"); } @@ -1920,7 +1935,8 @@ impl AppContext { let vault_keys = placements .into_iter() .map(|(target, key_id)| (target.into(), key_id)); - match self.finish_identity_removal_cleanup(&kv, &id, vault_keys) { + // A surviving manifest is a removal of stored data, so its backups go too. + match self.finish_identity_removal_cleanup(&kv, &id, vault_keys, true) { Ok(IdentitySidecarCleanup::Complete) => resumed += 1, Ok(IdentitySidecarCleanup::Incomplete) => {} Err(error) => tracing::warn!( @@ -5372,6 +5388,34 @@ mod tests { staged.ctx.wallet_backend().unwrap().shutdown().await; } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn removing_unknown_identity_keeps_upgrade_backups() { + let staged = stage_identity_with_vaulted_keys([0x33; 32], [0x44; 32]).await; + let backup = staged + .ctx + .data_dir() + .join("det-app.sqlite.platform-67d4ef3-backup-kept.sqlite"); + std::fs::write(&backup, b"backup").unwrap(); + let unknown = Identifier::from([0x77; 32]); + staged + .ctx + .delete_local_qualified_identity(&unknown) + .expect("removing an unknown identity is a no-op"); + assert!( + backup.exists(), + "an unknown identity must not delete other data's recovery snapshots" + ); + staged + .ctx + .delete_local_qualified_identity(&staged.id) + .unwrap(); + assert!( + !backup.exists(), + "removing a stored identity still deletes the snapshots that copy it" + ); + staged.ctx.wallet_backend().unwrap().shutdown().await; + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn removal_recovery_retains_manifest_until_backups_are_removed() { let staged = stage_identity_with_vaulted_keys([0x33; 32], [0x44; 32]).await; diff --git a/src/wallet_backend/platform_compatibility/engine.rs b/src/wallet_backend/platform_compatibility/engine.rs index 7114dbadf..b384f3a36 100644 --- a/src/wallet_backend/platform_compatibility/engine.rs +++ b/src/wallet_backend/platform_compatibility/engine.rs @@ -469,8 +469,9 @@ pub(super) fn retain_one_backup_locked( /// Delete every retained upgrade backup of the database at `path`. /// /// Backups never contain vault secrets. Attempts every file and returns the first failure. -/// `Ok` means the deletions are durable: every directory an entry was removed -/// from is synced, so callers may retire their retry state afterwards. +/// `Ok` means the deletions are durable: every existing backup directory is +/// synced on every call (including a retry that finds nothing left to delete), +/// so callers may retire their retry state afterwards. pub(crate) fn remove_backups(path: &Path) -> std::io::Result<()> { remove_backups_with_sync(path, sync_directory) } @@ -480,24 +481,33 @@ fn remove_backups_with_sync( mut sync_dir: impl FnMut(&Path) -> std::io::Result<()>, ) -> std::io::Result<()> { let _guard = backup_lock(path)?; - let scan = scan_backups_in(path, Some(&default_auto_dir(path)))?; + let auto_dir = default_auto_dir(path); + let scan = scan_backups_in(path, Some(&auto_dir))?; let mut first_error = scan.first_rejection; - let mut touched = std::collections::BTreeSet::new(); for backup in scan.found { - match remove_backup(&backup, path) { - Ok(()) => { - if let Some(parent) = backup.parent() { - touched.insert(parent.to_owned()); - } - } + if let Err(error) = remove_backup(&backup, path) { + first_error.get_or_insert(error); + } + } + // Sync every candidate directory, not just those unlinked from in this call: an + // earlier call may have unlinked successfully and then failed its sync, and a + // retry that finds nothing left to delete must still make that deletion durable. + for directory in [path.parent(), Some(auto_dir.as_path())] + .into_iter() + .flatten() + { + match std::fs::symlink_metadata(directory) { + // No directory means no entry was ever removed from it. + Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, Err(error) => { first_error.get_or_insert(error); + continue; } + // The scan already rejected a non-directory; never open it here. + Ok(metadata) if !metadata.is_dir() => continue, + Ok(_) => {} } - } - // One sync per directory after all unlinks, still under the guard. - for directory in touched { - if let Err(error) = sync_dir(&directory) { + if let Err(error) = sync_dir(directory) { first_error.get_or_insert(error); } } @@ -556,16 +566,27 @@ fn backup_with_hook_locked( verify(&dest)?; drop(dest); file.as_file().sync_all()?; - // A verified snapshot supersedes old backups only while the original is still untouched. - for old in backups(path)? { - if old != file.path() { - remove_backup(&old, path)?; + // Strict scan before publication: an unexpected candidate aborts before anything changes. + let pending = file.path().to_owned(); + let superseded: Vec = backups(path)? + .into_iter() + .filter(|old| *old != pending) + .collect(); + let kept = pending.with_extension("sqlite"); + file.persist_noclobber(&kept).map_err(|e| e.error)?; + sync_directory(parent)?; + // Prune only once the replacement is durably published, so a failed or interrupted + // publication never leaves zero recovery snapshots. The original is still untouched. + let mut touched = BTreeSet::new(); + for old in superseded { + remove_backup(&old, path)?; + if let Some(directory) = old.parent() { + touched.insert(directory.to_owned()); } } - let kept = file.path().with_extension("sqlite"); - file.persist_noclobber(&kept).map_err(|e| e.error)?; - #[cfg(unix)] - std::fs::File::open(parent)?.sync_all()?; + for directory in touched { + sync_directory(&directory)?; + } Ok(kept) } diff --git a/src/wallet_backend/platform_compatibility/engine/tests.rs b/src/wallet_backend/platform_compatibility/engine/tests.rs index 959444823..b86478d6c 100644 --- a/src/wallet_backend/platform_compatibility/engine/tests.rs +++ b/src/wallet_backend/platform_compatibility/engine/tests.rs @@ -331,14 +331,70 @@ fn platform_compatibility_backup_removal_fails_when_directory_sync_fails() { assert_eq!(synced, expected, "every touched directory must be synced"); } -/// Nothing deleted means nothing to make durable. +/// A retry that finds no backups left must still settle an earlier failed +/// directory sync; otherwise the caller retires its retry state while the +/// earlier unlink could still be lost on power failure. #[test] -fn platform_compatibility_backup_removal_skips_sync_without_deletions() { +fn platform_compatibility_backup_removal_retries_directory_sync_without_deletions() { let dir = tempfile::tempdir().unwrap(); - remove_backups_with_sync(&dir.path().join("det-testnet.sqlite"), |_| { - panic!("no directory changed, so none may be synced") + let path = dir.path().join("det-testnet.sqlite"); + let sibling = dir + .path() + .join("det-testnet.sqlite.platform-67d4ef3-backup-a1.sqlite"); + std::fs::write(&sibling, b"backup").unwrap(); + + remove_backups_with_sync(&path, |_| { + Err(std::io::Error::other("injected directory sync failure")) + }) + .unwrap_err(); + assert!(!sibling.exists()); + + let mut synced = Vec::new(); + remove_backups_with_sync(&path, |directory| { + synced.push(directory.to_owned()); + Ok(()) }) .unwrap(); + assert_eq!( + synced, + vec![dir.path().to_owned()], + "the retry must re-sync the directory whose earlier sync failed; \ + the missing auto-backup directory is skipped" + ); +} + +/// The previous recovery snapshot must survive until its replacement is +/// published: a failed publication must not leave zero snapshots behind. +#[test] +fn platform_compatibility_failed_publication_keeps_previous_snapshot() { + let (_dir, path, _target) = fixture(); + let previous = backup(&path).unwrap(); + let error = backup_with_hook(&path, |pending| { + // Occupy the name the replacement would be published under. + std::fs::write(pending.with_extension("sqlite"), b"collision")?; + Ok(()) + }) + .unwrap_err(); + assert!( + matches!(&error, UpgradeError::Io(e) if e.kind() == std::io::ErrorKind::AlreadyExists), + "{error:?}" + ); + assert!( + previous.exists(), + "the published snapshot must survive a failed replacement" + ); + assert_eq!(snapshot(&previous), snapshot(&path)); +} + +/// A successful publication still leaves exactly one snapshot. +#[test] +fn platform_compatibility_publication_prunes_superseded_snapshots() { + let (dir, path, _target) = fixture(); + let previous = backup(&path).unwrap(); + let replacement = backup(&path).unwrap(); + assert_ne!(previous, replacement); + assert!(!previous.exists()); + assert_eq!(backup_files(dir.path()), vec![replacement]); } /// One invalid candidate must not shield the valid snapshots from deletion: diff --git a/src/wallet_backend/platform_compatibility/mod.rs b/src/wallet_backend/platform_compatibility/mod.rs index cd74d93f8..44347fd41 100644 --- a/src/wallet_backend/platform_compatibility/mod.rs +++ b/src/wallet_backend/platform_compatibility/mod.rs @@ -20,7 +20,17 @@ fn open_with_retention( let guard = engine::backup_lock(&config.path).map_err(lock_error)?; let auto_dir = config.auto_backup_dir.as_deref(); // Check retention before another attempt can create a snapshot, including after a restart. - retain(&guard, auto_dir).map_err(lock_error)?; + // Retention stays strict only when a snapshot could follow: a database that is already + // current still opens, since failed housekeeping does not endanger it. + if let Err(retention) = retain(&guard, auto_dir) { + return open_current_only(&config).map_err(|probe| { + tracing::debug!( + error = ?probe, + "Wallet database is not openable without a snapshot; backup retention failure stands" + ); + lock_error(retention) + }); + } let result = open_inner(&config, &guard); // Post-open retention is best-effort housekeeping: it must neither discard a successful // open nor mask the open's own error. The next open retries it before any new snapshot. @@ -33,6 +43,26 @@ fn open_with_retention( result } +/// Open only when no snapshot can be taken: with automatic backups disabled, upstream +/// refuses any pending migration before touching the database, and a brand-new database +/// has nothing to back up. The pinned-profile upgrade is never attempted here. +/// +/// On success the probe is dropped and the database reopened with the caller's config — +/// the database is current, so that open creates no snapshot, while the persister keeps +/// its backup directory for later destructive operations (e.g. wallet deletion). +/// The caller holds the lifecycle guard, so no other DET open can migrate in between. +fn open_current_only(config: &SqlitePersisterConfig) -> Result { + let probe = SqlitePersister::open(config.clone().with_auto_backup_dir(None)) + .map_err(TaskError::from_wallet_storage_open_error)?; + drop(probe); + let persister = + SqlitePersister::open(config.clone()).map_err(TaskError::from_wallet_storage_open_error)?; + tracing::warn!( + "Upgrade backup retention failed; opened the current wallet database and will retry retention on the next open" + ); + Ok(persister) +} + fn open_inner( config: &SqlitePersisterConfig, guard: &engine::BackupGuard, @@ -224,6 +254,96 @@ mod tests { ); } + fn failing_retention( + calls: &mut usize, + ) -> impl FnMut(&engine::BackupGuard, Option<&std::path::Path>) -> std::io::Result<()> + '_ + { + move |_, _| { + *calls += 1; + Err(std::io::Error::other("stray entry rejected by backup scan")) + } + } + + #[test] + fn platform_compatibility_pre_open_retention_failure_still_opens_current_database() { + let dir = tempfile::tempdir().unwrap(); + crate::app_dir::ensure_data_dir_exists(dir.path()).unwrap(); + let path = dir.path().join("wallet.sqlite"); + let auto = platform_wallet_storage::default_auto_backup_dir(&path); + // First a brand-new database, then the same database once it is current. + for _ in 0..2 { + let mut calls = 0; + let persister = open_with_retention( + SqlitePersisterConfig::new(&path), + failing_retention(&mut calls), + ) + .expect("failed housekeeping must not block a database that needs no snapshot"); + persister.load().unwrap(); + drop(persister); + assert_eq!(calls, 1); + assert!( + !auto.exists() || std::fs::read_dir(&auto).unwrap().next().is_none(), + "no snapshot may be taken while retention is failing" + ); + } + } + + #[test] + fn platform_compatibility_pre_open_retention_failure_blocks_upgrade() { + let dir = tempfile::tempdir().unwrap(); + crate::app_dir::ensure_data_dir_exists(dir.path()).unwrap(); + let path = dir.path().join("wallet.sqlite"); + rusqlite::Connection::open(&path) + .unwrap() + .execute_batch(include_str!("fixtures/67d4ef3.sql")) + .unwrap(); + let schema = || { + rusqlite::Connection::open(&path) + .unwrap() + .prepare("SELECT type, name, sql FROM sqlite_master ORDER BY type, name") + .unwrap() + .query_map([], |r| { + Ok(format!( + "{}|{}|{:?}", + r.get::<_, String>(0)?, + r.get::<_, String>(1)?, + r.get::<_, Option>(2)? + )) + }) + .unwrap() + .collect::, _>>() + .unwrap() + }; + let before = schema(); + let mut calls = 0; + let error = match open_with_retention( + SqlitePersisterConfig::new(&path), + failing_retention(&mut calls), + ) { + Ok(_) => panic!("an upgrade must not run while backup retention is failing"), + Err(error) => error, + }; + assert_eq!(calls, 1); + assert!( + matches!(&error, TaskError::FileSystem { source } if source.to_string() == "stray entry rejected by backup scan"), + "the strict retention failure must be reported: {error:?}" + ); + assert_eq!( + schema(), + before, + "the original database must stay untouched" + ); + let auto = platform_wallet_storage::default_auto_backup_dir(&path); + assert!(!auto.exists() || std::fs::read_dir(&auto).unwrap().next().is_none()); + assert!( + std::fs::read_dir(dir.path()) + .unwrap() + .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) + .all(|name| !name.contains(".platform-67d4ef3-backup-")), + "no bridge snapshot may be published while retention is failing" + ); + } + #[test] fn platform_compatibility_open_reports_lifecycle_lock_as_retryable() { let dir = tempfile::tempdir().unwrap(); From 0d1c6a50c44a7c40666ebcd48d994147650b40fc Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Fri, 25 Sep 2026 08:34:25 +0000 Subject: [PATCH 11/30] test(wallet): prove the current-only probe leaves no files behind Co-Authored-By: Claude Opus 5.5 --- .../platform_compatibility/mod.rs | 29 +++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/src/wallet_backend/platform_compatibility/mod.rs b/src/wallet_backend/platform_compatibility/mod.rs index 44347fd41..126e520da 100644 --- a/src/wallet_backend/platform_compatibility/mod.rs +++ b/src/wallet_backend/platform_compatibility/mod.rs @@ -254,6 +254,15 @@ mod tests { ); } + fn dir_entries(dir: &std::path::Path) -> Vec { + let mut names: Vec = std::fs::read_dir(dir) + .unwrap() + .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) + .collect(); + names.sort(); + names + } + fn failing_retention( calls: &mut usize, ) -> impl FnMut(&engine::BackupGuard, Option<&std::path::Path>) -> std::io::Result<()> + '_ @@ -282,8 +291,17 @@ mod tests { drop(persister); assert_eq!(calls, 1); assert!( - !auto.exists() || std::fs::read_dir(&auto).unwrap().next().is_none(), - "no snapshot may be taken while retention is failing" + !auto.exists(), + "the probe and reopen must not create the backup directory or any snapshot" + ); + // Only the database itself and its lifecycle lock may exist; the fresh + // profile's file is treated as current by the full reopen. + assert_eq!( + dir_entries(dir.path()), + vec![ + "wallet.sqlite".to_owned(), + "wallet.sqlite.platform-upgrade.lock".to_owned() + ] ); } } @@ -315,6 +333,7 @@ mod tests { .unwrap() }; let before = schema(); + let entries_before = dir_entries(dir.path()); let mut calls = 0; let error = match open_with_retention( SqlitePersisterConfig::new(&path), @@ -333,6 +352,12 @@ mod tests { before, "the original database must stay untouched" ); + let mut entries_after = dir_entries(dir.path()); + entries_after.retain(|name| name != "wallet.sqlite.platform-upgrade.lock"); + assert_eq!( + entries_after, entries_before, + "a refused probe must leave no files or directories behind" + ); let auto = platform_wallet_storage::default_auto_backup_dir(&path); assert!(!auto.exists() || std::fs::read_dir(&auto).unwrap().next().is_none()); assert!( From e64ba3b9df4502d2dac8c4346c21e9b92709fa86 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Fri, 25 Sep 2026 10:56:03 +0000 Subject: [PATCH 12/30] fix(wallet): retain only a proven-usable upgrade snapshot Retention picked the survivor by mtime alone, so an empty or truncated file left by an interrupted copy under a final snapshot name could win and get an older complete snapshot deleted. Retention now keeps the newest snapshot that opens (immutable, read-only), passes PRAGMA quick_check and has a non-empty migration history; when none qualifies it deletes nothing. Clearing a network's data also records a legacy shielded-file removal failure instead of returning early, so upgrade backups are still removed and the clear is reported incomplete. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 6 +- Cargo.lock | 1 + Cargo.toml | 2 + src/context/wallet_lifecycle/spv.rs | 7 +- src/context/wallet_lifecycle/tests.rs | 37 ++++++ .../platform_compatibility/engine.rs | 63 +++++++++- .../platform_compatibility/engine/tests.rs | 108 ++++++++++++++++++ .../platform_compatibility/mod.rs | 6 +- 8 files changed, 225 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a2f9b3e2..38adea669 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,7 +54,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). A backup cleanup problem no longer stops up-to-date wallet data from opening; it blocks only an upgrade that would create a new backup. A new upgrade backup replaces the previous one only after it is saved. Removing an identity that is - not stored on this device leaves upgrade backups in place. + not stored on this device leaves upgrade backups in place. The retained backup + is always one that opens and passes an integrity check, so an unfinished copy + never replaces a complete one. Clearing a network's data still removes upgrade + backups when an old shielded file cannot be deleted, and reports the clear as + incomplete. ### Added diff --git a/Cargo.lock b/Cargo.lock index 00d31d4ed..1243cc6a6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2150,6 +2150,7 @@ dependencies = [ "tz-rs", "unicode-general-category", "unicode-segmentation", + "url", "urlencoding", "which 8.0.6", "winres", diff --git a/Cargo.toml b/Cargo.toml index 9a93d0c0a..0998112f0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -119,6 +119,8 @@ tz-rs = { version = "0.7.0" } tempfile = "3.20.0" arc-swap = "1" urlencoding = "2" +# Builds SQLite `file:` URIs (percent-encoded, cross-platform); already a transitive dependency. +url = "2.5" rmcp = { version = "3.3", optional = true, default-features = false } axum = { version = "0.8", default-features = false, features = ["http1", "tokio", "json"], optional = true } # Constant-time equality for `model::secret::Secret` (unconditional) and the diff --git a/src/context/wallet_lifecycle/spv.rs b/src/context/wallet_lifecycle/spv.rs index e8988ef17..cd6f82402 100644 --- a/src/context/wallet_lifecycle/spv.rs +++ b/src/context/wallet_lifecycle/spv.rs @@ -106,7 +106,12 @@ impl AppContext { // empties the per-network store) and unlink DET's two retired legacy // shielded files. The legacy-file unlinks are synchronous and scoped // strictly to THIS network's spv directory. - cleanup_legacy_shielded_files(backend.spv_storage_dir())?; + // A failure here must not skip the steps below: upgrade backups hold the + // wallet data being cleared, so record it and keep going. + if let Err(error) = cleanup_legacy_shielded_files(backend.spv_storage_dir()) { + tracing::warn!(?error, "Legacy shielded file removal failed during clear"); + failures.push(error); + } if let Err(error) = backend.clear_shielded().await { tracing::warn!(%error, "Shielded coordinator reset failed during clear"); diff --git a/src/context/wallet_lifecycle/tests.rs b/src/context/wallet_lifecycle/tests.rs index e1e73c7fd..f88045b82 100644 --- a/src/context/wallet_lifecycle/tests.rs +++ b/src/context/wallet_lifecycle/tests.rs @@ -2489,6 +2489,43 @@ async fn clear_network_database_reports_incomplete_when_shielded_clear_fails() { } } +/// A legacy shielded-file removal failure must not skip upgrade-backup removal: +/// the backups copy the wallet data being cleared, and the failure still makes +/// the clear incomplete. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn clear_network_database_removes_backups_when_legacy_shielded_cleanup_fails() { + let (ctx, sender, _tmp) = offline_testnet_context(); + ctx.ensure_wallet_backend(sender) + .await + .expect("ensure_wallet_backend should succeed offline"); + let backend = ctx.wallet_backend().expect("backend wired"); + // A directory at a legacy file path makes its unlink fail. + let blocker = backend.spv_storage_dir().join("det-shielded.sqlite"); + std::fs::create_dir_all(blocker.join("occupied")).unwrap(); + let backup = ctx + .data_dir() + .join("det-app.sqlite.platform-67d4ef3-backup-fixture.sqlite"); + std::fs::write(&backup, b"old wallet history").unwrap(); + + let result = ctx.clear_network_database().await; + + backend.shutdown().await; + assert!(!backup.exists(), "upgrade backups must still be removed"); + match result { + Err(TaskError::WalletDataClearIncomplete { + failed, + first_error, + }) => { + assert_eq!(failed, 1, "the legacy cleanup should be the only failure"); + assert!( + matches!(*first_error, TaskError::FileSystem { .. }), + "the aggregate must preserve the legacy cleanup error: {first_error:?}" + ); + } + other => panic!("legacy cleanup failure must make clear incomplete: {other:?}"), + } +} + /// Clear-all must fail before changing any state when the wallet backend is /// unavailable, because persisted secrets from an earlier run may still exist. #[tokio::test] diff --git a/src/wallet_backend/platform_compatibility/engine.rs b/src/wallet_backend/platform_compatibility/engine.rs index b384f3a36..d1f2ed55a 100644 --- a/src/wallet_backend/platform_compatibility/engine.rs +++ b/src/wallet_backend/platform_compatibility/engine.rs @@ -434,7 +434,7 @@ pub(super) fn backup_lock(path: &Path) -> std::io::Result { }) } -/// Retain the newest snapshot across both upgrade-backup formats. +/// Retain the newest usable snapshot across both upgrade-backup formats. #[cfg(test)] pub(super) fn retain_one_backup(path: &Path, auto_dir: Option<&Path>) -> std::io::Result<()> { let guard = backup_lock(path)?; @@ -459,13 +459,72 @@ pub(super) fn retain_one_backup_locked( } } snapshots.sort(); - snapshots.pop(); + // Keep the newest snapshot that proves usable. An interrupted copy (e.g. a killed + // bridge from an older build that wrote straight to the final name) can be the newest + // file; it must never supersede an older complete snapshot. + let Some(survivor) = snapshots + .iter() + .rposition(|(_, backup)| usable_snapshot(backup)) + else { + if !snapshots.is_empty() { + // Nothing is proven usable, so nothing is proven expendable either. + tracing::warn!( + count = snapshots.len(), + "No upgrade backup passed validation; keeping all of them" + ); + } + return Ok(()); + }; + snapshots.remove(survivor); for (_, backup) in snapshots { remove_backup(&backup, path)?; } Ok(()) } +/// Whether `backup` is a complete, readable wallet database snapshot. +/// +/// Opens the file as immutable and read-only, then requires a clean `PRAGMA quick_check` +/// plus a non-empty migration history — an empty or truncated file fails one of these. +/// Any error while checking counts as unusable: retention then keeps an older proven copy. +fn usable_snapshot(backup: &Path) -> bool { + let check = || -> Result> { + // `immutable=1`: snapshots copied from a WAL database carry a WAL header, and a plain + // read-only open would leave `-wal`/`-shm` sidecars next to them. The caller holds + // the backup lock, so no DET writer changes the file while it is checked. + let mut uri = url::Url::from_file_path(std::path::absolute(backup)?) + .map_err(|()| std::io::Error::from(std::io::ErrorKind::InvalidInput))?; + uri.set_query(Some("immutable=1")); + let conn = Connection::open_with_flags( + uri.as_str(), + OpenFlags::SQLITE_OPEN_READ_ONLY + | OpenFlags::SQLITE_OPEN_URI + | OpenFlags::SQLITE_OPEN_NOFOLLOW, + )?; + let mut rows = conn.prepare("PRAGMA quick_check")?; + let report = rows + .query_map([], |r| r.get::<_, String>(0))? + .collect::, _>>()?; + if report != ["ok"] { + return Ok(false); + } + let has_history = conn + .prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?1")? + .exists([HISTORY])?; + Ok(has_history + && conn + .prepare(&format!("SELECT 1 FROM {}", quote(HISTORY)))? + .exists([])?) + }; + match check() { + Ok(usable) => usable, + Err(error) => { + tracing::debug!(backup = %backup.display(), ?error, "Upgrade backup is not usable"); + false + } + } +} + /// Delete every retained upgrade backup of the database at `path`. /// /// Backups never contain vault secrets. Attempts every file and returns the first failure. diff --git a/src/wallet_backend/platform_compatibility/engine/tests.rs b/src/wallet_backend/platform_compatibility/engine/tests.rs index b86478d6c..c86387b31 100644 --- a/src/wallet_backend/platform_compatibility/engine/tests.rs +++ b/src/wallet_backend/platform_compatibility/engine/tests.rs @@ -812,3 +812,111 @@ fn platform_compatibility_one_guard_covers_retention_and_upgrade() { retain_one_backup_locked(&guard, None).unwrap(); assert_eq!(snapshot(&backup), before); } + +fn set_mtime(path: &Path, seconds_from_now: i64) { + let now = std::time::SystemTime::now(); + let offset = std::time::Duration::from_secs(seconds_from_now.unsigned_abs()); + let time = if seconds_from_now < 0 { + now - offset + } else { + now + offset + }; + std::fs::File::options() + .write(true) + .open(path) + .unwrap() + .set_modified(time) + .unwrap(); +} + +/// An interrupted copy left under a final snapshot name (empty or truncated) +/// must never replace an older, complete recovery snapshot — in either format. +#[test] +fn platform_compatibility_retention_keeps_valid_snapshot_over_newer_incomplete_one() { + let (dir, path, _target) = fixture(); + let before = snapshot(&path); + let full = std::fs::read(backup(&path).unwrap()).unwrap(); + let auto = dir.path().join("backups/auto"); + std::fs::create_dir_all(&auto).unwrap(); + let cases: [(&str, &[u8], PathBuf); 4] = [ + ( + "empty bridge", + b"", + dir.path() + .join("wallet.sqlite.platform-67d4ef3-backup-killed.sqlite"), + ), + ( + "truncated bridge", + &full[..full.len() / 2], + dir.path() + .join("wallet.sqlite.platform-67d4ef3-backup-killed.sqlite"), + ), + ( + "header-only bridge", + &full[..100], + dir.path() + .join("wallet.sqlite.platform-67d4ef3-backup-killed.sqlite"), + ), + ( + "empty upstream", + b"", + auto.join("pre-migration-wallet-1-to-2-20260915T120000Z.db"), + ), + ]; + for (case, bytes, incomplete) in cases { + let valid = backup_files(dir.path()) + .into_iter() + .find(|p| *p != incomplete) + .unwrap(); + set_mtime(&valid, -3600); + std::fs::write(&incomplete, bytes).unwrap(); + retain_one_backup(&path, Some(&auto)).unwrap(); + assert!(valid.exists(), "{case}: the valid snapshot must survive"); + assert_eq!(snapshot(&valid), before, "{case}"); + assert!( + !incomplete.exists(), + "{case}: the incomplete copy is pruned" + ); + } +} + +/// With no snapshot proven usable, retention deletes nothing. +#[test] +fn platform_compatibility_retention_keeps_all_when_none_is_usable() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("wallet.sqlite"); + let older = dir + .path() + .join("wallet.sqlite.platform-67d4ef3-backup-a1.sqlite"); + let newer = dir + .path() + .join("wallet.sqlite.platform-67d4ef3-backup-b2.sqlite"); + std::fs::write(&older, b"not a database").unwrap(); + std::fs::write(&newer, b"").unwrap(); + set_mtime(&older, -3600); + retain_one_backup(&path, None).unwrap(); + assert!(older.exists()); + assert!(newer.exists()); +} + +/// Validating a snapshot copied from a WAL-mode database opens it read-only +/// without leaving journal sidecars next to it. +#[test] +fn platform_compatibility_snapshot_validation_leaves_no_sidecars() { + let (dir, path, _target) = fixture(); + Connection::open(&path) + .unwrap() + .query_row("PRAGMA journal_mode=WAL", [], |_| Ok(())) + .unwrap(); + let kept = backup(&path).unwrap(); + let before: BTreeSet<_> = std::fs::read_dir(dir.path()) + .unwrap() + .map(|e| e.unwrap().file_name()) + .collect(); + assert!(usable_snapshot(&kept)); + let after: BTreeSet<_> = std::fs::read_dir(dir.path()) + .unwrap() + .map(|e| e.unwrap().file_name()) + .collect(); + assert_eq!(after, before); +} diff --git a/src/wallet_backend/platform_compatibility/mod.rs b/src/wallet_backend/platform_compatibility/mod.rs index 126e520da..001c0557e 100644 --- a/src/wallet_backend/platform_compatibility/mod.rs +++ b/src/wallet_backend/platform_compatibility/mod.rs @@ -200,7 +200,11 @@ mod tests { let old = auto.join(format!( "pre-migration-wallet-1-to-2-20260915T12000{attempt}Z.db" )); - std::fs::write(&old, b"snapshot from an earlier attempt").unwrap(); + // A complete snapshot: retention only prunes in favour of a proven-usable copy. + rusqlite::Connection::open(&old) + .unwrap() + .execute_batch(include_str!("fixtures/67d4ef3.sql")) + .unwrap(); assert!(open(SqlitePersisterConfig::new(&path)).is_err()); assert_eq!(std::fs::read_dir(&auto).unwrap().count(), 1); let retained = std::fs::read_dir(&auto) From 89e77f3d875a3725bdd4ab15f36d3f4338c72bcd Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Mon, 28 Sep 2026 08:16:29 +0000 Subject: [PATCH 13/30] fix(identity): persist upgrade-backup removal authorization across retries An unknown-id removal still leaves a vault-cleanup manifest behind, and a direct retry or the startup sweep treated that manifest as permission to delete every upgrade backup. Record the authorization in a dedicated Global marker, written only after the identity is confirmed listed and before it is delisted, and retire it as soon as the backups are removed. Co-Authored-By: Claude Opus 5.5 --- src/context/identity_db.rs | 124 +++++++++++++++++++++++++++++++------ 1 file changed, 105 insertions(+), 19 deletions(-) diff --git a/src/context/identity_db.rs b/src/context/identity_db.rs index 9f6f37815..196be7ac9 100644 --- a/src/context/identity_db.rs +++ b/src/context/identity_db.rs @@ -69,6 +69,14 @@ const TOP_UPS_KEY: &str = "det:top_ups:v1"; /// `det:vault_cleanup_pending:v1:`. const VAULT_CLEANUP_PENDING_PREFIX: &str = "det:vault_cleanup_pending:v1:"; +/// Per-identity marker authorizing upgrade-backup removal for an identity removal. +/// Written only once the identity is confirmed listed, before it is delisted, so a +/// retry or the startup sweep deletes backups only for a removal of stored data; +/// a vault-cleanup manifest alone (e.g. from an unknown id) never authorizes it. +/// Global-scoped for the same reason as the manifest. Key shape: +/// `det:upgrade_backup_cleanup_pending:v1:`. +const UPGRADE_BACKUP_CLEANUP_PENDING_PREFIX: &str = "det:upgrade_backup_cleanup_pending:v1:"; + /// Per-identity marker recording that the user deliberately unloaded an /// identity from this device. Global-scoped for the same reason the /// vault-cleanup manifest is: an `Identity`-scoped key is reaped by @@ -134,6 +142,13 @@ fn identity_unloaded_key(id: &[u8; 32]) -> String { ) } +fn upgrade_backup_cleanup_pending_key(id: &[u8; 32]) -> String { + format!( + "{UPGRADE_BACKUP_CLEANUP_PENDING_PREFIX}{}", + Identifier::from(*id).to_string(Encoding::Base58) + ) +} + fn vault_cleanup_pending_key(id: &[u8; 32]) -> String { format!( "{VAULT_CLEANUP_PENDING_PREFIX}{}", @@ -1629,21 +1644,22 @@ impl AppContext { // re-derive the delete set from. The manifest below is what survives // that: persisted before any mutation runs, retained across every // error, and cleared only once every key and owner sidecar is removed. - // Upgrade backups copy every identity, so they are wiped only for a removal that - // concerns stored data: a listed identity, or a retry of an interrupted removal. - // An unknown id must not delete unrelated wallets' recovery snapshots. - let resumes_removal = kv - .get::>( - DetScope::Global, - &vault_cleanup_pending_key(&id), - ) - .map_err(identity_err)? - .is_some(); let vault_keys = self.pending_vault_key_placements(&kv, &id)?; self.persist_vault_cleanup_manifest(&kv, &id, &vault_keys)?; // Read the roster only after the manifest is durable, so an unreadable index // still leaves the retry record behind. - let removes_stored_identity = resumes_removal || identity_is_listed(&kv, &id)?; + // Upgrade backups copy every identity, so they are wiped only for a removal of + // stored data. That authorization is persisted before delisting, so a retry or the + // startup sweep inherits it; the manifest alone never grants it, since an unknown id + // also leaves one behind and must not delete unrelated wallets' recovery snapshots. + if identity_is_listed(&kv, &id)? { + kv.put( + DetScope::Global, + &upgrade_backup_cleanup_pending_key(&id), + &true, + ) + .map_err(identity_err)?; + } // Before delisting, so a failure between the two leaves a marker for a // still-listed identity — inert, since discovery only consults it when // storage says the identity is absent. The reverse order would leave a @@ -1651,8 +1667,7 @@ impl AppContext { self.mark_identity_unloaded(&kv, &id)?; index_remove_identity(&kv, &id)?; purge_identity_scope(&kv, &id)?; - let sidecar_cleanup = - self.finish_identity_removal_cleanup(&kv, &id, vault_keys, removes_stored_identity)?; + let sidecar_cleanup = self.finish_identity_removal_cleanup(&kv, &id, vault_keys)?; // Mirror removal into the upstream unowned scope; wallet-owned identities are unaffected. if let Ok(backend) = self.wallet_backend() && let Err(error) = backend.remove_unowned_identity(identifier) @@ -1672,7 +1687,6 @@ impl AppContext { kv: &DetKv, id: &[u8; 32], vault_keys: impl IntoIterator, - remove_upgrade_backups: bool, ) -> std::result::Result { let identifier = Identifier::from(*id); // Both callers hold the record lock through inventory retirement. @@ -1690,9 +1704,25 @@ impl AppContext { let token_cleanup = super::contract_token_db::forget_identity_token_state(kv, &identifier); let mut sidecar_cleanup = sidecar_cleanup_outcome(dashpay_cleanup, token_cleanup, &identifier); - if remove_upgrade_backups && let Err(error) = self.remove_upgrade_backups() { - sidecar_cleanup = IdentitySidecarCleanup::Incomplete; - tracing::warn!(identity_id = %identifier, ?error, "Upgrade backup removal remains pending after identity deletion"); + let backup_marker = upgrade_backup_cleanup_pending_key(id); + let removes_upgrade_backups = kv + .get::(DetScope::Global, &backup_marker) + .map_err(identity_err)? + .is_some(); + if removes_upgrade_backups { + match self.remove_upgrade_backups() { + // Retired at once, so a later retry of unrelated sidecar cleanup cannot + // delete snapshots taken after this identity was already gone. + Ok(()) => { + if let Err(error) = kv.delete(DetScope::Global, &backup_marker) { + tracing::warn!(identity_id = %identifier, %error, "Upgrade backups removed but their cleanup marker remains"); + } + } + Err(error) => { + sidecar_cleanup = IdentitySidecarCleanup::Incomplete; + tracing::warn!(identity_id = %identifier, ?error, "Upgrade backup removal remains pending after identity deletion"); + } + } } if !sidecar_cleanup.is_incomplete() { clear_vault_cleanup_manifest(kv, id); @@ -1852,8 +1882,7 @@ impl AppContext { let vault_keys = placements .into_iter() .map(|(target, key_id)| (target.into(), key_id)); - // A surviving manifest is a removal of stored data, so its backups go too. - match self.finish_identity_removal_cleanup(&kv, &id, vault_keys, true) { + match self.finish_identity_removal_cleanup(&kv, &id, vault_keys) { Ok(IdentitySidecarCleanup::Complete) => resumed += 1, Ok(IdentitySidecarCleanup::Incomplete) => {} Err(error) => tracing::warn!( @@ -5426,6 +5455,63 @@ mod tests { staged.ctx.wallet_backend().unwrap().shutdown().await; } + /// An interrupted unknown-id removal leaves a manifest behind; neither the startup sweep + /// nor a direct retry may treat that manifest as permission to delete upgrade backups. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn interrupted_unknown_identity_removal_keeps_upgrade_backups() { + let staged = stage_identity_with_vaulted_keys([0x33; 32], [0x44; 32]).await; + let kv = staged.ctx.det_kv().unwrap(); + let backup = staged + .ctx + .data_dir() + .join("det-app.sqlite.platform-67d4ef3-backup-kept.sqlite"); + std::fs::write(&backup, b"backup").unwrap(); + let unknown = Identifier::from([0x77; 32]); + kv.put(DetScope::Global, "det:token_order:v1", &"invalid ordering") + .unwrap(); + assert!(matches!( + staged.ctx.delete_local_qualified_identity(&unknown), + Err(TaskError::IdentitySidecarCleanupIncomplete) + )); + assert!( + kv.get::>( + DetScope::Global, + &vault_cleanup_pending_key(&unknown.to_buffer()), + ) + .unwrap() + .is_some(), + "the interrupted removal leaves its manifest for a retry" + ); + assert!(matches!( + staged.ctx.delete_local_qualified_identity(&unknown), + Err(TaskError::IdentitySidecarCleanupIncomplete) + )); + assert!(backup.exists(), "a direct retry keeps unrelated backups"); + kv.delete(DetScope::Global, "det:token_order:v1").unwrap(); + staged.ctx.resume_pending_vault_cleanups(); + assert!( + kv.list(DetScope::Global, Some(VAULT_CLEANUP_PENDING_PREFIX)) + .unwrap() + .is_empty(), + "the sweep retires the repaired removal" + ); + assert!(backup.exists(), "the startup sweep keeps unrelated backups"); + + // A crash right after the manifest write leaves no backup-removal marker either. + kv.put( + DetScope::Global, + &vault_cleanup_pending_key(&unknown.to_buffer()), + &Vec::<(StoredPrivateKeyTarget, KeyID)>::new(), + ) + .unwrap(); + staged.ctx.resume_pending_vault_cleanups(); + assert!( + backup.exists(), + "a bare manifest never authorizes backup removal" + ); + staged.ctx.wallet_backend().unwrap().shutdown().await; + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn removal_recovery_retains_manifest_until_backups_are_removed() { let staged = stage_identity_with_vaulted_keys([0x33; 32], [0x44; 32]).await; From 820b5a6954aedf2c84972c583df2f070f9456648 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Mon, 28 Sep 2026 08:16:29 +0000 Subject: [PATCH 14/30] fix(wallet): validate upgrade snapshots beneath symlinked ancestors SQLITE_OPEN_NOFOLLOW rejects a symlink anywhere in the path, so snapshots under e.g. macOS /var/folders were never usable and retention kept every copy. Canonicalize the parent directory and keep the entry name, so a symlink at the snapshot itself is still refused. Co-Authored-By: Claude Opus 5.5 --- .../platform_compatibility/engine.rs | 10 ++++++++-- .../platform_compatibility/engine/tests.rs | 17 +++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/src/wallet_backend/platform_compatibility/engine.rs b/src/wallet_backend/platform_compatibility/engine.rs index d1f2ed55a..5e43f7bd9 100644 --- a/src/wallet_backend/platform_compatibility/engine.rs +++ b/src/wallet_backend/platform_compatibility/engine.rs @@ -492,8 +492,14 @@ fn usable_snapshot(backup: &Path) -> bool { // `immutable=1`: snapshots copied from a WAL database carry a WAL header, and a plain // read-only open would leave `-wal`/`-shm` sidecars next to them. The caller holds // the backup lock, so no DET writer changes the file while it is checked. - let mut uri = url::Url::from_file_path(std::path::absolute(backup)?) - .map_err(|()| std::io::Error::from(std::io::ErrorKind::InvalidInput))?; + // `SQLITE_OPEN_NOFOLLOW` rejects a symlink anywhere in the path, so resolve the + // parent directory (e.g. macOS `/var` -> `/private/var`) and keep the entry name: + // a symlink at the snapshot itself is still refused. + let invalid = || std::io::Error::from(std::io::ErrorKind::InvalidInput); + let absolute = std::path::absolute(backup)?; + let parent = absolute.parent().ok_or_else(invalid)?.canonicalize()?; + let name = absolute.file_name().ok_or_else(invalid)?; + let mut uri = url::Url::from_file_path(parent.join(name)).map_err(|()| invalid())?; uri.set_query(Some("immutable=1")); let conn = Connection::open_with_flags( uri.as_str(), diff --git a/src/wallet_backend/platform_compatibility/engine/tests.rs b/src/wallet_backend/platform_compatibility/engine/tests.rs index c86387b31..ca106b3ae 100644 --- a/src/wallet_backend/platform_compatibility/engine/tests.rs +++ b/src/wallet_backend/platform_compatibility/engine/tests.rs @@ -920,3 +920,20 @@ fn platform_compatibility_snapshot_validation_leaves_no_sidecars() { .collect(); assert_eq!(after, before); } + +/// A symlinked ancestor directory (e.g. macOS `/var` -> `/private/var`) does not make a +/// complete snapshot unusable, while a symlink at the snapshot entry itself still does. +#[cfg(unix)] +#[test] +fn platform_compatibility_snapshot_validation_follows_ancestor_symlinks() { + let (dir, path, _target) = fixture(); + let kept = backup(&path).unwrap(); + let links = tempfile::tempdir().unwrap(); + let alias = links.path().join("alias"); + std::os::unix::fs::symlink(dir.path(), &alias).unwrap(); + let name = kept.file_name().unwrap(); + assert!(usable_snapshot(&alias.join(name))); + let entry_link = links.path().join(name); + std::os::unix::fs::symlink(&kept, &entry_link).unwrap(); + assert!(!usable_snapshot(&entry_link)); +} From 430a1ae66ea65a693a8f5264b1df1764c9d0d441 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Tue, 29 Sep 2026 07:26:48 +0000 Subject: [PATCH 15/30] fix(identity): scope upgrade-backup removal to snapshots present at removal The backup-removal marker was a bare flag, and a failed retirement was only logged: cleanup still reported complete, the manifest was retired, and a later removal of the already-absent identity read the orphaned marker and deleted every current upgrade snapshot, including ones taken after the identity was gone. The marker (key bumped to v2) now records the backup file names that exist while the identity is still listed, and cleanup deletes only those. A failed retirement leaves the removal incomplete, keeping the manifest so the startup sweep retries it; repeating the scoped deletion can no longer reach newer snapshots. Co-Authored-By: Claude Opus 5.5 --- src/context/identity_db.rs | 94 ++++++++++++++++--- src/context/wallet_lifecycle/mod.rs | 1 + src/context/wallet_lifecycle/removal.rs | 55 ++++++++++- .../platform_compatibility/engine.rs | 65 +++++++++---- .../platform_compatibility/engine/tests.rs | 75 ++++++++++++--- .../platform_compatibility/mod.rs | 2 +- 6 files changed, 247 insertions(+), 45 deletions(-) diff --git a/src/context/identity_db.rs b/src/context/identity_db.rs index 196be7ac9..f9bf1971b 100644 --- a/src/context/identity_db.rs +++ b/src/context/identity_db.rs @@ -1,4 +1,5 @@ use super::AppContext; +use super::wallet_lifecycle::UpgradeBackupScope; use crate::backend_task::contested_names::ScheduledDPNSVote; use crate::backend_task::error::TaskError; use crate::model::identity_discovery::DiscoveryIntent; @@ -73,9 +74,12 @@ const VAULT_CLEANUP_PENDING_PREFIX: &str = "det:vault_cleanup_pending:v1:"; /// Written only once the identity is confirmed listed, before it is delisted, so a /// retry or the startup sweep deletes backups only for a removal of stored data; /// a vault-cleanup manifest alone (e.g. from an unknown id) never authorizes it. +/// The value is an [`UpgradeBackupScope`]: it names the snapshots that existed at that +/// point, so a marker that outlives its removal can never reach later snapshots. /// Global-scoped for the same reason as the manifest. Key shape: -/// `det:upgrade_backup_cleanup_pending:v1:`. -const UPGRADE_BACKUP_CLEANUP_PENDING_PREFIX: &str = "det:upgrade_backup_cleanup_pending:v1:"; +/// `det:upgrade_backup_cleanup_pending:v2:`; unscoped `v1` markers are +/// ignored, which keeps the backups they would have authorized. +const UPGRADE_BACKUP_CLEANUP_PENDING_PREFIX: &str = "det:upgrade_backup_cleanup_pending:v2:"; /// Per-identity marker recording that the user deliberately unloaded an /// identity from this device. Global-scoped for the same reason the @@ -1652,11 +1656,13 @@ impl AppContext { // stored data. That authorization is persisted before delisting, so a retry or the // startup sweep inherits it; the manifest alone never grants it, since an unknown id // also leaves one behind and must not delete unrelated wallets' recovery snapshots. + // It names the snapshots that exist now: only those can copy this identity. if identity_is_listed(&kv, &id)? { + let scope = self.upgrade_backup_scope()?; kv.put( DetScope::Global, &upgrade_backup_cleanup_pending_key(&id), - &true, + &scope, ) .map_err(identity_err)?; } @@ -1705,16 +1711,17 @@ impl AppContext { let mut sidecar_cleanup = sidecar_cleanup_outcome(dashpay_cleanup, token_cleanup, &identifier); let backup_marker = upgrade_backup_cleanup_pending_key(id); - let removes_upgrade_backups = kv - .get::(DetScope::Global, &backup_marker) - .map_err(identity_err)? - .is_some(); - if removes_upgrade_backups { - match self.remove_upgrade_backups() { - // Retired at once, so a later retry of unrelated sidecar cleanup cannot - // delete snapshots taken after this identity was already gone. + let backup_scope = kv + .get::(DetScope::Global, &backup_marker) + .map_err(identity_err)?; + if let Some(scope) = backup_scope { + // The scope never reaches snapshots taken after this identity was gone, so + // repeating this after a failed retirement below is harmless. + match self.remove_scoped_upgrade_backups(&scope) { Ok(()) => { + // Keep the manifest while the marker remains, so startup retries it. if let Err(error) = kv.delete(DetScope::Global, &backup_marker) { + sidecar_cleanup = IdentitySidecarCleanup::Incomplete; tracing::warn!(identity_id = %identifier, %error, "Upgrade backups removed but their cleanup marker remains"); } } @@ -5512,6 +5519,71 @@ mod tests { staged.ctx.wallet_backend().unwrap().shutdown().await; } + /// Retiring the backup-removal marker can fail after its snapshots are gone. That + /// removal must stay incomplete, and neither the startup sweep nor a later retry of + /// the absent identity may delete snapshots created after the identity was removed. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn failed_backup_marker_retirement_never_deletes_newer_snapshots() { + let staged = stage_identity_with_vaulted_keys([0x33; 32], [0x44; 32]).await; + let kv = staged.ctx.det_kv().unwrap(); + let data_dir = staged.ctx.data_dir().to_path_buf(); + let copied = data_dir.join("det-app.sqlite.platform-67d4ef3-backup-copied.sqlite"); + std::fs::write(&copied, b"backup").unwrap(); + let conn = rusqlite::Connection::open(data_dir.join("det-testnet.sqlite")).unwrap(); + conn.execute_batch(&format!( + "CREATE TRIGGER fail_marker_retirement BEFORE DELETE ON meta_global \ + WHEN OLD.key LIKE '{UPGRADE_BACKUP_CLEANUP_PENDING_PREFIX}%' \ + BEGIN SELECT RAISE(FAIL, 'injected marker retirement failure'); END;" + )) + .unwrap(); + + assert!(matches!( + staged.ctx.delete_local_qualified_identity(&staged.id), + Err(TaskError::IdentitySidecarCleanupIncomplete) + )); + assert!( + !copied.exists(), + "the snapshot copying the identity is removed" + ); + assert!( + !kv.list(DetScope::Global, Some(VAULT_CLEANUP_PENDING_PREFIX)) + .unwrap() + .is_empty(), + "an unretired marker keeps the removal discoverable at startup" + ); + + conn.execute_batch("DROP TRIGGER fail_marker_retirement;") + .unwrap(); + let newer = data_dir.join("det-app.sqlite.platform-67d4ef3-backup-newer.sqlite"); + std::fs::write(&newer, b"backup").unwrap(); + staged.ctx.resume_pending_vault_cleanups(); + assert!(newer.exists(), "the startup sweep keeps later snapshots"); + assert!( + kv.list( + DetScope::Global, + Some(UPGRADE_BACKUP_CLEANUP_PENDING_PREFIX) + ) + .unwrap() + .is_empty(), + "the startup sweep retires the marker" + ); + assert!( + kv.list(DetScope::Global, Some(VAULT_CLEANUP_PENDING_PREFIX)) + .unwrap() + .is_empty(), + "the startup sweep retires the repaired removal" + ); + staged + .ctx + .delete_local_qualified_identity(&staged.id) + .expect("removing an absent identity is a no-op"); + assert!( + newer.exists(), + "a retry of the absent identity keeps later snapshots" + ); + staged.ctx.wallet_backend().unwrap().shutdown().await; + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn removal_recovery_retains_manifest_until_backups_are_removed() { let staged = stage_identity_with_vaulted_keys([0x33; 32], [0x44; 32]).await; diff --git a/src/context/wallet_lifecycle/mod.rs b/src/context/wallet_lifecycle/mod.rs index 6380db287..7e47d750f 100644 --- a/src/context/wallet_lifecycle/mod.rs +++ b/src/context/wallet_lifecycle/mod.rs @@ -109,6 +109,7 @@ mod prepare; pub use prepare::PrepareGateGuard; mod registration; mod removal; +pub(crate) use removal::UpgradeBackupScope; mod spv; mod unlock; diff --git a/src/context/wallet_lifecycle/removal.rs b/src/context/wallet_lifecycle/removal.rs index 847587f7e..e84438366 100644 --- a/src/context/wallet_lifecycle/removal.rs +++ b/src/context/wallet_lifecycle/removal.rs @@ -89,10 +89,7 @@ impl AppContext { /// Filesystem cleanup is available even when the wallet backend could not be opened. pub(crate) fn remove_upgrade_backups(&self) -> Result<(), TaskError> { let mut first_error = None; - for database in [ - self.data_dir().join("det-app.sqlite"), - crate::wallet_backend::wallet_database_path(self.data_dir(), self.network), - ] { + for database in self.upgrade_backup_databases() { if let Err(source) = crate::wallet_backend::platform_compatibility::remove_backups(&database) { @@ -101,6 +98,56 @@ impl AppContext { } first_error.map_or(Ok(()), Err) } + + /// Name every upgrade backup [`Self::remove_upgrade_backups`] would delete right now. + pub(crate) fn upgrade_backup_scope(&self) -> Result { + let [app, wallet] = self.upgrade_backup_databases().map(|database| { + crate::wallet_backend::platform_compatibility::backup_names(&database) + .map_err(|source| TaskError::FileSystem { source }) + }); + Ok(UpgradeBackupScope { + app: app?, + wallet: wallet?, + }) + } + + /// Delete only the upgrade backups named in `scope`; later snapshots are kept. + pub(crate) fn remove_scoped_upgrade_backups( + &self, + scope: &UpgradeBackupScope, + ) -> Result<(), TaskError> { + let mut first_error = None; + for (database, names) in self + .upgrade_backup_databases() + .into_iter() + .zip([&scope.app, &scope.wallet]) + { + if let Err(source) = crate::wallet_backend::platform_compatibility::remove_named_backups( + &database, names, + ) { + first_error.get_or_insert(TaskError::FileSystem { source }); + } + } + first_error.map_or(Ok(()), Err) + } + + /// The app database and this network's wallet database, whose upgrade backups + /// copy wallet and identity history. + fn upgrade_backup_databases(&self) -> [std::path::PathBuf; 2] { + [ + self.data_dir().join("det-app.sqlite"), + crate::wallet_backend::wallet_database_path(self.data_dir(), self.network), + ] + } +} + +/// The upgrade backups that existed when a deletion was authorized, by file name per +/// database. Persisted so a retried deletion reaches exactly these snapshots and never +/// ones created afterwards, which cannot contain the deleted data. +#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub(crate) struct UpgradeBackupScope { + app: std::collections::BTreeSet, + wallet: std::collections::BTreeSet, } #[cfg(test)] diff --git a/src/wallet_backend/platform_compatibility/engine.rs b/src/wallet_backend/platform_compatibility/engine.rs index 5e43f7bd9..2ac3229ed 100644 --- a/src/wallet_backend/platform_compatibility/engine.rs +++ b/src/wallet_backend/platform_compatibility/engine.rs @@ -256,31 +256,31 @@ fn default_auto_dir(path: &Path) -> PathBuf { platform_wallet_storage::default_auto_backup_dir(path) } -/// Backup candidates that passed validation, plus the first rejected one. +/// Backup candidates that passed validation, plus every rejected one with its reason. /// /// Directory-level failures still abort the scan; a single rejected /// candidate does not, so deletion can remove every valid snapshot before /// reporting the rejection. struct BackupScan { found: Vec, - first_rejection: Option, + rejected: Vec<(PathBuf, std::io::Error)>, } /// Strict scan: any rejected candidate fails the whole scan. Retention and /// publication rely on this to never proceed past an unexpected file. fn backups_in(path: &Path, auto_dir: Option<&Path>) -> std::io::Result> { let scan = scan_backups_in(path, auto_dir)?; - scan.first_rejection.map_or(Ok(scan.found), Err) + match scan.rejected.into_iter().next() { + Some((_, error)) => Err(error), + None => Ok(scan.found), + } } fn scan_backups_in(path: &Path, auto_dir: Option<&Path>) -> std::io::Result { let mut found = Vec::new(); - let mut first_rejection = None; + let mut rejected = Vec::new(); let (Some(parent), Some(prefix)) = (path.parent(), backup_prefix(path)) else { - return Ok(BackupScan { - found, - first_rejection, - }); + return Ok(BackupScan { found, rejected }); }; for directory in [Some(parent), auto_dir].into_iter().flatten() { let metadata = match std::fs::symlink_metadata(directory) { @@ -312,19 +312,14 @@ fn scan_backups_in(path: &Path, auto_dir: Option<&Path>) -> std::io::Result found.push(entry.path()), - Err(error) => { - first_rejection.get_or_insert(error); - } + Err(error) => rejected.push((entry.path(), error)), } } } } found.sort(); found.dedup(); - Ok(BackupScan { - found, - first_rejection, - }) + Ok(BackupScan { found, rejected }) } fn upstream_backup_name(path: &Path, name: &str) -> bool { @@ -538,18 +533,52 @@ fn usable_snapshot(backup: &Path) -> bool { /// synced on every call (including a retry that finds nothing left to delete), /// so callers may retire their retry state afterwards. pub(crate) fn remove_backups(path: &Path) -> std::io::Result<()> { - remove_backups_with_sync(path, sync_directory) + remove_backups_with_sync(path, |_| true, sync_directory) +} + +/// File names of every retained upgrade-backup candidate of the database at `path`, +/// including candidates that fail validation (removing those later reports the failure). +/// +/// Snapshot names are never reused (random or timestamped suffixes), so a name set +/// captured now identifies exactly the snapshots that exist now. +pub(crate) fn backup_names(path: &Path) -> std::io::Result> { + let _guard = backup_lock(path)?; + let scan = scan_backups_in(path, Some(&default_auto_dir(path)))?; + Ok(scan + .found + .iter() + .chain(scan.rejected.iter().map(|(backup, _)| backup)) + .filter_map(|backup| backup.file_name()?.to_str().map(str::to_owned)) + .collect()) +} + +/// [`remove_backups`] limited to the snapshots named in `names`, as captured by +/// [`backup_names`]. Named snapshots already gone are skipped, and snapshots created +/// since are kept, so a retry never reaches data the caller was not granted. +pub(crate) fn remove_named_backups(path: &Path, names: &BTreeSet) -> std::io::Result<()> { + let named = |backup: &Path| { + backup + .file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| names.contains(name)) + }; + remove_backups_with_sync(path, named, sync_directory) } fn remove_backups_with_sync( path: &Path, + selected: impl Fn(&Path) -> bool, mut sync_dir: impl FnMut(&Path) -> std::io::Result<()>, ) -> std::io::Result<()> { let _guard = backup_lock(path)?; let auto_dir = default_auto_dir(path); let scan = scan_backups_in(path, Some(&auto_dir))?; - let mut first_error = scan.first_rejection; - for backup in scan.found { + let mut first_error = scan + .rejected + .into_iter() + .find(|(backup, _)| selected(backup)) + .map(|(_, error)| error); + for backup in scan.found.into_iter().filter(|backup| selected(backup)) { if let Err(error) = remove_backup(&backup, path) { first_error.get_or_insert(error); } diff --git a/src/wallet_backend/platform_compatibility/engine/tests.rs b/src/wallet_backend/platform_compatibility/engine/tests.rs index ca106b3ae..2cc1c5762 100644 --- a/src/wallet_backend/platform_compatibility/engine/tests.rs +++ b/src/wallet_backend/platform_compatibility/engine/tests.rs @@ -297,6 +297,49 @@ fn platform_compatibility_removes_only_matching_upstream_backups() { assert!(kept.exists()); } +/// A captured name set reaches exactly the snapshots it names: snapshots created +/// afterwards survive, already-removed names are skipped, and a named candidate +/// that cannot be removed still fails the call. +#[test] +fn platform_compatibility_named_removal_keeps_later_snapshots() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("det-testnet.sqlite"); + let auto = dir.path().join("backups/auto"); + std::fs::create_dir_all(&auto).unwrap(); + let sibling = dir + .path() + .join("det-testnet.sqlite.platform-67d4ef3-backup-a1.sqlite"); + let upstream = auto.join("pre-migration-det-testnet-1-to-2-20260915T120000Z.db"); + let blocked = dir + .path() + .join("det-testnet.sqlite.platform-67d4ef3-backup-blocked.sqlite"); + for backup in [&sibling, &upstream] { + std::fs::write(backup, b"backup").unwrap(); + } + std::fs::create_dir(&blocked).unwrap(); + let names = backup_names(&path).unwrap(); + assert_eq!( + names.len(), + 3, + "rejected candidates are named too: {names:?}" + ); + + let later = dir + .path() + .join("det-testnet.sqlite.platform-67d4ef3-backup-b2.sqlite"); + std::fs::write(&later, b"backup").unwrap(); + remove_named_backups(&path, &names).unwrap_err(); + assert!(!sibling.exists()); + assert!(!upstream.exists()); + + std::fs::remove_dir(&blocked).unwrap(); + remove_named_backups(&path, &names).unwrap(); + assert!( + later.exists(), + "a snapshot created after the capture is kept" + ); +} + /// An unlink is only durable once its directory is synced: removal must sync /// every directory it deleted from, and a failed sync must fail the removal so /// callers keep their retry state (the identity-cleanup manifest) instead of @@ -316,10 +359,14 @@ fn platform_compatibility_backup_removal_fails_when_directory_sync_fails() { } let mut synced = Vec::new(); - let error = remove_backups_with_sync(&path, |directory| { - synced.push(directory.to_owned()); - Err(std::io::Error::other("injected directory sync failure")) - }) + let error = remove_backups_with_sync( + &path, + |_| true, + |directory| { + synced.push(directory.to_owned()); + Err(std::io::Error::other("injected directory sync failure")) + }, + ) .unwrap_err(); assert_eq!(error.kind(), std::io::ErrorKind::Other); @@ -343,17 +390,23 @@ fn platform_compatibility_backup_removal_retries_directory_sync_without_deletion .join("det-testnet.sqlite.platform-67d4ef3-backup-a1.sqlite"); std::fs::write(&sibling, b"backup").unwrap(); - remove_backups_with_sync(&path, |_| { - Err(std::io::Error::other("injected directory sync failure")) - }) + remove_backups_with_sync( + &path, + |_| true, + |_| Err(std::io::Error::other("injected directory sync failure")), + ) .unwrap_err(); assert!(!sibling.exists()); let mut synced = Vec::new(); - remove_backups_with_sync(&path, |directory| { - synced.push(directory.to_owned()); - Ok(()) - }) + remove_backups_with_sync( + &path, + |_| true, + |directory| { + synced.push(directory.to_owned()); + Ok(()) + }, + ) .unwrap(); assert_eq!( synced, diff --git a/src/wallet_backend/platform_compatibility/mod.rs b/src/wallet_backend/platform_compatibility/mod.rs index 001c0557e..b3cc0b448 100644 --- a/src/wallet_backend/platform_compatibility/mod.rs +++ b/src/wallet_backend/platform_compatibility/mod.rs @@ -2,7 +2,7 @@ mod engine; pub use engine::UpgradeError; -pub(crate) use engine::remove_backups; +pub(crate) use engine::{backup_names, remove_backups, remove_named_backups}; use platform_wallet::changeset::PlatformWalletPersistence; use platform_wallet_storage::{SqlitePersister, SqlitePersisterConfig, WalletStorageError}; From 4dd746c64b057235e44a3804b2f271a29d23f02e Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Wed, 30 Sep 2026 07:14:47 +0000 Subject: [PATCH 16/30] fix(wallet): expire upgrade backups by age instead of on identity removal Identity removal no longer deletes storage-upgrade backups. The persisted per-identity cleanup marker, the captured backup scope, and the named-backup removal helpers are gone, along with their retry logic and tests. Removing an identity keeps the vault key manifest cleanup unchanged. Upgrade backups now expire after a configurable retention period, 90 days by default. Retention covers bridge snapshots, upstream pre-migration snapshots, and legacy data.db migration backups. A backup's age is taken from the later of its modification time and the timestamp in its name, so a reset or skewed signal can only delay deletion. Pruning runs once both databases have opened successfully, at startup and whenever the user changes the setting. An expired snapshot is removed only after the data it protects has opened successfully. The policy lives under its own app k/v key, det:upgrade_backup_retention:v1. It is not a field in the positional AppSettings blob, so older builds and existing installs still decode their settings. An unreadable policy deletes nothing. Settings gains an Upgrade Backups section where the user can change the period (1 to 3650 days) or keep backups forever. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 21 +- docs/user-stories.md | 10 + src/backend_task/error.rs | 21 +- src/backend_task/identity/remove_identity.rs | 10 +- src/backend_task/mod.rs | 2 + src/backend_task/system_task/mod.rs | 25 ++ src/context/identity_db.rs | 241 +----------------- src/context/mod.rs | 3 + src/context/settings_db.rs | 48 ++++ src/context/wallet_lifecycle/mod.rs | 2 +- src/context/wallet_lifecycle/removal.rs | 43 +--- src/context/wallet_lifecycle/retention.rs | 151 +++++++++++ src/database/initialization.rs | 5 +- src/database/legacy_backups.rs | 163 ++++++++++++ src/database/mod.rs | 1 + src/model/backup_retention.rs | 156 ++++++++++++ src/model/mod.rs | 1 + src/ui/identity/mod.rs | 12 +- src/ui/network_chooser_screen.rs | 154 +++++++++++ .../platform_compatibility/engine.rs | 88 +++++-- .../platform_compatibility/engine/tests.rs | 158 ++++++++---- .../platform_compatibility/mod.rs | 2 +- 22 files changed, 940 insertions(+), 377 deletions(-) create mode 100644 src/context/wallet_lifecycle/retention.rs create mode 100644 src/database/legacy_backups.rs create mode 100644 src/model/backup_retention.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 67131576f..4f26b24a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,15 +56,17 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). A database temporarily held by another session now asks you to close that session and try again instead of reporting incompatible data. -- **Upgrade backups no longer pile up or outlive deleted data**: a failed - database upgrade no longer leaves a new backup on every attempt. Each +- **Upgrade backups no longer pile up, and old ones expire automatically**: a + failed database upgrade no longer leaves a new backup on every attempt. Each database keeps at most one upgrade backup, and it is deleted when you remove - a wallet or identity or clear a network's data. An upgrade that can never - succeed now shows its message instead of being retried silently. - Retention and deletion cover upstream pre-migration snapshots too. Cleanup - rejects links and non-files, reports pruning failures, and retries incomplete - identity backup deletion at the next startup. - Cleanup also removes unfinished snapshots left by a crash and runs when + a wallet or clear a network's data. Upgrade backups older than 90 days are + deleted automatically once the app has opened your data successfully; change + the period or keep backups forever under Settings → Upgrade Backups. The same + retention covers upstream pre-migration snapshots and legacy `data.db` + migration backups. Removing an identity no longer touches upgrade backups. + An upgrade that can never succeed now shows its message instead of being + retried silently. Cleanup rejects links and non-files, reports pruning + failures, removes unfinished snapshots left by a crash, and runs when removing a wallet without an available backend. Temporary storage and memory failures remain retryable; permission and invalid-input failures surface directly. Concurrent sessions now protect active snapshots from cleanup, and staged @@ -72,8 +74,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). One lock now covers the complete open, upgrade, backup, and retention cycle. A backup cleanup problem no longer stops up-to-date wallet data from opening; it blocks only an upgrade that would create a new backup. A new upgrade backup - replaces the previous one only after it is saved. Removing an identity that is - not stored on this device leaves upgrade backups in place. The retained backup + replaces the previous one only after it is saved. The retained backup is always one that opens and passes an integrity check, so an unfinished copy never replaces a complete one. Clearing a network's data still removes upgrade backups when an old shielded file cannot be deleted, and reports the clear as diff --git a/docs/user-stories.md b/docs/user-stories.md index 279d60a04..2ce9a44cb 100644 --- a/docs/user-stories.md +++ b/docs/user-stories.md @@ -325,6 +325,16 @@ As a user opening an older wallet installation, I want the app to update its sto - Standalone command-line and MCP use never wait for a window that is not present. Without a password they ask the user to open the desktop app once, then try again. - An operator without the desktop app can supply the wallet password non-interactively (`app_storage_update`; det-cli `--password-stdin` or `--password-file`, never as a command-line argument or environment variable). The update then finishes for every wallet sharing that password, or fails at once without skipping any wallet. +### WAL-034: Control how long upgrade backups are kept [Implemented] +**Persona:** Alex, Priya, Jordan + +As a user whose wallet data was copied before a storage upgrade, I want old upgrade backups to be deleted after a while so that copies of my wallet data do not stay on this device forever. + +- Upgrade backups older than 90 days are deleted automatically by default, covering app, wallet, upstream pre-migration and legacy `data.db` backups. +- Settings → Upgrade Backups lets the user change the period (1 to 3650 days) or keep backups forever; a change applies right away. +- Old backups are deleted only after the app has opened the data they protect, so a backup is never removed while it may still be needed for recovery. +- Removing an identity does not delete upgrade backups; retention alone decides when they go. + ### WAL-033: Wallets, addresses and identities survive an app upgrade [Gap] **Persona:** Alex, Priya, Jordan diff --git a/src/backend_task/error.rs b/src/backend_task/error.rs index 771691ddb..8d238cc2c 100644 --- a/src/backend_task/error.rs +++ b/src/backend_task/error.rs @@ -1185,6 +1185,25 @@ pub enum TaskError { )] WalletSeedDecryptFailed, + /// The upgrade-backup retention setting could not be read, so no backup was + /// deleted. + #[error( + "Could not read your backup retention setting, so no old backups were deleted. Restart the application to try again." + )] + BackupRetentionRead { + #[source] + source: crate::wallet_backend::KvAdapterError, + }, + + /// Expired upgrade backups could not all be deleted. + #[error( + "Some old upgrade backups could not be deleted. The app will try again the next time it starts." + )] + UpgradeBackupCleanup { + #[source] + source: std::io::Error, + }, + /// A local filesystem operation failed (e.g. creating a data directory). #[error( "Could not access local files. Check available disk space and restart the application." @@ -1381,7 +1400,7 @@ pub enum TaskError { /// The identity and its private keys are gone, but at least one optional /// owner-scoped sidecar could not be removed. #[error( - "The identity was removed, but some local identity data or upgrade backups may still be stored on this device. The app will keep trying to clear this local data automatically." + "The identity was removed, but some DashPay or token-list data may still be stored on this device. The app will keep trying to clear this local data automatically." )] IdentitySidecarCleanupIncomplete, diff --git a/src/backend_task/identity/remove_identity.rs b/src/backend_task/identity/remove_identity.rs index 70c2a5870..5d5face4c 100644 --- a/src/backend_task/identity/remove_identity.rs +++ b/src/backend_task/identity/remove_identity.rs @@ -149,10 +149,10 @@ mod tests { .any(|key_id| view.get(&MAIN, *key_id).unwrap().is_some()) } - /// Upgrade backups copy the network database with every identity in it, so a - /// removed identity must not survive in them. + /// Upgrade backups are governed by time-based retention alone; removing an + /// identity must leave every recovery snapshot in place. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn removing_an_identity_deletes_upgrade_backups() { + async fn removing_an_identity_keeps_upgrade_backups() { let staged = stage_identity_with_vaulted_keys(HIGH, MEDIUM).await; let database = crate::wallet_backend::wallet_database_path( staged.ctx.data_dir(), @@ -170,8 +170,8 @@ mod tests { .expect("the identity must be removed"); assert!( - !backup.exists(), - "the upgrade backup must be deleted together with the identity" + backup.exists(), + "identity removal must not delete upgrade backups" ); } diff --git a/src/backend_task/mod.rs b/src/backend_task/mod.rs index d71d56282..7fa35c369 100644 --- a/src/backend_task/mod.rs +++ b/src/backend_task/mod.rs @@ -675,6 +675,8 @@ pub enum BackendTaskSuccessResult { prices: Option, }, UpdatedThemePreference(crate::ui::theme::ThemeMode), + /// The upgrade-backup retention policy was saved. + UpdatedBackupRetention(crate::model::backup_retention::BackupRetention), PlatformInfo(PlatformInfoTaskResult), // DashPay related results diff --git a/src/backend_task/system_task/mod.rs b/src/backend_task/system_task/mod.rs index f7e729ac1..28a42ef6e 100644 --- a/src/backend_task/system_task/mod.rs +++ b/src/backend_task/system_task/mod.rs @@ -2,6 +2,7 @@ use crate::app::TaskResult; use crate::backend_task::BackendTaskSuccessResult; use crate::backend_task::error::TaskError; use crate::context::AppContext; +use crate::model::backup_retention::BackupRetention; use crate::ui::theme::ThemeMode; use std::sync::Arc; @@ -10,6 +11,8 @@ pub enum SystemTask { ClearNetworkDatabase, WipePlatformData, UpdateThemePreference(ThemeMode), + /// Persist the upgrade-backup retention policy and apply it right away. + UpdateBackupRetention(BackupRetention), } impl AppContext { @@ -29,6 +32,9 @@ impl AppContext { SystemTask::UpdateThemePreference(theme_mode) => { self.handle_update_theme_preference(theme_mode) } + SystemTask::UpdateBackupRetention(retention) => { + self.handle_update_backup_retention(retention) + } } } @@ -57,4 +63,23 @@ impl AppContext { Ok(BackendTaskSuccessResult::UpdatedThemePreference(theme_mode)) } + + /// Backend-task handler for `SystemTask::UpdateBackupRetention`. + /// + /// Only the save can fail the task. Pruning under the new policy is + /// best-effort and is retried on every start. + pub fn handle_update_backup_retention( + self: &Arc, + retention: BackupRetention, + ) -> Result { + let retention = retention.sanitized(); + self.set_backup_retention(retention) + .map_err(|source| TaskError::AppSettingsWrite { source })?; + // Only the databases this process has already opened successfully are safe + // to prune; before the wallet backend is up, the next start applies it. + if self.wallet_backend().is_ok() { + self.prune_expired_upgrade_backups_best_effort(); + } + Ok(BackendTaskSuccessResult::UpdatedBackupRetention(retention)) + } } diff --git a/src/context/identity_db.rs b/src/context/identity_db.rs index 1e3a9a4b3..7255ac5f5 100644 --- a/src/context/identity_db.rs +++ b/src/context/identity_db.rs @@ -1,5 +1,4 @@ use super::AppContext; -use super::wallet_lifecycle::UpgradeBackupScope; use crate::backend_task::contested_names::ScheduledDPNSVote; use crate::backend_task::error::TaskError; use crate::model::identity_discovery::DiscoveryIntent; @@ -70,17 +69,6 @@ const TOP_UPS_KEY: &str = "det:top_ups:v1"; /// `det:vault_cleanup_pending:v1:`. const VAULT_CLEANUP_PENDING_PREFIX: &str = "det:vault_cleanup_pending:v1:"; -/// Per-identity marker authorizing upgrade-backup removal for an identity removal. -/// Written only once the identity is confirmed listed, before it is delisted, so a -/// retry or the startup sweep deletes backups only for a removal of stored data; -/// a vault-cleanup manifest alone (e.g. from an unknown id) never authorizes it. -/// The value is an [`UpgradeBackupScope`]: it names the snapshots that existed at that -/// point, so a marker that outlives its removal can never reach later snapshots. -/// Global-scoped for the same reason as the manifest. Key shape: -/// `det:upgrade_backup_cleanup_pending:v2:`; unscoped `v1` markers are -/// ignored, which keeps the backups they would have authorized. -const UPGRADE_BACKUP_CLEANUP_PENDING_PREFIX: &str = "det:upgrade_backup_cleanup_pending:v2:"; - /// Per-identity marker recording that the user deliberately unloaded an /// identity from this device. Global-scoped for the same reason the /// vault-cleanup manifest is: an `Identity`-scoped key is reaped by @@ -146,13 +134,6 @@ fn identity_unloaded_key(id: &[u8; 32]) -> String { ) } -fn upgrade_backup_cleanup_pending_key(id: &[u8; 32]) -> String { - format!( - "{UPGRADE_BACKUP_CLEANUP_PENDING_PREFIX}{}", - Identifier::from(*id).to_string(Encoding::Base58) - ) -} - fn vault_cleanup_pending_key(id: &[u8; 32]) -> String { format!( "{VAULT_CLEANUP_PENDING_PREFIX}{}", @@ -1650,22 +1631,6 @@ impl AppContext { // error, and cleared only once every key and owner sidecar is removed. let vault_keys = self.pending_vault_key_placements(&kv, &id)?; self.persist_vault_cleanup_manifest(&kv, &id, &vault_keys)?; - // Read the roster only after the manifest is durable, so an unreadable index - // still leaves the retry record behind. - // Upgrade backups copy every identity, so they are wiped only for a removal of - // stored data. That authorization is persisted before delisting, so a retry or the - // startup sweep inherits it; the manifest alone never grants it, since an unknown id - // also leaves one behind and must not delete unrelated wallets' recovery snapshots. - // It names the snapshots that exist now: only those can copy this identity. - if identity_is_listed(&kv, &id)? { - let scope = self.upgrade_backup_scope()?; - kv.put( - DetScope::Global, - &upgrade_backup_cleanup_pending_key(&id), - &scope, - ) - .map_err(identity_err)?; - } // Before delisting, so a failure between the two leaves a marker for a // still-listed identity — inert, since discovery only consults it when // storage says the identity is absent. The reverse order would leave a @@ -1684,7 +1649,6 @@ impl AppContext { "Deleted identity still registered with the wallet store" ); } - Ok(sidecar_cleanup) } @@ -1708,29 +1672,7 @@ impl AppContext { .wallet_backend() .and_then(|backend| backend.dashpay_clear_owner_overlays(&identifier)); let token_cleanup = super::contract_token_db::forget_identity_token_state(kv, &identifier); - let mut sidecar_cleanup = - sidecar_cleanup_outcome(dashpay_cleanup, token_cleanup, &identifier); - let backup_marker = upgrade_backup_cleanup_pending_key(id); - let backup_scope = kv - .get::(DetScope::Global, &backup_marker) - .map_err(identity_err)?; - if let Some(scope) = backup_scope { - // The scope never reaches snapshots taken after this identity was gone, so - // repeating this after a failed retirement below is harmless. - match self.remove_scoped_upgrade_backups(&scope) { - Ok(()) => { - // Keep the manifest while the marker remains, so startup retries it. - if let Err(error) = kv.delete(DetScope::Global, &backup_marker) { - sidecar_cleanup = IdentitySidecarCleanup::Incomplete; - tracing::warn!(identity_id = %identifier, %error, "Upgrade backups removed but their cleanup marker remains"); - } - } - Err(error) => { - sidecar_cleanup = IdentitySidecarCleanup::Incomplete; - tracing::warn!(identity_id = %identifier, ?error, "Upgrade backup removal remains pending after identity deletion"); - } - } - } + let sidecar_cleanup = sidecar_cleanup_outcome(dashpay_cleanup, token_cleanup, &identifier); if !sidecar_cleanup.is_incomplete() { clear_vault_cleanup_manifest(kv, id); } @@ -5666,185 +5608,4 @@ mod tests { ); staged.ctx.wallet_backend().unwrap().shutdown().await; } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn removing_unknown_identity_keeps_upgrade_backups() { - let staged = stage_identity_with_vaulted_keys([0x33; 32], [0x44; 32]).await; - let backup = staged - .ctx - .data_dir() - .join("det-app.sqlite.platform-67d4ef3-backup-kept.sqlite"); - std::fs::write(&backup, b"backup").unwrap(); - let unknown = Identifier::from([0x77; 32]); - staged - .ctx - .delete_local_qualified_identity(&unknown) - .expect("removing an unknown identity is a no-op"); - assert!( - backup.exists(), - "an unknown identity must not delete other data's recovery snapshots" - ); - staged - .ctx - .delete_local_qualified_identity(&staged.id) - .unwrap(); - assert!( - !backup.exists(), - "removing a stored identity still deletes the snapshots that copy it" - ); - staged.ctx.wallet_backend().unwrap().shutdown().await; - } - - /// An interrupted unknown-id removal leaves a manifest behind; neither the startup sweep - /// nor a direct retry may treat that manifest as permission to delete upgrade backups. - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn interrupted_unknown_identity_removal_keeps_upgrade_backups() { - let staged = stage_identity_with_vaulted_keys([0x33; 32], [0x44; 32]).await; - let kv = staged.ctx.det_kv().unwrap(); - let backup = staged - .ctx - .data_dir() - .join("det-app.sqlite.platform-67d4ef3-backup-kept.sqlite"); - std::fs::write(&backup, b"backup").unwrap(); - let unknown = Identifier::from([0x77; 32]); - kv.put(DetScope::Global, "det:token_order:v1", &"invalid ordering") - .unwrap(); - assert!(matches!( - staged.ctx.delete_local_qualified_identity(&unknown), - Err(TaskError::IdentitySidecarCleanupIncomplete) - )); - assert!( - kv.get::>( - DetScope::Global, - &vault_cleanup_pending_key(&unknown.to_buffer()), - ) - .unwrap() - .is_some(), - "the interrupted removal leaves its manifest for a retry" - ); - assert!(matches!( - staged.ctx.delete_local_qualified_identity(&unknown), - Err(TaskError::IdentitySidecarCleanupIncomplete) - )); - assert!(backup.exists(), "a direct retry keeps unrelated backups"); - kv.delete(DetScope::Global, "det:token_order:v1").unwrap(); - staged.ctx.resume_pending_vault_cleanups(); - assert!( - kv.list(DetScope::Global, Some(VAULT_CLEANUP_PENDING_PREFIX)) - .unwrap() - .is_empty(), - "the sweep retires the repaired removal" - ); - assert!(backup.exists(), "the startup sweep keeps unrelated backups"); - - // A crash right after the manifest write leaves no backup-removal marker either. - kv.put( - DetScope::Global, - &vault_cleanup_pending_key(&unknown.to_buffer()), - &Vec::<(StoredPrivateKeyTarget, KeyID)>::new(), - ) - .unwrap(); - staged.ctx.resume_pending_vault_cleanups(); - assert!( - backup.exists(), - "a bare manifest never authorizes backup removal" - ); - staged.ctx.wallet_backend().unwrap().shutdown().await; - } - - /// Retiring the backup-removal marker can fail after its snapshots are gone. That - /// removal must stay incomplete, and neither the startup sweep nor a later retry of - /// the absent identity may delete snapshots created after the identity was removed. - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn failed_backup_marker_retirement_never_deletes_newer_snapshots() { - let staged = stage_identity_with_vaulted_keys([0x33; 32], [0x44; 32]).await; - let kv = staged.ctx.det_kv().unwrap(); - let data_dir = staged.ctx.data_dir().to_path_buf(); - let copied = data_dir.join("det-app.sqlite.platform-67d4ef3-backup-copied.sqlite"); - std::fs::write(&copied, b"backup").unwrap(); - let conn = rusqlite::Connection::open(data_dir.join("det-testnet.sqlite")).unwrap(); - conn.execute_batch(&format!( - "CREATE TRIGGER fail_marker_retirement BEFORE DELETE ON meta_global \ - WHEN OLD.key LIKE '{UPGRADE_BACKUP_CLEANUP_PENDING_PREFIX}%' \ - BEGIN SELECT RAISE(FAIL, 'injected marker retirement failure'); END;" - )) - .unwrap(); - - assert!(matches!( - staged.ctx.delete_local_qualified_identity(&staged.id), - Err(TaskError::IdentitySidecarCleanupIncomplete) - )); - assert!( - !copied.exists(), - "the snapshot copying the identity is removed" - ); - assert!( - !kv.list(DetScope::Global, Some(VAULT_CLEANUP_PENDING_PREFIX)) - .unwrap() - .is_empty(), - "an unretired marker keeps the removal discoverable at startup" - ); - - conn.execute_batch("DROP TRIGGER fail_marker_retirement;") - .unwrap(); - let newer = data_dir.join("det-app.sqlite.platform-67d4ef3-backup-newer.sqlite"); - std::fs::write(&newer, b"backup").unwrap(); - staged.ctx.resume_pending_vault_cleanups(); - assert!(newer.exists(), "the startup sweep keeps later snapshots"); - assert!( - kv.list( - DetScope::Global, - Some(UPGRADE_BACKUP_CLEANUP_PENDING_PREFIX) - ) - .unwrap() - .is_empty(), - "the startup sweep retires the marker" - ); - assert!( - kv.list(DetScope::Global, Some(VAULT_CLEANUP_PENDING_PREFIX)) - .unwrap() - .is_empty(), - "the startup sweep retires the repaired removal" - ); - staged - .ctx - .delete_local_qualified_identity(&staged.id) - .expect("removing an absent identity is a no-op"); - assert!( - newer.exists(), - "a retry of the absent identity keeps later snapshots" - ); - staged.ctx.wallet_backend().unwrap().shutdown().await; - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn removal_recovery_retains_manifest_until_backups_are_removed() { - let staged = stage_identity_with_vaulted_keys([0x33; 32], [0x44; 32]).await; - let kv = staged.ctx.det_kv().unwrap(); - let blocked = staged - .ctx - .data_dir() - .join("det-app.sqlite.platform-67d4ef3-backup-blocked.sqlite"); - std::fs::create_dir(&blocked).unwrap(); - assert!(matches!( - staged.ctx.delete_local_qualified_identity(&staged.id), - Err(TaskError::IdentitySidecarCleanupIncomplete) - )); - staged.ctx.resume_pending_vault_cleanups(); - assert!( - !kv.list(DetScope::Global, Some(VAULT_CLEANUP_PENDING_PREFIX)) - .unwrap() - .is_empty() - ); - std::fs::remove_dir(&blocked).unwrap(); - std::fs::write(&blocked, b"backup").unwrap(); - staged.ctx.resume_pending_vault_cleanups(); - assert!(!blocked.exists()); - assert!( - kv.list(DetScope::Global, Some(VAULT_CLEANUP_PENDING_PREFIX)) - .unwrap() - .is_empty() - ); - staged.ctx.wallet_backend().unwrap().shutdown().await; - } } diff --git a/src/context/mod.rs b/src/context/mod.rs index 30d4a14ed..773a58f36 100644 --- a/src/context/mod.rs +++ b/src/context/mod.rs @@ -1159,6 +1159,9 @@ impl AppContext { .await?; self.wallet_backend.store(Some(Arc::new(backend))); drop(_build_guard); + // Both databases opened successfully, so an expired snapshot is no longer + // needed for recovery. + self.prune_expired_upgrade_backups_best_effort(); if let Err(error) = self.refresh_pending_dpns_usernames() { tracing::warn!( ?error, diff --git a/src/context/settings_db.rs b/src/context/settings_db.rs index 96635350c..933c10409 100644 --- a/src/context/settings_db.rs +++ b/src/context/settings_db.rs @@ -7,6 +7,7 @@ //! stale value. use super::{AppContext, SettingsCacheGuard}; +use crate::model::backup_retention::BackupRetention; use crate::model::settings::{AppSettings, detect_dash_qt_path}; use crate::model::user_role::UserRole; use crate::ui::RootScreenType; @@ -225,6 +226,25 @@ impl AppContext { Ok(with_default_user_role(settings)) } + /// The persisted upgrade-backup retention policy. + /// + /// An absent or out-of-range value resolves to the default. An unreadable + /// value is an error, never the default: pruning with a guessed policy could + /// delete backups a user chose to keep forever. + pub fn backup_retention(&self) -> Result { + Ok(self + .app_kv + .get::(DetScope::Global, BackupRetention::KV_KEY)? + .unwrap_or_default() + .sanitized()) + } + + /// Persist the upgrade-backup retention policy. + pub fn set_backup_retention(&self, retention: BackupRetention) -> Result<(), KvAdapterError> { + self.app_kv + .put(DetScope::Global, BackupRetention::KV_KEY, &retention) + } + /// Write the [`AppSettings`] blob to the shared app k/v store. /// /// The blob is stored verbatim — a `user_role` of `None` stays `None` on disk @@ -273,6 +293,34 @@ mod tests { DetKv::from_store(Arc::new(InMemoryKv::default())) } + #[test] + fn backup_retention_round_trips_and_defaults_when_absent_or_invalid() { + let tmp = tempfile::tempdir().unwrap(); + let ctx = test_app_context(tmp.path()); + assert_eq!(ctx.backup_retention().unwrap(), BackupRetention::default()); + for retention in [ + BackupRetention::KeepForever, + BackupRetention::DeleteAfterDays(7), + ] { + ctx.set_backup_retention(retention).unwrap(); + assert_eq!(ctx.backup_retention().unwrap(), retention); + } + ctx.set_backup_retention(BackupRetention::DeleteAfterDays(0)) + .unwrap(); + assert_eq!(ctx.backup_retention().unwrap(), BackupRetention::default()); + } + + #[test] + fn unreadable_backup_retention_is_an_error_not_the_default() { + let tmp = tempfile::tempdir().unwrap(); + let store = Arc::new(FailingKv::default()); + let ctx = test_app_context_with_kv(tmp.path(), Arc::new(DetKv::from_store(store.clone()))); + ctx.set_backup_retention(BackupRetention::KeepForever) + .unwrap(); + store.fail_reads(true); + assert!(ctx.backup_retention().is_err()); + } + /// F69 — the `AppSettings` blob survives a put/get round-trip through the /// k/v store with every field intact, and an absent blob reads as `None` /// (the path `get_app_settings` maps to its `Default`). diff --git a/src/context/wallet_lifecycle/mod.rs b/src/context/wallet_lifecycle/mod.rs index 7e47d750f..74e1579ce 100644 --- a/src/context/wallet_lifecycle/mod.rs +++ b/src/context/wallet_lifecycle/mod.rs @@ -109,7 +109,7 @@ mod prepare; pub use prepare::PrepareGateGuard; mod registration; mod removal; -pub(crate) use removal::UpgradeBackupScope; +mod retention; mod spv; mod unlock; diff --git a/src/context/wallet_lifecycle/removal.rs b/src/context/wallet_lifecycle/removal.rs index e84438366..fce8208b0 100644 --- a/src/context/wallet_lifecycle/removal.rs +++ b/src/context/wallet_lifecycle/removal.rs @@ -99,41 +99,9 @@ impl AppContext { first_error.map_or(Ok(()), Err) } - /// Name every upgrade backup [`Self::remove_upgrade_backups`] would delete right now. - pub(crate) fn upgrade_backup_scope(&self) -> Result { - let [app, wallet] = self.upgrade_backup_databases().map(|database| { - crate::wallet_backend::platform_compatibility::backup_names(&database) - .map_err(|source| TaskError::FileSystem { source }) - }); - Ok(UpgradeBackupScope { - app: app?, - wallet: wallet?, - }) - } - - /// Delete only the upgrade backups named in `scope`; later snapshots are kept. - pub(crate) fn remove_scoped_upgrade_backups( - &self, - scope: &UpgradeBackupScope, - ) -> Result<(), TaskError> { - let mut first_error = None; - for (database, names) in self - .upgrade_backup_databases() - .into_iter() - .zip([&scope.app, &scope.wallet]) - { - if let Err(source) = crate::wallet_backend::platform_compatibility::remove_named_backups( - &database, names, - ) { - first_error.get_or_insert(TaskError::FileSystem { source }); - } - } - first_error.map_or(Ok(()), Err) - } - /// The app database and this network's wallet database, whose upgrade backups /// copy wallet and identity history. - fn upgrade_backup_databases(&self) -> [std::path::PathBuf; 2] { + pub(super) fn upgrade_backup_databases(&self) -> [std::path::PathBuf; 2] { [ self.data_dir().join("det-app.sqlite"), crate::wallet_backend::wallet_database_path(self.data_dir(), self.network), @@ -141,15 +109,6 @@ impl AppContext { } } -/// The upgrade backups that existed when a deletion was authorized, by file name per -/// database. Persisted so a retried deletion reaches exactly these snapshots and never -/// ones created afterwards, which cannot contain the deleted data. -#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -pub(crate) struct UpgradeBackupScope { - app: std::collections::BTreeSet, - wallet: std::collections::BTreeSet, -} - #[cfg(test)] mod tests { use super::*; diff --git a/src/context/wallet_lifecycle/retention.rs b/src/context/wallet_lifecycle/retention.rs new file mode 100644 index 000000000..1ec1a2fd7 --- /dev/null +++ b/src/context/wallet_lifecycle/retention.rs @@ -0,0 +1,151 @@ +//! Time-based retention of storage-upgrade backups. + +use super::*; +use std::time::SystemTime; + +impl AppContext { + /// Delete upgrade and legacy migration backups older than the configured retention period. + /// + /// Covers the app database and this network's wallet database (bridge and upstream + /// `pre-migration-*` snapshots), plus legacy `data.db` backups. Call only after those + /// databases opened successfully: an expired snapshot may be the last one, which is + /// expendable only once the database it protects is known to work. + /// + /// Attempts every location and returns the first failure; `Ok` carries the number of + /// backups deleted. Nothing is deleted when the policy cannot be read. + pub(crate) fn prune_expired_upgrade_backups(&self) -> Result { + let retention = self + .backup_retention() + .map_err(|source| TaskError::BackupRetentionRead { source })?; + let Some(max_age) = retention.max_age() else { + return Ok(0); + }; + let now = SystemTime::now(); + let mut removed = 0; + let mut first_error = None; + let outcomes = self + .upgrade_backup_databases() + .into_iter() + .map(|database| { + crate::wallet_backend::platform_compatibility::prune_expired_backups( + &database, max_age, now, + ) + }) + .chain(std::iter::once( + crate::database::legacy_backups::prune_expired(self.data_dir(), max_age, now), + )); + for outcome in outcomes { + match outcome { + Ok(count) => removed += count, + Err(source) => { + first_error.get_or_insert(TaskError::UpgradeBackupCleanup { source }); + } + } + } + if removed > 0 { + tracing::info!(removed, ?retention, "Deleted expired upgrade backups"); + } + first_error.map_or(Ok(removed), Err) + } + + /// Best-effort [`Self::prune_expired_upgrade_backups`]: failures are logged and + /// retried on the next start, never surfaced as a failed operation. + pub(crate) fn prune_expired_upgrade_backups_best_effort(&self) { + if let Err(error) = self.prune_expired_upgrade_backups() { + tracing::warn!( + ?error, + "Expired upgrade backups remain; retrying on next start" + ); + } + } +} + +#[cfg(test)] +mod tests { + use crate::context::test_support::test_app_context; + use crate::model::backup_retention::BackupRetention; + use std::path::Path; + use std::time::{Duration, SystemTime}; + + const DAY: Duration = Duration::from_secs(24 * 60 * 60); + + fn write_aged(path: &Path, age: Duration) { + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, b"backup").unwrap(); + std::fs::File::options() + .write(true) + .open(path) + .unwrap() + .set_modified(SystemTime::now() - age) + .unwrap(); + } + + struct Fixture { + expired: Vec, + fresh: Vec, + } + + fn backups(ctx: &crate::context::AppContext) -> Fixture { + let dir = ctx.data_dir(); + let [app, wallet] = ctx.upgrade_backup_databases(); + let bridge = |database: &Path, suffix: &str| { + database.with_file_name(format!( + "{}.platform-67d4ef3-backup-{suffix}.sqlite", + database.file_name().unwrap().to_string_lossy() + )) + }; + let expired = vec![ + bridge(&app, "old"), + bridge(&wallet, "old"), + dir.join("backups/data_backup_20000101_000000.db"), + ]; + let fresh = vec![bridge(&app, "new"), bridge(&wallet, "new")]; + for path in &expired { + write_aged(path, DAY * 120); + } + for path in &fresh { + write_aged(path, DAY); + } + Fixture { expired, fresh } + } + + #[test] + fn startup_retention_deletes_only_expired_backups() { + let tmp = tempfile::tempdir().unwrap(); + let ctx = test_app_context(tmp.path()); + let fixture = backups(&ctx); + + assert_eq!(ctx.prune_expired_upgrade_backups().unwrap(), 3); + + assert!(fixture.expired.iter().all(|path| !path.exists())); + assert!(fixture.fresh.iter().all(|path| path.exists())); + } + + #[test] + fn keep_forever_deletes_nothing() { + let tmp = tempfile::tempdir().unwrap(); + let ctx = test_app_context(tmp.path()); + ctx.set_backup_retention(BackupRetention::KeepForever) + .unwrap(); + let fixture = backups(&ctx); + + assert_eq!(ctx.prune_expired_upgrade_backups().unwrap(), 0); + + assert!(fixture.expired.iter().all(|path| path.exists())); + } + + #[test] + fn shorter_retention_deletes_more() { + let tmp = tempfile::tempdir().unwrap(); + let ctx = test_app_context(tmp.path()); + ctx.set_backup_retention(BackupRetention::DeleteAfterDays(200)) + .unwrap(); + let fixture = backups(&ctx); + assert_eq!(ctx.prune_expired_upgrade_backups().unwrap(), 0); + + ctx.set_backup_retention(BackupRetention::DeleteAfterDays(30)) + .unwrap(); + assert_eq!(ctx.prune_expired_upgrade_backups().unwrap(), 3); + assert!(fixture.fresh.iter().all(|path| path.exists())); + } +} diff --git a/src/database/initialization.rs b/src/database/initialization.rs index 29f4aa8eb..42e461216 100644 --- a/src/database/initialization.rs +++ b/src/database/initialization.rs @@ -713,7 +713,7 @@ impl Database { let backups_dir = db_file_path .parent() .expect("Expected parent directory in creating db backup folder") - .join("backups"); + .join(super::legacy_backups::BACKUP_DIR); fs::create_dir_all(&backups_dir).map_err(|e| { rusqlite::Error::ToSqlConversionFailure( format!("Failed to create db backups directory: {}", e).into(), @@ -721,8 +721,7 @@ impl Database { })?; // Generate a unique filename with a timestamp for the backup - let timestamp = Utc::now().format("%Y%m%d_%H%M%S"); - let backup_filename = format!("data_backup_{}.db", timestamp); + let backup_filename = super::legacy_backups::backup_file_name(Utc::now()); let backup_path = backups_dir.join(backup_filename); // Copy `data.db` to the unique backup file diff --git a/src/database/legacy_backups.rs b/src/database/legacy_backups.rs new file mode 100644 index 000000000..97a41543e --- /dev/null +++ b/src/database/legacy_backups.rs @@ -0,0 +1,163 @@ +//! Legacy `data.db` migration backups (`backups/data_backup_.db`). +//! +//! The legacy migration ladder copies `data.db` before upgrading it. These copies are +//! subject to the same time-based retention as the storage-upgrade backups. + +use std::path::Path; +use std::time::{Duration, SystemTime}; + +/// Directory, next to the database, that holds the legacy backups. +pub(crate) const BACKUP_DIR: &str = "backups"; +const PREFIX: &str = "data_backup_"; +const SUFFIX: &str = ".db"; +/// UTC timestamp format embedded in the backup name. +const TIMESTAMP_FORMAT: &str = "%Y%m%d_%H%M%S"; + +/// File name of a legacy backup taken at `timestamp` (UTC). +pub(crate) fn backup_file_name(timestamp: chrono::DateTime) -> String { + format!("{PREFIX}{}{SUFFIX}", timestamp.format(TIMESTAMP_FORMAT)) +} + +/// The UTC creation time embedded in a legacy backup name, or `None` for any other file. +fn named_timestamp(name: &str) -> Option { + let timestamp = name.strip_prefix(PREFIX)?.strip_suffix(SUFFIX)?; + let parsed = chrono::NaiveDateTime::parse_from_str(timestamp, TIMESTAMP_FORMAT).ok()?; + Some(parsed.and_utc().into()) +} + +/// Delete legacy backups in `/backups` older than `max_age`. +/// +/// Only regular files with the exact legacy name are candidates; anything else, +/// including symlinks and subdirectories, is left alone. Age is measured from the +/// later of the name timestamp and the modification time, so a reset or skewed signal +/// only delays deletion. Attempts every candidate and returns the first failure; +/// `Ok` carries the number of backups deleted. +pub(crate) fn prune_expired( + data_dir: &Path, + max_age: Duration, + now: SystemTime, +) -> std::io::Result { + let directory = data_dir.join(BACKUP_DIR); + let entries = match std::fs::read_dir(&directory) { + Ok(entries) => entries, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(0), + Err(error) => return Err(error), + }; + let mut removed = 0; + let mut first_error = None; + for entry in entries { + let outcome = entry.and_then(|entry| { + let Some(named) = entry.file_name().to_str().and_then(named_timestamp) else { + return Ok(false); + }; + let metadata = std::fs::symlink_metadata(entry.path())?; + if !metadata.is_file() { + return Ok(false); + } + let created = named.max(metadata.modified()?); + if !crate::model::backup_retention::backup_expired(created, now, max_age) { + return Ok(false); + } + std::fs::remove_file(entry.path()).map(|()| true) + }); + match outcome { + Ok(true) => removed += 1, + Ok(false) => {} + Err(error) => { + first_error.get_or_insert(error); + } + } + } + first_error.map_or(Ok(removed), Err) +} + +#[cfg(test)] +mod tests { + use super::*; + + const DAY: Duration = Duration::from_secs(24 * 60 * 60); + + fn set_mtime(path: &Path, time: SystemTime) { + std::fs::File::options() + .write(true) + .open(path) + .unwrap() + .set_modified(time) + .unwrap(); + } + + #[test] + fn legacy_backup_names_round_trip() { + let taken = chrono::DateTime::parse_from_rfc3339("2024-03-05T06:07:08Z") + .unwrap() + .to_utc(); + let name = backup_file_name(taken); + assert_eq!(name, "data_backup_20240305_060708.db"); + assert_eq!(named_timestamp(&name), Some(taken.into())); + for other in [ + "data_backup_20240305_060708.db-journal", + "data_backup_latest.db", + "data.db", + ] { + assert_eq!(named_timestamp(other), None, "{other}"); + } + } + + #[test] + fn legacy_backups_expire_by_later_of_name_and_mtime() { + let dir = tempfile::tempdir().unwrap(); + let backups = dir.path().join(BACKUP_DIR); + std::fs::create_dir_all(backups.join("auto")).unwrap(); + let now = SystemTime::now(); + let long_ago = now - DAY * 200; + let old_name = backup_file_name(chrono::DateTime::from(long_ago)); + let expired = backups.join(&old_name); + std::fs::write(&expired, b"old").unwrap(); + set_mtime(&expired, long_ago); + let recently_written = + backups.join(backup_file_name(chrono::DateTime::from(long_ago - DAY))); + std::fs::write(&recently_written, b"restored copy").unwrap(); + let fresh = backups.join(backup_file_name(chrono::DateTime::from(now))); + std::fs::write(&fresh, b"new").unwrap(); + set_mtime(&fresh, long_ago); + let unrelated = backups.join("notes.db"); + std::fs::write(&unrelated, b"keep").unwrap(); + set_mtime(&unrelated, long_ago); + + assert_eq!(prune_expired(dir.path(), DAY * 90, now).unwrap(), 1); + + assert!(!expired.exists()); + assert!(recently_written.exists()); + assert!(fresh.exists()); + assert!(unrelated.exists()); + assert!(backups.join("auto").is_dir()); + } + + #[test] + fn missing_legacy_backup_directory_is_a_no_op() { + let dir = tempfile::tempdir().unwrap(); + assert_eq!( + prune_expired(dir.path(), DAY, SystemTime::now()).unwrap(), + 0 + ); + } + + #[cfg(unix)] + #[test] + fn legacy_backup_symlinks_are_never_followed() { + let dir = tempfile::tempdir().unwrap(); + let backups = dir.path().join(BACKUP_DIR); + std::fs::create_dir_all(&backups).unwrap(); + let target = dir.path().join("precious.db"); + std::fs::write(&target, b"keep").unwrap(); + let link = backups.join("data_backup_20000101_000000.db"); + std::os::unix::fs::symlink(&target, &link).unwrap(); + + assert_eq!( + prune_expired(dir.path(), DAY, SystemTime::now()).unwrap(), + 0 + ); + assert!(link.symlink_metadata().is_ok()); + assert!(target.exists()); + } +} diff --git a/src/database/mod.rs b/src/database/mod.rs index 221dfef38..4ad077af2 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -6,6 +6,7 @@ mod initialization; /// it, and the cross-version migration matrix (`tests/migration-matrix/`, a /// separate integration-test crate) asserts against it directly. pub use initialization::DEFAULT_DB_VERSION; +pub(crate) mod legacy_backups; pub(crate) mod legacy_import; mod settings; mod single_key_wallet; diff --git a/src/model/backup_retention.rs b/src/model/backup_retention.rs new file mode 100644 index 000000000..cf3961707 --- /dev/null +++ b/src/model/backup_retention.rs @@ -0,0 +1,156 @@ +//! Time-based retention policy for storage-upgrade backups. +//! +//! Upgrade backups are copies of the app and wallet databases taken before a +//! storage upgrade. They exist for recovery only, so they are deleted once +//! they are older than the retention period. Pure data and validation — the +//! filesystem work lives in `wallet_backend::platform_compatibility` and +//! `database::legacy_backups`. + +use serde::{Deserialize, Serialize}; +use std::time::{Duration, SystemTime}; + +/// Seconds in one retention day. +const SECONDS_PER_DAY: u64 = 24 * 60 * 60; + +/// How long storage-upgrade backups are kept before they are deleted automatically. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum BackupRetention { + /// Never delete upgrade backups automatically. + KeepForever, + /// Delete upgrade backups older than this many days. + DeleteAfterDays(u16), +} + +impl BackupRetention { + /// Global k/v key of the persisted policy. A dedicated key, not an + /// `AppSettings` field: that blob has a positional wire layout, so a new + /// field would break decoding for existing installs and older builds. + pub const KV_KEY: &'static str = "det:upgrade_backup_retention:v1"; + + /// Default retention period: roughly three months. + pub const DEFAULT_DAYS: u16 = 90; + + /// Shortest accepted retention period, in days. + pub const MIN_DAYS: u16 = 1; + + /// Longest accepted retention period, in days (about ten years). + pub const MAX_DAYS: u16 = 3650; + + /// Maximum backup age, or `None` when backups are kept forever. + pub fn max_age(self) -> Option { + match self { + Self::KeepForever => None, + Self::DeleteAfterDays(days) => Some(Duration::from_secs( + u64::from(days).saturating_mul(SECONDS_PER_DAY), + )), + } + } + + /// The policy itself when valid, otherwise the default. + /// + /// Guards against a persisted value outside the accepted range, which + /// would otherwise delete backups sooner than any user could choose. + pub fn sanitized(self) -> Self { + match self { + Self::DeleteAfterDays(days) if validate_retention_days(u32::from(days)).is_err() => { + Self::default() + } + policy => policy, + } + } +} + +impl Default for BackupRetention { + fn default() -> Self { + Self::DeleteAfterDays(Self::DEFAULT_DAYS) + } +} + +/// A retention period outside the accepted range. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +#[error("Enter a number of days between {min} and {max}.")] +pub struct InvalidRetentionDays { + /// Shortest accepted period. + pub min: u16, + /// Longest accepted period. + pub max: u16, +} + +/// Validate a retention period in days. +pub fn validate_retention_days(days: u32) -> Result { + u16::try_from(days) + .ok() + .filter(|days| (BackupRetention::MIN_DAYS..=BackupRetention::MAX_DAYS).contains(days)) + .ok_or(InvalidRetentionDays { + min: BackupRetention::MIN_DAYS, + max: BackupRetention::MAX_DAYS, + }) +} + +/// Whether a backup created at `created` is older than `max_age` at `now`. +/// +/// `created` must be the latest of every creation signal available (file +/// modification time, a timestamp in the file name), so a reset or skewed +/// signal only postpones deletion. A creation time in the future is never +/// expired. +pub fn backup_expired(created: SystemTime, now: SystemTime, max_age: Duration) -> bool { + now.duration_since(created).is_ok_and(|age| age > max_age) +} + +#[cfg(test)] +mod tests { + use super::*; + + const DAY: Duration = Duration::from_secs(SECONDS_PER_DAY); + + #[test] + fn default_retention_is_ninety_days() { + assert_eq!( + BackupRetention::default(), + BackupRetention::DeleteAfterDays(90) + ); + assert_eq!(BackupRetention::default().max_age(), Some(DAY * 90)); + assert_eq!(BackupRetention::KeepForever.max_age(), None); + } + + #[test] + fn retention_days_are_bounded() { + assert_eq!(validate_retention_days(1), Ok(1)); + assert_eq!(validate_retention_days(3650), Ok(3650)); + for invalid in [0, 3651, u32::from(u16::MAX) + 1] { + let error = validate_retention_days(invalid).unwrap_err(); + assert_eq!( + error.to_string(), + "Enter a number of days between 1 and 3650." + ); + } + } + + #[test] + fn out_of_range_persisted_policy_falls_back_to_default() { + assert_eq!( + BackupRetention::DeleteAfterDays(0).sanitized(), + BackupRetention::default() + ); + assert_eq!( + BackupRetention::DeleteAfterDays(30).sanitized(), + BackupRetention::DeleteAfterDays(30) + ); + assert_eq!( + BackupRetention::KeepForever.sanitized(), + BackupRetention::KeepForever + ); + } + + #[test] + fn backup_expires_only_after_max_age() { + let now = SystemTime::UNIX_EPOCH + DAY * 1000; + assert!(backup_expired(now - DAY * 91, now, DAY * 90)); + assert!(!backup_expired(now - DAY * 90, now, DAY * 90)); + assert!(!backup_expired(now - DAY, now, DAY * 90)); + assert!( + !backup_expired(now + DAY * 365, now, DAY * 90), + "a future creation time is never expired" + ); + } +} diff --git a/src/model/mod.rs b/src/model/mod.rs index 3694af15b..189f369b0 100644 --- a/src/model/mod.rs +++ b/src/model/mod.rs @@ -1,6 +1,7 @@ pub mod address; pub mod amount; pub mod asset_lock; +pub mod backup_retention; pub mod contested_name; pub mod dashpay; pub mod dashpay_derivation; diff --git a/src/ui/identity/mod.rs b/src/ui/identity/mod.rs index 9167752c8..3bc35d894 100644 --- a/src/ui/identity/mod.rs +++ b/src/ui/identity/mod.rs @@ -106,18 +106,18 @@ pub const IDENTITY_REMOVED_VOTER_LEFT_AND_CLEANUP_PENDING: &str = "The identity /// Shown when the identity and its keys are gone but secondary local data /// could not be fully cleared. -pub const IDENTITY_REMOVED_LOCAL_DATA_CLEANUP_FAILED: &str = "The identity and its private keys were removed, but some local identity data or upgrade backups may still be stored on this device. The app will keep trying to clear this local data automatically."; +pub const IDENTITY_REMOVED_LOCAL_DATA_CLEANUP_FAILED: &str = "The identity and its private keys were removed, but some DashPay or token-list data may still be stored on this device. The app will keep trying to clear this local data automatically."; /// Shown when the voter identity remains and secondary local data cleanup also /// failed for at least one identity. -pub const IDENTITY_REMOVED_VOTER_LEFT_AND_LOCAL_DATA_CLEANUP_FAILED: &str = "The identity was removed, but its associated voter identity could not be removed. Some local identity data or upgrade backups may also remain on this device. Retry the voter removal after restarting the app. The app will keep trying to clear this local data automatically."; +pub const IDENTITY_REMOVED_VOTER_LEFT_AND_LOCAL_DATA_CLEANUP_FAILED: &str = "The identity was removed, but its associated voter identity could not be removed. Some DashPay or token-list data may also remain on this device. Retry the voter removal after restarting the app. The app will keep trying to clear this local data automatically."; /// Shown when both vault cleanup and secondary local-data cleanup are /// incomplete for an identity that is already delisted. -pub const IDENTITY_REMOVED_ALL_CLEANUP_PENDING: &str = "The identity was removed, but its private keys may still be stored here, and some local identity data or upgrade backups may remain. The app will keep trying to clear the keys automatically. Until then, treat this device as if it still holds them. The app will keep trying to clear this local data automatically."; +pub const IDENTITY_REMOVED_ALL_CLEANUP_PENDING: &str = "The identity was removed, but its private keys may still be stored here, and some DashPay or token-list data may remain. The app will keep trying to clear the keys automatically. Until then, treat this device as if it still holds them. The app will keep trying to clear this local data automatically."; /// Shown when the voter remains and both cleanup categories are incomplete. -pub const IDENTITY_REMOVED_VOTER_LEFT_AND_ALL_CLEANUP_PENDING: &str = "The identity was removed, but its associated voter identity could not be removed. Private keys for one or both identities may still be stored here, and some local identity data or upgrade backups may remain. The app will keep trying to clear the keys automatically. Until then, treat this device as if it still holds them. Retry the voter removal after restarting the app. The app will keep trying to clear this local data automatically."; +pub const IDENTITY_REMOVED_VOTER_LEFT_AND_ALL_CLEANUP_PENDING: &str = "The identity was removed, but its associated voter identity could not be removed. Private keys for one or both identities may still be stored here, and some DashPay or token-list data may remain. The app will keep trying to clear the keys automatically. Until then, treat this device as if it still holds them. Retry the voter removal after restarting the app. The app will keep trying to clear this local data automatically."; /// Shown when a removal is refused because the storage update is still running. pub const IDENTITY_REMOVAL_BLOCKED_BY_STORAGE_UPDATE: &str = @@ -345,7 +345,7 @@ mod tests { fn removed_identities_banner_reports_incomplete_local_data_cleanup() { let (message, message_type) = removed_identities_banner(false, false, true); assert_eq!(message, IDENTITY_REMOVED_LOCAL_DATA_CLEANUP_FAILED); - assert!(message.contains("local identity data or upgrade backups")); + assert!(message.contains("DashPay or token-list data")); assert!(message.contains("keep trying to clear this local data automatically")); assert_eq!(message_type, MessageType::Warning); } @@ -355,7 +355,7 @@ mod tests { let (message, message_type) = removed_identities_banner(false, true, true); assert_eq!(message, IDENTITY_REMOVED_ALL_CLEANUP_PENDING); assert!(message.contains("private keys")); - assert!(message.contains("local identity data or upgrade backups")); + assert!(message.contains("DashPay or token-list data")); assert_eq!(message_type, MessageType::Warning); } diff --git a/src/ui/network_chooser_screen.rs b/src/ui/network_chooser_screen.rs index b407b884c..8414105dc 100644 --- a/src/ui/network_chooser_screen.rs +++ b/src/ui/network_chooser_screen.rs @@ -6,6 +6,7 @@ use crate::backend_task::system_task::SystemTask; use crate::backend_task::{BackendTask, BackendTaskContext, BackendTaskSuccessResult}; use crate::context::AppContext; use crate::context::connection_status::OverallConnectionState; +use crate::model::backup_retention::{BackupRetention, validate_retention_days}; use crate::model::spv_status::{SpvStatus, SpvStatusSnapshot}; use crate::model::user_role::UserRole; use crate::ui::components::MessageBanner; @@ -80,6 +81,7 @@ pub struct NetworkChooserScreen { db_clear_in_progress: bool, wipe_platform_data_dialog: Option, auto_start_spv: bool, + backup_retention: BackupRetentionForm, discovery_in_progress: bool, fetch_confirmation_dialog: Option, /// Set when DAPI discovery completes and an SDK reinit is needed. @@ -100,6 +102,14 @@ impl NetworkChooserScreen { let settings = current_context.get_app_settings(); let theme_preference = settings.theme_mode; let auto_start_spv = settings.auto_start_spv; + let backup_retention = + BackupRetentionForm::new(current_context.backup_retention().unwrap_or_else(|error| { + tracing::warn!( + ?error, + "Backup retention setting unreadable; showing the default" + ); + BackupRetention::default() + })); Self { network_contexts: contexts.clone(), @@ -120,6 +130,7 @@ impl NetworkChooserScreen { db_clear_in_progress: false, wipe_platform_data_dialog: None, auto_start_spv, + backup_retention, discovery_in_progress: false, fetch_confirmation_dialog: None, pending_reinit_after_discovery: false, @@ -695,6 +706,12 @@ impl NetworkChooserScreen { ui.separator(); ui.add_space(12.0); + app_action |= self.backup_retention.show(ui, dark_mode); + + ui.add_space(12.0); + ui.separator(); + ui.add_space(12.0); + ui.label( egui::RichText::new("Database Maintenance") .strong() @@ -1362,6 +1379,102 @@ impl NetworkChooserScreen { } } +/// Settings-screen state for the upgrade-backup retention policy. +struct BackupRetentionForm { + /// Whether old backups are deleted automatically. + enabled: bool, + /// Retention period being edited, in days. + days: u32, + /// The policy last sent for saving; edits that match it are not re-sent. + saved: BackupRetention, +} + +impl BackupRetentionForm { + fn new(saved: BackupRetention) -> Self { + let days = match saved { + BackupRetention::DeleteAfterDays(days) => days, + BackupRetention::KeepForever => BackupRetention::DEFAULT_DAYS, + }; + Self { + enabled: saved != BackupRetention::KeepForever, + days: u32::from(days), + saved, + } + } + + /// The policy the form currently describes, or `None` while the period is invalid. + fn desired(&self) -> Option { + if !self.enabled { + return Some(BackupRetention::KeepForever); + } + validate_retention_days(self.days) + .ok() + .map(BackupRetention::DeleteAfterDays) + } + + /// The save task for a finished edit that differs from the saved policy. + fn commit(&mut self) -> AppAction { + match self.desired() { + Some(desired) if desired != self.saved => { + self.saved = desired; + AppAction::BackendTask(BackendTask::SystemTask(SystemTask::UpdateBackupRetention( + desired, + ))) + } + _ => AppAction::None, + } + } + + fn show(&mut self, ui: &mut Ui, dark_mode: bool) -> AppAction { + ui.label( + egui::RichText::new("Upgrade Backups") + .strong() + .color(DashColors::text_primary(dark_mode)), + ); + ui.add_space(6.0); + ui.label( + egui::RichText::new( + "Before a storage upgrade, the app saves a backup copy of your wallet data on this device. Old backups can be deleted automatically.", + ) + .color(DashColors::text_secondary(dark_mode)), + ); + ui.add_space(8.0); + + StyledCheckbox::new( + &mut self.enabled, + "Delete old upgrade backups automatically", + ) + .show(ui); + let mut editing = false; + if self.enabled { + ui.add_space(6.0); + ui.horizontal(|ui| { + ui.label("Keep upgrade backups for this many days:"); + let response = ui.add( + egui::DragValue::new(&mut self.days) + .range(BackupRetention::MIN_DAYS..=BackupRetention::MAX_DAYS) + .speed(1), + ); + // Save only a finished edit, not every intermediate drag or keystroke. + editing = response.dragged() || response.has_focus(); + }); + if let Err(error) = validate_retention_days(self.days) { + ui.colored_label(DashColors::ERROR, error.to_string()); + } + } else { + ui.label( + egui::RichText::new("Upgrade backups are kept until you delete them.") + .color(DashColors::text_secondary(dark_mode)), + ); + } + if editing { + AppAction::None + } else { + self.commit() + } + } +} + fn wipe_platform_data_available(role: UserRole, network: Network) -> bool { role.at_least(UserRole::Developer) && network == Network::Devnet } @@ -1513,6 +1626,47 @@ mod tests { assert_eq!(chooser_network_label(Network::Regtest), "Local"); } + fn saved_retention(action: AppAction) -> Option { + match action { + AppAction::BackendTask(BackendTask::SystemTask(SystemTask::UpdateBackupRetention( + retention, + ))) => Some(retention), + _ => None, + } + } + + #[test] + fn backup_retention_form_saves_only_changed_valid_policies() { + let mut form = BackupRetentionForm::new(BackupRetention::default()); + assert!(form.enabled); + assert_eq!(form.days, 90); + assert_eq!(saved_retention(form.commit()), None, "unchanged"); + + form.days = 30; + assert_eq!( + saved_retention(form.commit()), + Some(BackupRetention::DeleteAfterDays(30)) + ); + assert_eq!(saved_retention(form.commit()), None, "already sent"); + + form.days = 0; + assert_eq!(saved_retention(form.commit()), None, "invalid period"); + + form.enabled = false; + assert_eq!( + saved_retention(form.commit()), + Some(BackupRetention::KeepForever) + ); + } + + #[test] + fn keep_forever_backup_retention_form_starts_disabled() { + let form = BackupRetentionForm::new(BackupRetention::KeepForever); + assert!(!form.enabled); + assert_eq!(form.days, u32::from(BackupRetention::DEFAULT_DAYS)); + assert_eq!(form.desired(), Some(BackupRetention::KeepForever)); + } + #[test] fn wipe_platform_data_is_available_only_to_developers_on_devnet() { assert!(!wipe_platform_data_available( diff --git a/src/wallet_backend/platform_compatibility/engine.rs b/src/wallet_backend/platform_compatibility/engine.rs index 2ac3229ed..0c833d255 100644 --- a/src/wallet_backend/platform_compatibility/engine.rs +++ b/src/wallet_backend/platform_compatibility/engine.rs @@ -536,33 +536,73 @@ pub(crate) fn remove_backups(path: &Path) -> std::io::Result<()> { remove_backups_with_sync(path, |_| true, sync_directory) } -/// File names of every retained upgrade-backup candidate of the database at `path`, -/// including candidates that fail validation (removing those later reports the failure). +/// Delete the upgrade backups of the database at `path` that are older than `max_age`. /// -/// Snapshot names are never reused (random or timestamped suffixes), so a name set -/// captured now identifies exactly the snapshots that exist now. -pub(crate) fn backup_names(path: &Path) -> std::io::Result> { +/// Covers both the bridge snapshots next to the database and upstream `pre-migration-*` +/// snapshots. Call it only once the database has opened successfully: it may remove the +/// last snapshot, which is safe only when the database no longer needs recovery. +/// +/// Attempts every candidate and returns the first failure; a candidate that fails +/// validation is reported but never blocks removing the others. `Ok` carries the number +/// of backups deleted, and their directories are synced. +pub(crate) fn prune_expired_backups( + path: &Path, + max_age: std::time::Duration, + now: std::time::SystemTime, +) -> std::io::Result { let _guard = backup_lock(path)?; let scan = scan_backups_in(path, Some(&default_auto_dir(path)))?; - Ok(scan - .found - .iter() - .chain(scan.rejected.iter().map(|(backup, _)| backup)) - .filter_map(|backup| backup.file_name()?.to_str().map(str::to_owned)) - .collect()) -} - -/// [`remove_backups`] limited to the snapshots named in `names`, as captured by -/// [`backup_names`]. Named snapshots already gone are skipped, and snapshots created -/// since are kept, so a retry never reaches data the caller was not granted. -pub(crate) fn remove_named_backups(path: &Path, names: &BTreeSet) -> std::io::Result<()> { - let named = |backup: &Path| { - backup - .file_name() - .and_then(|name| name.to_str()) - .is_some_and(|name| names.contains(name)) - }; - remove_backups_with_sync(path, named, sync_directory) + let mut first_error = scan.rejected.into_iter().next().map(|(_, error)| error); + let mut removed = 0; + let mut touched = BTreeSet::new(); + for backup in scan.found { + let outcome = backup_created(&backup).and_then(|created| { + if !crate::model::backup_retention::backup_expired(created, now, max_age) { + return Ok(false); + } + remove_backup(&backup, path).map(|()| true) + }); + match outcome { + Ok(true) => { + removed += 1; + if let Some(directory) = backup.parent() { + touched.insert(directory.to_owned()); + } + } + Ok(false) => {} + Err(error) => { + first_error.get_or_insert(error); + } + } + } + for directory in touched { + if let Err(error) = sync_directory(&directory) { + first_error.get_or_insert(error); + } + } + first_error.map_or(Ok(removed), Err) +} + +/// When `backup` was created: the later of its modification time and the timestamp in +/// its name, if any. Taking the later one means a reset or skewed signal only delays +/// deletion, never hastens it. +fn backup_created(backup: &Path) -> std::io::Result { + let modified = std::fs::symlink_metadata(backup)?.modified()?; + let named = backup + .file_name() + .and_then(|name| name.to_str()) + .and_then(upstream_backup_timestamp); + Ok(named.map_or(modified, |named| named.max(modified))) +} + +/// The UTC creation timestamp embedded in an upstream `pre-migration-*` snapshot name. +fn upstream_backup_timestamp(name: &str) -> Option { + let (_, timestamp) = name + .strip_prefix("pre-migration-")? + .strip_suffix(".db")? + .rsplit_once('-')?; + let parsed = chrono::NaiveDateTime::parse_from_str(timestamp, "%Y%m%dT%H%M%SZ").ok()?; + Some(parsed.and_utc().into()) } fn remove_backups_with_sync( diff --git a/src/wallet_backend/platform_compatibility/engine/tests.rs b/src/wallet_backend/platform_compatibility/engine/tests.rs index 2cc1c5762..510d00d06 100644 --- a/src/wallet_backend/platform_compatibility/engine/tests.rs +++ b/src/wallet_backend/platform_compatibility/engine/tests.rs @@ -297,52 +297,9 @@ fn platform_compatibility_removes_only_matching_upstream_backups() { assert!(kept.exists()); } -/// A captured name set reaches exactly the snapshots it names: snapshots created -/// afterwards survive, already-removed names are skipped, and a named candidate -/// that cannot be removed still fails the call. -#[test] -fn platform_compatibility_named_removal_keeps_later_snapshots() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("det-testnet.sqlite"); - let auto = dir.path().join("backups/auto"); - std::fs::create_dir_all(&auto).unwrap(); - let sibling = dir - .path() - .join("det-testnet.sqlite.platform-67d4ef3-backup-a1.sqlite"); - let upstream = auto.join("pre-migration-det-testnet-1-to-2-20260915T120000Z.db"); - let blocked = dir - .path() - .join("det-testnet.sqlite.platform-67d4ef3-backup-blocked.sqlite"); - for backup in [&sibling, &upstream] { - std::fs::write(backup, b"backup").unwrap(); - } - std::fs::create_dir(&blocked).unwrap(); - let names = backup_names(&path).unwrap(); - assert_eq!( - names.len(), - 3, - "rejected candidates are named too: {names:?}" - ); - - let later = dir - .path() - .join("det-testnet.sqlite.platform-67d4ef3-backup-b2.sqlite"); - std::fs::write(&later, b"backup").unwrap(); - remove_named_backups(&path, &names).unwrap_err(); - assert!(!sibling.exists()); - assert!(!upstream.exists()); - - std::fs::remove_dir(&blocked).unwrap(); - remove_named_backups(&path, &names).unwrap(); - assert!( - later.exists(), - "a snapshot created after the capture is kept" - ); -} - /// An unlink is only durable once its directory is synced: removal must sync /// every directory it deleted from, and a failed sync must fail the removal so -/// callers keep their retry state (the identity-cleanup manifest) instead of +/// callers keep their retry state instead of /// retiring it while the deletion could still be lost on power failure. #[test] fn platform_compatibility_backup_removal_fails_when_directory_sync_fails() { @@ -990,3 +947,116 @@ fn platform_compatibility_snapshot_validation_follows_ancestor_symlinks() { std::os::unix::fs::symlink(&kept, &entry_link).unwrap(); assert!(!usable_snapshot(&entry_link)); } + +const RETENTION_DAY: std::time::Duration = std::time::Duration::from_secs(24 * 60 * 60); + +/// Expiry covers both backup formats, measures age from the later of the file's +/// modification time and its name timestamp, and never touches the database itself. +#[test] +fn platform_compatibility_prune_expired_backups_by_age() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("det-testnet.sqlite"); + std::fs::write(&path, b"live database").unwrap(); + let auto = dir.path().join("backups/auto"); + std::fs::create_dir_all(&auto).unwrap(); + let old_bridge = dir + .path() + .join("det-testnet.sqlite.platform-67d4ef3-backup-old.sqlite"); + let fresh_bridge = dir + .path() + .join("det-testnet.sqlite.platform-67d4ef3-backup-fresh.sqlite"); + let old_upstream = auto.join("pre-migration-det-testnet-1-to-2-20200101T000000Z.db"); + // An old name whose file was rewritten recently (e.g. restored from a copy). + let old_name_fresh_file = auto.join("pre-migration-det-testnet-2-to-3-20200102T000000Z.db"); + // A recent name whose modification time was reset to long ago. + let fresh_name_old_file = auto.join("pre-migration-det-testnet-3-to-4-20991231T000000Z.db"); + let unrelated = auto.join("pre-migration-det-mainnet-1-to-2-20200101T000000Z.db"); + for file in [ + &old_bridge, + &fresh_bridge, + &old_upstream, + &old_name_fresh_file, + &fresh_name_old_file, + &unrelated, + ] { + std::fs::write(file, b"backup").unwrap(); + } + let hundred_days = -100 * 24 * 60 * 60; + for file in [&old_bridge, &old_upstream, &fresh_name_old_file, &unrelated] { + set_mtime(file, hundred_days); + } + set_mtime(&path, hundred_days); + + let removed = + prune_expired_backups(&path, RETENTION_DAY * 90, std::time::SystemTime::now()).unwrap(); + + assert_eq!(removed, 2); + assert!(!old_bridge.exists()); + assert!(!old_upstream.exists()); + assert!(fresh_bridge.exists()); + assert!( + old_name_fresh_file.exists(), + "a recent modification time keeps an old-named snapshot" + ); + assert!( + fresh_name_old_file.exists(), + "a recent name timestamp keeps a snapshot with an old modification time" + ); + assert!(unrelated.exists(), "another database's snapshot is kept"); + assert!(path.exists(), "the live database is never a candidate"); +} + +/// Expiry also removes the only remaining snapshot: callers run it after a successful +/// open, when the database no longer needs recovery. +#[test] +fn platform_compatibility_prune_removes_expired_sole_snapshot() { + let (dir, path, _target) = fixture(); + let snapshot = backup(&path).unwrap(); + set_mtime(&snapshot, -2 * 24 * 60 * 60); + assert_eq!( + prune_expired_backups(&path, RETENTION_DAY, std::time::SystemTime::now()).unwrap(), + 1 + ); + assert!(!snapshot.exists()); + assert!(backups(&path).unwrap().is_empty()); + drop(dir); +} + +/// A rejected candidate is reported, but expired valid snapshots are still removed and +/// the rejected entry is left alone. +#[test] +fn platform_compatibility_prune_reports_rejected_candidate_after_removing_others() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("det-testnet.sqlite"); + let expired = dir + .path() + .join("det-testnet.sqlite.platform-67d4ef3-backup-old.sqlite"); + std::fs::write(&expired, b"backup").unwrap(); + set_mtime(&expired, -10 * 24 * 60 * 60); + let blocked = dir + .path() + .join("det-testnet.sqlite.platform-67d4ef3-backup-blocked.sqlite"); + std::fs::create_dir(&blocked).unwrap(); + + prune_expired_backups(&path, RETENTION_DAY, std::time::SystemTime::now()).unwrap_err(); + + assert!(!expired.exists()); + assert!(blocked.is_dir()); +} + +/// A held lifecycle lock means an open or upgrade is running: pruning backs off. +#[test] +fn platform_compatibility_prune_backs_off_while_lock_is_held() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("det-testnet.sqlite"); + let expired = dir + .path() + .join("det-testnet.sqlite.platform-67d4ef3-backup-old.sqlite"); + std::fs::write(&expired, b"backup").unwrap(); + set_mtime(&expired, -10 * 24 * 60 * 60); + let _guard = backup_lock(&path).unwrap(); + let error = + prune_expired_backups(&path, RETENTION_DAY, std::time::SystemTime::now()).unwrap_err(); + assert_eq!(error.kind(), std::io::ErrorKind::WouldBlock); + assert!(expired.exists()); +} diff --git a/src/wallet_backend/platform_compatibility/mod.rs b/src/wallet_backend/platform_compatibility/mod.rs index b3cc0b448..53124f41f 100644 --- a/src/wallet_backend/platform_compatibility/mod.rs +++ b/src/wallet_backend/platform_compatibility/mod.rs @@ -2,7 +2,7 @@ mod engine; pub use engine::UpgradeError; -pub(crate) use engine::{backup_names, remove_backups, remove_named_backups}; +pub(crate) use engine::{prune_expired_backups, remove_backups}; use platform_wallet::changeset::PlatformWalletPersistence; use platform_wallet_storage::{SqlitePersister, SqlitePersisterConfig, WalletStorageError}; From e3773e02ec8f7186b4e9e4886f69cb1dedea28f5 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Wed, 30 Sep 2026 07:17:47 +0000 Subject: [PATCH 17/30] fix(wallet): keep upgrade backups when removing a wallet Removing a wallet no longer deletes upgrade backups. Each backup copies a whole database, including the data of the other wallets, so deleting it with a single wallet destroys recovery snapshots the remaining wallets still depend on. Time-based retention decides when backups are removed. Clearing a network's data still deletes them. Replace the wallet-removal deletion tests with tests asserting the backups survive removal, with and without a wallet backend. Drop the test for the cleanup-failure warning, since removal no longer attempts that cleanup. Update the CHANGELOG, the upgrade notes, and user stories WAL-007/WAL-034. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 10 +++--- .../2026-09-10-platform-pin/upgrade-notes.md | 10 +++--- docs/user-stories.md | 3 +- src/context/wallet_lifecycle/removal.rs | 9 ++--- src/context/wallet_lifecycle/tests.rs | 36 ++++--------------- 5 files changed, 23 insertions(+), 45 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f26b24a9..b2eb1dbc6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,16 +58,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). - **Upgrade backups no longer pile up, and old ones expire automatically**: a failed database upgrade no longer leaves a new backup on every attempt. Each - database keeps at most one upgrade backup, and it is deleted when you remove - a wallet or clear a network's data. Upgrade backups older than 90 days are + database keeps at most one upgrade backup, and it is deleted when you clear + a network's data. Upgrade backups older than 90 days are deleted automatically once the app has opened your data successfully; change the period or keep backups forever under Settings → Upgrade Backups. The same retention covers upstream pre-migration snapshots and legacy `data.db` - migration backups. Removing an identity no longer touches upgrade backups. + migration backups. Removing a wallet or an identity does not touch upgrade + backups, since they also hold other wallets' data. An upgrade that can never succeed now shows its message instead of being retried silently. Cleanup rejects links and non-files, reports pruning - failures, removes unfinished snapshots left by a crash, and runs when - removing a wallet without an available backend. Temporary storage and memory + failures, and removes unfinished snapshots left by a crash. Temporary storage and memory failures remain retryable; permission and invalid-input failures surface directly. Concurrent sessions now protect active snapshots from cleanup, and staged validation preserves retry options and guidance for temporary resource failures. diff --git a/docs/ai-design/2026-09-10-platform-pin/upgrade-notes.md b/docs/ai-design/2026-09-10-platform-pin/upgrade-notes.md index 2ba0c275a..4d51b51de 100644 --- a/docs/ai-design/2026-09-10-platform-pin/upgrade-notes.md +++ b/docs/ai-design/2026-09-10-platform-pin/upgrade-notes.md @@ -65,10 +65,12 @@ the rebuild, and keeps at most one backup per database. Retention includes upstr `backups/auto/pre-migration--*.db` snapshots and runs before and after storage opens, including failed opens. Pruning errors stop the operation; backup candidates must have the exact database-specific name format and be regular files, -not symbolic links (or hard links on Unix). Identity cleanup retains its pending -manifest until backup deletion succeeds, so startup retries incomplete cleanup. -Deleting a wallet or an identity, or clearing a network's data, also deletes these backups, because they -copy wallet and identity history. Downgrading does not automatically reverse the +not symbolic links (or hard links on Unix). Clearing a network's data also +deletes these backups, because they copy wallet and identity history. Removing a +single wallet or identity does not: the backups also hold other wallets' data. +Instead, backups older than a user-configurable retention period (default 90 days, +or never) are deleted once the databases they protect have opened successfully, +covering bridge, upstream pre-migration, and legacy `data.db` backups. Downgrading does not automatically reverse the database conversion; recovery requires the corresponding backup and the prior application version. Validation uses synthetic upstream fixtures, not a user's real profile. diff --git a/docs/user-stories.md b/docs/user-stories.md index 2ce9a44cb..a820ea551 100644 --- a/docs/user-stories.md +++ b/docs/user-stories.md @@ -100,6 +100,7 @@ As a user, I want to remove a wallet I no longer need so that it does not clutte - Confirmation prompt before removal. - Current wallet data is deleted from local storage. If an older recovery database exists, it remains untouched. +- Upgrade backups are not deleted with the wallet, since they also hold other wallets' data; they expire through backup retention (WAL-034). ### WAL-008: View wallet balances [Implemented] **Persona:** Alex, Priya, Jordan @@ -333,7 +334,7 @@ As a user whose wallet data was copied before a storage upgrade, I want old upgr - Upgrade backups older than 90 days are deleted automatically by default, covering app, wallet, upstream pre-migration and legacy `data.db` backups. - Settings → Upgrade Backups lets the user change the period (1 to 3650 days) or keep backups forever; a change applies right away. - Old backups are deleted only after the app has opened the data they protect, so a backup is never removed while it may still be needed for recovery. -- Removing an identity does not delete upgrade backups; retention alone decides when they go. +- Removing a wallet or an identity does not delete upgrade backups; retention alone decides when they go. Clearing a network's data still deletes them. ### WAL-033: Wallets, addresses and identities survive an app upgrade [Gap] **Persona:** Alex, Priya, Jordan diff --git a/src/context/wallet_lifecycle/removal.rs b/src/context/wallet_lifecycle/removal.rs index fce8208b0..c1c42ff64 100644 --- a/src/context/wallet_lifecycle/removal.rs +++ b/src/context/wallet_lifecycle/removal.rs @@ -75,18 +75,15 @@ impl AppContext { } } - if let Err(error) = self.remove_upgrade_backups() { - tracing::warn!(?error, "Failed to delete wallet upgrade backups on removal"); - show_wallet_data_removal_warning(self.egui_ctx(), error); - } - Ok(()) } /// Delete the retained compatibility-upgrade backups of the app and this network's - /// wallet databases, which copy wallet and identity history a deletion must not leave behind. + /// wallet databases, which copy the wallet and identity history a network clear wipes. /// /// Filesystem cleanup is available even when the wallet backend could not be opened. + /// Removing a single wallet or identity never calls this: those backups also hold + /// other wallets' data and expire through time-based retention instead. pub(crate) fn remove_upgrade_backups(&self) -> Result<(), TaskError> { let mut first_error = None; for database in self.upgrade_backup_databases() { diff --git a/src/context/wallet_lifecycle/tests.rs b/src/context/wallet_lifecycle/tests.rs index b8f0d1e33..c3f9a57cf 100644 --- a/src/context/wallet_lifecycle/tests.rs +++ b/src/context/wallet_lifecycle/tests.rs @@ -1714,10 +1714,10 @@ async fn remove_wallet_wipes_seed_envelope() { backend.shutdown().await; } -/// Upgrade backups copy whole wallet databases (xpubs, history, identities), so -/// removing a wallet must not leave them behind. +/// Upgrade backups copy whole databases, including other wallets' data, so removing +/// one wallet leaves them to time-based retention. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn remove_wallet_deletes_upgrade_backups() { +async fn remove_wallet_keeps_upgrade_backups() { let (ctx, sender, _tmp) = offline_testnet_context(); ctx.ensure_wallet_backend(sender) .await @@ -1749,8 +1749,8 @@ async fn remove_wallet_deletes_upgrade_backups() { for backup in &backups { assert!( - !backup.exists(), - "{} must be deleted together with the wallet", + backup.exists(), + "{} must survive wallet removal", backup.display() ); } @@ -7278,7 +7278,7 @@ async fn reconcile_managed_identities_skips_identities_linked_to_another_wallet( } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn remove_wallet_deletes_upgrade_backups_without_backend() { +async fn remove_wallet_keeps_upgrade_backups_without_backend() { let (ctx, _sender, _tmp) = offline_testnet_context(); assert!(ctx.wallet_backend().is_err()); let wallet = @@ -7299,28 +7299,6 @@ async fn remove_wallet_deletes_upgrade_backups_without_backend() { } ctx.remove_wallet(&seed_hash).unwrap(); for backup in &backups { - assert!(!backup.exists(), "{}", backup.display()); + assert!(backup.exists(), "{}", backup.display()); } } - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn remove_wallet_warns_when_backup_cleanup_fails_without_backend() { - let (ctx, _sender, _tmp) = offline_testnet_context(); - assert!(ctx.wallet_backend().is_err()); - let wallet = - crate::model::wallet::Wallet::new_from_seed([0xA7; 64], Network::Testnet, None, None) - .expect("build wallet"); - let seed_hash = wallet.seed_hash(); - ctx.wallet_context() - .insert_test_wallet(seed_hash, Arc::new(RwLock::new(wallet))); - let backup = ctx - .data_dir() - .join("det-app.sqlite.platform-67d4ef3-backup-blocked.pending"); - std::fs::create_dir(&backup).unwrap(); - crate::ui::components::MessageBanner::clear_all_global(ctx.egui_ctx()); - ctx.remove_wallet(&seed_hash).unwrap(); - assert!(backup.is_dir()); - assert!(crate::ui::components::MessageBanner::has_global( - ctx.egui_ctx() - )); -} From 87d8260371dd85b11da78c72a828c1d967e31b4f Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Wed, 30 Sep 2026 07:29:17 +0000 Subject: [PATCH 18/30] fix(wallet): confirm stricter backup retention and harden pruning Address QA findings on time-based upgrade-backup retention: - Settings asks for a danger-mode confirmation before saving a stricter policy (shorter period, or turning deletion on), because that deletes backups immediately. The saved result reports how many backups were deleted and shows it in a banner. - The form treats a policy as saved only once the backend confirms it. A failed save restores the previous inputs instead of showing a policy that was never persisted. An unreadable setting is shown as not loaded (nothing is deleted), and an explicit Save Setting button repairs it. - Legacy backup pruning refuses a symlinked or non-directory `backups` path, and counts a backup deleted concurrently by another prune as handled instead of failing. - Correct the user-story wording: a change applies right away only to shared and current-network backups. - Add tests for the ensure_wallet_backend prune wiring, for skipping the prune when the wallet open fails, for both branches of handle_update_backup_retention, and for rejecting malformed upstream timestamps. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 3 +- docs/user-stories.md | 3 +- src/backend_task/mod.rs | 16 +- src/backend_task/system_task/mod.rs | 16 +- src/context/mod.rs | 1 + src/context/wallet_lifecycle/mod.rs | 1 + src/context/wallet_lifecycle/retention.rs | 70 +++- src/context/wallet_lifecycle/tests.rs | 105 +++++ src/database/legacy_backups.rs | 81 +++- src/model/backup_retention.rs | 20 + src/ui/network_chooser_screen.rs | 387 +++++++++++++++--- .../platform_compatibility/engine/tests.rs | 43 ++ 12 files changed, 643 insertions(+), 103 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b2eb1dbc6..dc4855603 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,7 +61,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). database keeps at most one upgrade backup, and it is deleted when you clear a network's data. Upgrade backups older than 90 days are deleted automatically once the app has opened your data successfully; change - the period or keep backups forever under Settings → Upgrade Backups. The same + the period or keep backups forever under Settings → Upgrade Backups (a shorter + period asks for confirmation, since it deletes backups at once). The same retention covers upstream pre-migration snapshots and legacy `data.db` migration backups. Removing a wallet or an identity does not touch upgrade backups, since they also hold other wallets' data. diff --git a/docs/user-stories.md b/docs/user-stories.md index a820ea551..47beb8551 100644 --- a/docs/user-stories.md +++ b/docs/user-stories.md @@ -332,7 +332,8 @@ As a user opening an older wallet installation, I want the app to update its sto As a user whose wallet data was copied before a storage upgrade, I want old upgrade backups to be deleted after a while so that copies of my wallet data do not stay on this device forever. - Upgrade backups older than 90 days are deleted automatically by default, covering app, wallet, upstream pre-migration and legacy `data.db` backups. -- Settings → Upgrade Backups lets the user change the period (1 to 3650 days) or keep backups forever; a change applies right away. +- Settings → Upgrade Backups lets the user change the period (1 to 3650 days) or keep backups forever. A shorter period asks for confirmation first, because it deletes backups immediately. +- A saved change applies right away to the shared app backups, the legacy backups, and the current network's wallet backups; other networks' wallet backups follow the next time that network is opened. - Old backups are deleted only after the app has opened the data they protect, so a backup is never removed while it may still be needed for recovery. - Removing a wallet or an identity does not delete upgrade backups; retention alone decides when they go. Clearing a network's data still deletes them. diff --git a/src/backend_task/mod.rs b/src/backend_task/mod.rs index 7fa35c369..e90b7d502 100644 --- a/src/backend_task/mod.rs +++ b/src/backend_task/mod.rs @@ -332,6 +332,8 @@ pub enum BackendTaskContext { TokenRewardEstimate(IdentityTokenIdentifier), /// The destructive per-network database clear. ClearNetworkDatabase, + /// Saving the upgrade-backup retention policy. + UpdateBackupRetention, /// A scheduled-vote sweep for one network. ScheduledVoteSweep { network: Network }, /// Receive-address derivation for one wallet's deposit flow. @@ -576,6 +578,9 @@ impl From<&BackendTask> for BackendTaskContext { identity_index: *identity_index, }, BackendTask::SystemTask(SystemTask::ClearNetworkDatabase) => Self::ClearNetworkDatabase, + BackendTask::SystemTask(SystemTask::UpdateBackupRetention(_)) => { + Self::UpdateBackupRetention + } BackendTask::WalletTask(WalletTask::GenerateReceiveAddress { seed_hash }) => { Self::GenerateReceiveAddress { seed_hash: *seed_hash, @@ -675,8 +680,15 @@ pub enum BackendTaskSuccessResult { prices: Option, }, UpdatedThemePreference(crate::ui::theme::ThemeMode), - /// The upgrade-backup retention policy was saved. - UpdatedBackupRetention(crate::model::backup_retention::BackupRetention), + /// The upgrade-backup retention policy was saved and applied. + UpdatedBackupRetention { + /// The policy now in effect. + retention: crate::model::backup_retention::BackupRetention, + /// Expired backups deleted under the new policy. + deleted: usize, + /// Whether some expired backups could not be deleted (retried at next start). + cleanup_incomplete: bool, + }, PlatformInfo(PlatformInfoTaskResult), // DashPay related results diff --git a/src/backend_task/system_task/mod.rs b/src/backend_task/system_task/mod.rs index 28a42ef6e..8b8812799 100644 --- a/src/backend_task/system_task/mod.rs +++ b/src/backend_task/system_task/mod.rs @@ -1,7 +1,7 @@ use crate::app::TaskResult; use crate::backend_task::BackendTaskSuccessResult; use crate::backend_task::error::TaskError; -use crate::context::AppContext; +use crate::context::{AppContext, BackupPruneReport}; use crate::model::backup_retention::BackupRetention; use crate::ui::theme::ThemeMode; use std::sync::Arc; @@ -77,9 +77,15 @@ impl AppContext { .map_err(|source| TaskError::AppSettingsWrite { source })?; // Only the databases this process has already opened successfully are safe // to prune; before the wallet backend is up, the next start applies it. - if self.wallet_backend().is_ok() { - self.prune_expired_upgrade_backups_best_effort(); - } - Ok(BackendTaskSuccessResult::UpdatedBackupRetention(retention)) + let report = if self.wallet_backend().is_ok() { + self.prune_expired_upgrade_backups_best_effort() + } else { + BackupPruneReport::default() + }; + Ok(BackendTaskSuccessResult::UpdatedBackupRetention { + retention, + deleted: report.deleted, + cleanup_incomplete: report.failure.is_some(), + }) } } diff --git a/src/context/mod.rs b/src/context/mod.rs index 773a58f36..4f1714ba0 100644 --- a/src/context/mod.rs +++ b/src/context/mod.rs @@ -13,6 +13,7 @@ mod settings_db; #[cfg(test)] pub(crate) mod test_support; mod wallet_lifecycle; +pub use wallet_lifecycle::BackupPruneReport; pub use wallet_lifecycle::PrepareGateGuard; pub use wallet_lifecycle::WalletUnlockRetention; diff --git a/src/context/wallet_lifecycle/mod.rs b/src/context/wallet_lifecycle/mod.rs index 74e1579ce..ac27d0e36 100644 --- a/src/context/wallet_lifecycle/mod.rs +++ b/src/context/wallet_lifecycle/mod.rs @@ -110,6 +110,7 @@ pub use prepare::PrepareGateGuard; mod registration; mod removal; mod retention; +pub use retention::BackupPruneReport; mod spv; mod unlock; diff --git a/src/context/wallet_lifecycle/retention.rs b/src/context/wallet_lifecycle/retention.rs index 1ec1a2fd7..cc1ff0418 100644 --- a/src/context/wallet_lifecycle/retention.rs +++ b/src/context/wallet_lifecycle/retention.rs @@ -3,6 +3,15 @@ use super::*; use std::time::SystemTime; +/// What one retention pass did. +#[derive(Debug, Default)] +pub struct BackupPruneReport { + /// Expired backups deleted. Undercounts when a location fails part-way. + pub deleted: usize, + /// The first failure; the remaining expired backups are retried at the next start. + pub failure: Option, +} + impl AppContext { /// Delete upgrade and legacy migration backups older than the configured retention period. /// @@ -11,18 +20,21 @@ impl AppContext { /// databases opened successfully: an expired snapshot may be the last one, which is /// expendable only once the database it protects is known to work. /// - /// Attempts every location and returns the first failure; `Ok` carries the number of - /// backups deleted. Nothing is deleted when the policy cannot be read. - pub(crate) fn prune_expired_upgrade_backups(&self) -> Result { - let retention = self - .backup_retention() - .map_err(|source| TaskError::BackupRetentionRead { source })?; + /// Attempts every location and reports the first failure. Nothing is deleted when the + /// policy cannot be read. + pub(crate) fn prune_expired_upgrade_backups(&self) -> BackupPruneReport { + let mut report = BackupPruneReport::default(); + let retention = match self.backup_retention() { + Ok(retention) => retention, + Err(source) => { + report.failure = Some(TaskError::BackupRetentionRead { source }); + return report; + } + }; let Some(max_age) = retention.max_age() else { - return Ok(0); + return report; }; let now = SystemTime::now(); - let mut removed = 0; - let mut first_error = None; let outcomes = self .upgrade_backup_databases() .into_iter() @@ -36,27 +48,35 @@ impl AppContext { )); for outcome in outcomes { match outcome { - Ok(count) => removed += count, + Ok(count) => report.deleted += count, Err(source) => { - first_error.get_or_insert(TaskError::UpgradeBackupCleanup { source }); + report + .failure + .get_or_insert(TaskError::UpgradeBackupCleanup { source }); } } } - if removed > 0 { - tracing::info!(removed, ?retention, "Deleted expired upgrade backups"); + if report.deleted > 0 { + tracing::info!( + deleted = report.deleted, + ?retention, + "Deleted expired upgrade backups" + ); } - first_error.map_or(Ok(removed), Err) + report } - /// Best-effort [`Self::prune_expired_upgrade_backups`]: failures are logged and - /// retried on the next start, never surfaced as a failed operation. - pub(crate) fn prune_expired_upgrade_backups_best_effort(&self) { - if let Err(error) = self.prune_expired_upgrade_backups() { + /// [`Self::prune_expired_upgrade_backups`] with any failure logged; it is retried on + /// the next start and never fails the surrounding operation. + pub(crate) fn prune_expired_upgrade_backups_best_effort(&self) -> BackupPruneReport { + let report = self.prune_expired_upgrade_backups(); + if let Some(error) = &report.failure { tracing::warn!( ?error, "Expired upgrade backups remain; retrying on next start" ); } + report } } @@ -69,6 +89,12 @@ mod tests { const DAY: Duration = Duration::from_secs(24 * 60 * 60); + fn pruned(ctx: &crate::context::AppContext) -> usize { + let report = ctx.prune_expired_upgrade_backups(); + assert!(report.failure.is_none(), "{:?}", report.failure); + report.deleted + } + fn write_aged(path: &Path, age: Duration) { std::fs::create_dir_all(path.parent().unwrap()).unwrap(); std::fs::write(path, b"backup").unwrap(); @@ -115,7 +141,7 @@ mod tests { let ctx = test_app_context(tmp.path()); let fixture = backups(&ctx); - assert_eq!(ctx.prune_expired_upgrade_backups().unwrap(), 3); + assert_eq!(pruned(&ctx), 3); assert!(fixture.expired.iter().all(|path| !path.exists())); assert!(fixture.fresh.iter().all(|path| path.exists())); @@ -129,7 +155,7 @@ mod tests { .unwrap(); let fixture = backups(&ctx); - assert_eq!(ctx.prune_expired_upgrade_backups().unwrap(), 0); + assert_eq!(pruned(&ctx), 0); assert!(fixture.expired.iter().all(|path| path.exists())); } @@ -141,11 +167,11 @@ mod tests { ctx.set_backup_retention(BackupRetention::DeleteAfterDays(200)) .unwrap(); let fixture = backups(&ctx); - assert_eq!(ctx.prune_expired_upgrade_backups().unwrap(), 0); + assert_eq!(pruned(&ctx), 0); ctx.set_backup_retention(BackupRetention::DeleteAfterDays(30)) .unwrap(); - assert_eq!(ctx.prune_expired_upgrade_backups().unwrap(), 3); + assert_eq!(pruned(&ctx), 3); assert!(fixture.fresh.iter().all(|path| path.exists())); } } diff --git a/src/context/wallet_lifecycle/tests.rs b/src/context/wallet_lifecycle/tests.rs index c3f9a57cf..8cace5407 100644 --- a/src/context/wallet_lifecycle/tests.rs +++ b/src/context/wallet_lifecycle/tests.rs @@ -7302,3 +7302,108 @@ async fn remove_wallet_keeps_upgrade_backups_without_backend() { assert!(backup.exists(), "{}", backup.display()); } } + +/// An upgrade-backup fixture next to `database`, last modified `days` ago. +fn aged_upgrade_backup(database: &std::path::Path, days: u64) -> std::path::PathBuf { + let backup = database.with_file_name(format!( + "{}.platform-67d4ef3-backup-aged.sqlite", + database.file_name().unwrap().to_string_lossy() + )); + std::fs::write(&backup, b"old wallet history").unwrap(); + std::fs::File::options() + .write(true) + .open(&backup) + .unwrap() + .set_modified( + std::time::SystemTime::now() - std::time::Duration::from_secs(days * 24 * 60 * 60), + ) + .unwrap(); + backup +} + +/// Wiring: once the wallet backend opens, expired upgrade backups of both databases go. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn ensure_wallet_backend_prunes_expired_upgrade_backups() { + let (ctx, sender, _tmp) = offline_testnet_context(); + let expired = [ + aged_upgrade_backup(&ctx.data_dir().join("det-app.sqlite"), 120), + aged_upgrade_backup( + &crate::wallet_backend::wallet_database_path(ctx.data_dir(), Network::Testnet), + 120, + ), + ]; + + ctx.ensure_wallet_backend(sender) + .await + .expect("ensure_wallet_backend should succeed offline"); + + for backup in &expired { + assert!(!backup.exists(), "{} must expire", backup.display()); + } + ctx.wallet_backend().unwrap().shutdown().await; +} + +/// A wallet database that fails to open may still need its snapshot, so nothing is pruned. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn failed_wallet_open_keeps_expired_upgrade_backups() { + let (ctx, sender, _tmp) = offline_testnet_context(); + let wallet_db = crate::wallet_backend::wallet_database_path(ctx.data_dir(), Network::Testnet); + std::fs::create_dir_all(wallet_db.parent().unwrap()).unwrap(); + std::fs::write(&wallet_db, b"not a sqlite database").unwrap(); + let expired = [ + aged_upgrade_backup(&ctx.data_dir().join("det-app.sqlite"), 120), + aged_upgrade_backup(&wallet_db, 120), + ]; + + ctx.ensure_wallet_backend(sender) + .await + .expect_err("a corrupt wallet database must not open"); + + for backup in &expired { + assert!(backup.exists(), "{} must be kept", backup.display()); + } +} + +/// Saving a policy always persists it, but prunes only once the wallet backend has +/// opened this network's databases; the result reports what was deleted. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn update_backup_retention_prunes_only_after_wallet_backend_opens() { + use crate::backend_task::BackendTaskSuccessResult; + use crate::model::backup_retention::BackupRetention; + + let (ctx, sender, _tmp) = offline_testnet_context(); + let backup = aged_upgrade_backup(&ctx.data_dir().join("det-app.sqlite"), 60); + let saved = |result| match result { + Ok(BackendTaskSuccessResult::UpdatedBackupRetention { + retention, + deleted, + cleanup_incomplete, + }) => (retention, deleted, cleanup_incomplete), + other => panic!("unexpected result: {other:?}"), + }; + + let strict = BackupRetention::DeleteAfterDays(30); + assert_eq!( + saved(ctx.handle_update_backup_retention(strict)), + (strict, 0, false) + ); + assert_eq!(ctx.backup_retention().unwrap(), strict); + assert!( + backup.exists(), + "no pruning before the wallet backend opens" + ); + + ctx.handle_update_backup_retention(BackupRetention::KeepForever) + .unwrap(); + ctx.ensure_wallet_backend(sender) + .await + .expect("ensure_wallet_backend should succeed offline"); + assert!(backup.exists(), "keep-forever deletes nothing at startup"); + + assert_eq!( + saved(ctx.handle_update_backup_retention(strict)), + (strict, 1, false) + ); + assert!(!backup.exists()); + ctx.wallet_backend().unwrap().shutdown().await; +} diff --git a/src/database/legacy_backups.rs b/src/database/legacy_backups.rs index 97a41543e..ea325a335 100644 --- a/src/database/legacy_backups.rs +++ b/src/database/legacy_backups.rs @@ -30,19 +30,28 @@ fn named_timestamp(name: &str) -> Option { /// Only regular files with the exact legacy name are candidates; anything else, /// including symlinks and subdirectories, is left alone. Age is measured from the /// later of the name timestamp and the modification time, so a reset or skewed signal -/// only delays deletion. Attempts every candidate and returns the first failure; -/// `Ok` carries the number of backups deleted. +/// only delays deletion. A `backups` path that is a symlink or not a directory is +/// refused. A candidate that vanishes mid-pass (e.g. a concurrent prune from another +/// network's context) counts as already handled, not as a failure. Attempts every +/// candidate and returns the first failure; `Ok` carries the number of backups this +/// call deleted. pub(crate) fn prune_expired( data_dir: &Path, max_age: Duration, now: SystemTime, ) -> std::io::Result { let directory = data_dir.join(BACKUP_DIR); - let entries = match std::fs::read_dir(&directory) { - Ok(entries) => entries, + match std::fs::symlink_metadata(&directory) { + Ok(metadata) if metadata.is_dir() => {} + Ok(_) => { + return Err(std::io::Error::other( + "Legacy backup directory is not a regular directory", + )); + } Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(0), Err(error) => return Err(error), - }; + } + let entries = std::fs::read_dir(&directory)?; let mut removed = 0; let mut first_error = None; for entry in entries { @@ -63,6 +72,7 @@ pub(crate) fn prune_expired( match outcome { Ok(true) => removed += 1, Ok(false) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} Err(error) => { first_error.get_or_insert(error); } @@ -142,6 +152,67 @@ mod tests { ); } + #[cfg(unix)] + #[test] + fn legacy_backup_directory_symlink_is_refused() { + let dir = tempfile::tempdir().unwrap(); + let elsewhere = tempfile::tempdir().unwrap(); + let victim = elsewhere.path().join("data_backup_20000101_000000.db"); + std::fs::write(&victim, b"keep").unwrap(); + std::os::unix::fs::symlink(elsewhere.path(), dir.path().join(BACKUP_DIR)).unwrap(); + + prune_expired(dir.path(), DAY, SystemTime::now()).unwrap_err(); + + assert!( + victim.exists(), + "a symlinked backups directory is never entered" + ); + } + + #[test] + fn legacy_backup_path_that_is_a_file_is_refused() { + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join(BACKUP_DIR), b"not a directory").unwrap(); + prune_expired(dir.path(), DAY, SystemTime::now()).unwrap_err(); + } + + /// Two contexts pruning the shared data directory at once must not report a + /// backup the other one already deleted as a failure. + #[test] + fn concurrent_legacy_backup_prunes_do_not_fail() { + let dir = tempfile::tempdir().unwrap(); + let backups = dir.path().join(BACKUP_DIR); + std::fs::create_dir_all(&backups).unwrap(); + let long_ago = SystemTime::now() - DAY * 400; + for second in 0..200 { + let path = backups.join(format!( + "data_backup_20000101_00{:02}{:02}.db", + second / 60, + second % 60 + )); + std::fs::write(&path, b"old").unwrap(); + set_mtime(&path, long_ago); + } + let barrier = std::sync::Barrier::new(4); + let results: Vec<_> = std::thread::scope(|scope| { + let handles: Vec<_> = (0..4) + .map(|_| { + scope.spawn(|| { + barrier.wait(); + prune_expired(dir.path(), DAY, SystemTime::now()) + }) + }) + .collect(); + handles.into_iter().map(|h| h.join().unwrap()).collect() + }); + let deleted: usize = results + .into_iter() + .map(|result| result.expect("a concurrently deleted backup is not a failure")) + .sum(); + assert_eq!(deleted, 200, "each backup is counted once"); + assert_eq!(std::fs::read_dir(&backups).unwrap().count(), 0); + } + #[cfg(unix)] #[test] fn legacy_backup_symlinks_are_never_followed() { diff --git a/src/model/backup_retention.rs b/src/model/backup_retention.rs index cf3961707..6f9192b1d 100644 --- a/src/model/backup_retention.rs +++ b/src/model/backup_retention.rs @@ -46,6 +46,15 @@ impl BackupRetention { } } + /// Whether switching from `current` to this policy can delete backups `current` keeps. + pub fn is_stricter_than(self, current: Self) -> bool { + match (self.max_age(), current.max_age()) { + (Some(new), Some(old)) => new < old, + (Some(_), None) => true, + (None, _) => false, + } + } + /// The policy itself when valid, otherwise the default. /// /// Guards against a persisted value outside the accepted range, which @@ -113,6 +122,17 @@ mod tests { assert_eq!(BackupRetention::KeepForever.max_age(), None); } + #[test] + fn stricter_retention_is_any_change_that_deletes_sooner() { + use BackupRetention::{DeleteAfterDays, KeepForever}; + assert!(DeleteAfterDays(30).is_stricter_than(DeleteAfterDays(90))); + assert!(DeleteAfterDays(3650).is_stricter_than(KeepForever)); + assert!(!DeleteAfterDays(90).is_stricter_than(DeleteAfterDays(90))); + assert!(!DeleteAfterDays(120).is_stricter_than(DeleteAfterDays(90))); + assert!(!KeepForever.is_stricter_than(DeleteAfterDays(1))); + assert!(!KeepForever.is_stricter_than(KeepForever)); + } + #[test] fn retention_days_are_bounded() { assert_eq!(validate_retention_days(1), Ok(1)); diff --git a/src/ui/network_chooser_screen.rs b/src/ui/network_chooser_screen.rs index 8414105dc..3999fc84f 100644 --- a/src/ui/network_chooser_screen.rs +++ b/src/ui/network_chooser_screen.rs @@ -102,14 +102,14 @@ impl NetworkChooserScreen { let settings = current_context.get_app_settings(); let theme_preference = settings.theme_mode; let auto_start_spv = settings.auto_start_spv; - let backup_retention = - BackupRetentionForm::new(current_context.backup_retention().unwrap_or_else(|error| { - tracing::warn!( - ?error, - "Backup retention setting unreadable; showing the default" - ); - BackupRetention::default() - })); + let backup_retention = BackupRetentionForm::new( + current_context + .backup_retention() + .inspect_err(|error| { + tracing::warn!(?error, "Backup retention setting unreadable"); + }) + .ok(), + ); Self { network_contexts: contexts.clone(), @@ -1379,26 +1379,69 @@ impl NetworkChooserScreen { } } +/// Shown when the retention setting cannot be read; the backend then deletes nothing. +const BACKUP_RETENTION_UNREADABLE: &str = "Your backup retention setting could not be loaded, so no upgrade backups are deleted automatically. Choose a setting and save it to fix this."; + +/// Shown when the new policy was saved but some expired backups could not be deleted. +const BACKUP_RETENTION_CLEANUP_INCOMPLETE: &str = "Your backup retention setting was saved, but some old upgrade backups could not be deleted. The app will try again the next time it starts."; + +/// Result banner after saving a retention policy. +fn backup_retention_saved_banner( + deleted: usize, + cleanup_incomplete: bool, +) -> (String, MessageType) { + if cleanup_incomplete { + return ( + BACKUP_RETENTION_CLEANUP_INCOMPLETE.to_owned(), + MessageType::Warning, + ); + } + let message = match deleted { + 0 => "Your backup retention setting was saved.".to_owned(), + 1 => "One old upgrade backup was deleted.".to_owned(), + count => format!("{count} old upgrade backups were deleted."), + }; + (message, MessageType::Success) +} + /// Settings-screen state for the upgrade-backup retention policy. +/// +/// Edits are saved only once finished, and only after the backend confirms is the +/// new policy treated as saved. A policy that deletes backups sooner than the saved +/// one needs explicit confirmation, since saving it deletes backups right away. struct BackupRetentionForm { /// Whether old backups are deleted automatically. enabled: bool, /// Retention period being edited, in days. days: u32, - /// The policy last sent for saving; edits that match it are not re-sent. - saved: BackupRetention, + /// The persisted policy, or `None` when it could not be read. + saved: Option, + /// Inputs of the last settled state; only edits away from it are saved. + baseline: (bool, u32), + /// A save sent to the backend and not yet answered. + pending: Option, + /// A stricter policy awaiting the user's confirmation. + confirmation: Option<(BackupRetention, ConfirmationDialog)>, } impl BackupRetentionForm { - fn new(saved: BackupRetention) -> Self { - let days = match saved { - BackupRetention::DeleteAfterDays(days) => days, - BackupRetention::KeepForever => BackupRetention::DEFAULT_DAYS, - }; + fn new(saved: Option) -> Self { + // An unreadable setting deletes nothing, which the unchecked box reflects. + let inputs = Self::inputs(saved.unwrap_or(BackupRetention::KeepForever)); Self { - enabled: saved != BackupRetention::KeepForever, - days: u32::from(days), + enabled: inputs.0, + days: inputs.1, saved, + baseline: inputs, + pending: None, + confirmation: None, + } + } + + fn inputs(policy: BackupRetention) -> (bool, u32) { + match policy { + BackupRetention::DeleteAfterDays(days) => (true, u32::from(days)), + BackupRetention::KeepForever => (false, u32::from(BackupRetention::DEFAULT_DAYS)), } } @@ -1412,19 +1455,88 @@ impl BackupRetentionForm { .map(BackupRetention::DeleteAfterDays) } - /// The save task for a finished edit that differs from the saved policy. - fn commit(&mut self) -> AppAction { - match self.desired() { - Some(desired) if desired != self.saved => { - self.saved = desired; - AppAction::BackendTask(BackendTask::SystemTask(SystemTask::UpdateBackupRetention( - desired, - ))) + fn busy(&self) -> bool { + self.pending.is_some() || self.confirmation.is_some() + } + + /// Handle a finished edit: save it, ask to confirm a stricter policy, or do nothing. + /// `force` saves even an unchanged form, to repair an unreadable setting. + fn settle(&mut self, force: bool) -> AppAction { + let inputs = (self.enabled, self.days); + if self.busy() || (inputs == self.baseline && !force) { + return AppAction::None; + } + let Some(desired) = self.desired() else { + return AppAction::None; + }; + if self.saved == Some(desired) { + self.baseline = inputs; + return AppAction::None; + } + let stricter = match self.saved { + Some(saved) => desired.is_stricter_than(saved), + None => desired.max_age().is_some(), + }; + match desired { + BackupRetention::DeleteAfterDays(days) if stricter => { + let dialog = ConfirmationDialog::new( + "Delete old upgrade backups?", + format!( + "Upgrade backups older than {days} days will be deleted now, and from then on automatically. Deleted backups cannot be restored. Continue?" + ), + ) + .confirm_text(Some("Delete Old Backups")) + .cancel_text(Some("Keep Backups")) + .danger_mode(true); + self.confirmation = Some((desired, dialog)); + AppAction::None + } + _ => self.send(desired), + } + } + + fn send(&mut self, policy: BackupRetention) -> AppAction { + self.pending = Some(policy); + AppAction::BackendTask(BackendTask::SystemTask(SystemTask::UpdateBackupRetention( + policy, + ))) + } + + /// Return the inputs to the last settled state. + fn revert(&mut self) { + (self.enabled, self.days) = self.baseline; + } + + /// Resolve a user's answer to the confirmation dialog. + fn resolve_confirmation(&mut self, status: ConfirmationStatus) -> AppAction { + let Some((policy, _)) = self.confirmation.take() else { + return AppAction::None; + }; + match status { + ConfirmationStatus::Confirmed => self.send(policy), + ConfirmationStatus::Canceled => { + self.revert(); + AppAction::None } - _ => AppAction::None, } } + /// The backend persisted `retention`. + fn save_succeeded(&mut self, retention: BackupRetention) { + self.pending = None; + self.saved = Some(retention); + if self.desired() != Some(retention) { + (self.enabled, self.days) = Self::inputs(retention); + } + self.baseline = (self.enabled, self.days); + } + + /// The backend could not persist the policy; the saved one still applies. + fn save_failed(&mut self) { + self.pending = None; + self.revert(); + } + fn show(&mut self, ui: &mut Ui, dark_mode: bool) -> AppAction { ui.label( egui::RichText::new("Upgrade Backups") @@ -1439,39 +1551,57 @@ impl BackupRetentionForm { .color(DashColors::text_secondary(dark_mode)), ); ui.add_space(8.0); + if self.saved.is_none() { + ui.colored_label(DashColors::WARNING, BACKUP_RETENTION_UNREADABLE); + ui.add_space(6.0); + } - StyledCheckbox::new( - &mut self.enabled, - "Delete old upgrade backups automatically", - ) - .show(ui); let mut editing = false; - if self.enabled { - ui.add_space(6.0); - ui.horizontal(|ui| { - ui.label("Keep upgrade backups for this many days:"); - let response = ui.add( - egui::DragValue::new(&mut self.days) - .range(BackupRetention::MIN_DAYS..=BackupRetention::MAX_DAYS) - .speed(1), + let mut force = false; + ui.add_enabled_ui(!self.busy(), |ui| { + StyledCheckbox::new( + &mut self.enabled, + "Delete old upgrade backups automatically", + ) + .show(ui); + if self.enabled { + ui.add_space(6.0); + ui.horizontal(|ui| { + ui.label("Keep upgrade backups for this many days:"); + let response = ui.add( + egui::DragValue::new(&mut self.days) + .range(BackupRetention::MIN_DAYS..=BackupRetention::MAX_DAYS) + .speed(1), + ); + // Save only a finished edit, not every intermediate drag or keystroke. + editing = response.dragged() || response.has_focus(); + }); + if let Err(error) = validate_retention_days(self.days) { + ui.colored_label(DashColors::ERROR, error.to_string()); + } + } else { + ui.label( + egui::RichText::new("Upgrade backups are kept until you delete them.") + .color(DashColors::text_secondary(dark_mode)), ); - // Save only a finished edit, not every intermediate drag or keystroke. - editing = response.dragged() || response.has_focus(); - }); - if let Err(error) = validate_retention_days(self.days) { - ui.colored_label(DashColors::ERROR, error.to_string()); } - } else { - ui.label( - egui::RichText::new("Upgrade backups are kept until you delete them.") - .color(DashColors::text_secondary(dark_mode)), - ); - } - if editing { + if self.saved.is_none() { + ui.add_space(6.0); + force = ui.button("Save Setting").clicked(); + } + }); + + let mut action = if editing { AppAction::None } else { - self.commit() + self.settle(force) + }; + if let Some((_, dialog)) = self.confirmation.as_mut() + && let Some(status) = dialog.show(ui).inner.dialog_response + { + action |= self.resolve_confirmation(status); } + action } } @@ -1560,7 +1690,17 @@ impl ScreenLike for NetworkChooserScreen { // refresh ("Updated to N node addresses."). // Handle DapiNodesDiscovered (from "Refresh DAPI endpoints" button) - if let BackendTaskSuccessResult::NetworkDatabaseCleared { .. } = + if let BackendTaskSuccessResult::UpdatedBackupRetention { + retention, + deleted, + cleanup_incomplete, + } = backend_task_success_result + { + self.backup_retention.save_succeeded(retention); + let (message, message_type) = + backup_retention_saved_banner(deleted, cleanup_incomplete); + MessageBanner::set_global(self.current_app_context().egui_ctx(), message, message_type); + } else if let BackendTaskSuccessResult::NetworkDatabaseCleared { .. } = &backend_task_success_result { self.db_clear_in_progress = false; @@ -1604,6 +1744,9 @@ impl ScreenLike for NetworkChooserScreen { if matches!(context, BackendTaskContext::ClearNetworkDatabase) { self.db_clear_in_progress = false; } + if matches!(context, BackendTaskContext::UpdateBackupRetention) { + self.backup_retention.save_failed(); + } } fn display_message(&mut self, _msg: &str, msg_type: MessageType) { @@ -1636,35 +1779,145 @@ mod tests { } #[test] - fn backup_retention_form_saves_only_changed_valid_policies() { - let mut form = BackupRetentionForm::new(BackupRetention::default()); + fn backup_retention_form_saves_looser_policies_without_confirmation() { + let mut form = BackupRetentionForm::new(Some(BackupRetention::default())); assert!(form.enabled); assert_eq!(form.days, 90); - assert_eq!(saved_retention(form.commit()), None, "unchanged"); + assert_eq!(saved_retention(form.settle(false)), None, "unchanged"); - form.days = 30; + form.days = 120; assert_eq!( - saved_retention(form.commit()), - Some(BackupRetention::DeleteAfterDays(30)) + saved_retention(form.settle(false)), + Some(BackupRetention::DeleteAfterDays(120)) ); - assert_eq!(saved_retention(form.commit()), None, "already sent"); - - form.days = 0; - assert_eq!(saved_retention(form.commit()), None, "invalid period"); + assert!(form.confirmation.is_none()); + assert_eq!( + saved_retention(form.settle(false)), + None, + "a save is in flight" + ); + form.save_succeeded(BackupRetention::DeleteAfterDays(120)); form.enabled = false; assert_eq!( - saved_retention(form.commit()), + saved_retention(form.settle(false)), Some(BackupRetention::KeepForever) ); + form.save_succeeded(BackupRetention::KeepForever); + assert_eq!(saved_retention(form.settle(false)), None, "settled"); + + form.enabled = true; + form.days = 0; + assert_eq!(saved_retention(form.settle(false)), None, "invalid period"); } + /// Shortening the period deletes backups at once, so it needs confirmation; + /// cancelling restores the saved inputs. #[test] - fn keep_forever_backup_retention_form_starts_disabled() { - let form = BackupRetentionForm::new(BackupRetention::KeepForever); + fn stricter_backup_retention_requires_confirmation() { + let mut form = BackupRetentionForm::new(Some(BackupRetention::default())); + form.days = 1; + assert_eq!(saved_retention(form.settle(false)), None); + assert!(form.confirmation.is_some()); + assert_eq!( + saved_retention(form.settle(false)), + None, + "awaiting the user" + ); + + form.resolve_confirmation(ConfirmationStatus::Canceled); + assert!(form.confirmation.is_none()); + assert_eq!((form.enabled, form.days), (true, 90)); + + form.days = 30; + form.settle(false); + assert_eq!( + saved_retention(form.resolve_confirmation(ConfirmationStatus::Confirmed)), + Some(BackupRetention::DeleteAfterDays(30)) + ); + + let mut form = BackupRetentionForm::new(Some(BackupRetention::KeepForever)); assert!(!form.enabled); assert_eq!(form.days, u32::from(BackupRetention::DEFAULT_DAYS)); - assert_eq!(form.desired(), Some(BackupRetention::KeepForever)); + form.enabled = true; + assert_eq!(saved_retention(form.settle(false)), None); + assert!( + form.confirmation.is_some(), + "enabling deletion is stricter than keeping forever" + ); + } + + /// The saved policy changes only when the backend confirms it; a failed save + /// restores the previous inputs instead of showing an unsaved policy. + #[test] + fn failed_backup_retention_save_keeps_the_saved_policy() { + let mut form = BackupRetentionForm::new(Some(BackupRetention::default())); + form.days = 200; + assert!(saved_retention(form.settle(false)).is_some()); + assert_eq!(form.saved, Some(BackupRetention::default())); + + form.save_failed(); + assert_eq!(form.saved, Some(BackupRetention::default())); + assert_eq!((form.enabled, form.days), (true, 90)); + assert_eq!(saved_retention(form.settle(false)), None, "no resend loop"); + + form.days = 200; + assert_eq!( + saved_retention(form.settle(false)), + Some(BackupRetention::DeleteAfterDays(200)), + "the user can retry" + ); + } + + /// An unreadable setting shows as unchecked (nothing is deleted) and can be + /// repaired by saving explicitly. + #[test] + fn unreadable_backup_retention_can_be_repaired() { + let mut form = BackupRetentionForm::new(None); + assert!(!form.enabled); + assert_eq!( + saved_retention(form.settle(false)), + None, + "never saved unasked" + ); + assert_eq!( + saved_retention(form.settle(true)), + Some(BackupRetention::KeepForever) + ); + + let mut form = BackupRetentionForm::new(None); + form.enabled = true; + form.settle(false); + assert!( + form.confirmation.is_some(), + "enabling deletion from an unknown policy needs confirmation" + ); + } + + #[test] + fn backup_retention_banner_reports_deleted_count() { + assert_eq!( + backup_retention_saved_banner(0, false), + ( + "Your backup retention setting was saved.".to_owned(), + MessageType::Success + ) + ); + assert_eq!( + backup_retention_saved_banner(1, false).0, + "One old upgrade backup was deleted." + ); + assert_eq!( + backup_retention_saved_banner(3, false).0, + "3 old upgrade backups were deleted." + ); + assert_eq!( + backup_retention_saved_banner(3, true), + ( + BACKUP_RETENTION_CLEANUP_INCOMPLETE.to_owned(), + MessageType::Warning + ) + ); } #[test] diff --git a/src/wallet_backend/platform_compatibility/engine/tests.rs b/src/wallet_backend/platform_compatibility/engine/tests.rs index 510d00d06..d6a9d2f7b 100644 --- a/src/wallet_backend/platform_compatibility/engine/tests.rs +++ b/src/wallet_backend/platform_compatibility/engine/tests.rs @@ -1060,3 +1060,46 @@ fn platform_compatibility_prune_backs_off_while_lock_is_held() { assert_eq!(error.kind(), std::io::ErrorKind::WouldBlock); assert!(expired.exists()); } + +/// Only a well-formed UTC timestamp counts as a creation time. A malformed name is +/// not an upstream snapshot at all, so pruning never touches it however old it is. +#[test] +fn platform_compatibility_prune_rejects_malformed_upstream_timestamps() { + assert_eq!( + upstream_backup_timestamp("pre-migration-det-testnet-1-to-2-20200101T000000Z.db"), + Some( + chrono::NaiveDate::from_ymd_opt(2020, 1, 1) + .unwrap() + .and_hms_opt(0, 0, 0) + .unwrap() + .and_utc() + .into() + ) + ); + let malformed = [ + "pre-migration-det-testnet-1-to-2-20201301T000000Z.db", + "pre-migration-det-testnet-1-to-2-20200101T000000.db", + "pre-migration-det-testnet-1-to-2-2020-01-01.db", + "pre-migration-det-testnet-1-to-2-.db", + ]; + for name in malformed { + assert_eq!(upstream_backup_timestamp(name), None, "{name}"); + } + + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("det-testnet.sqlite"); + let auto = dir.path().join("backups/auto"); + std::fs::create_dir_all(&auto).unwrap(); + for name in malformed { + let file = auto.join(name); + std::fs::write(&file, b"backup").unwrap(); + set_mtime(&file, -400 * 24 * 60 * 60); + } + assert_eq!( + prune_expired_backups(&path, RETENTION_DAY, std::time::SystemTime::now()).unwrap(), + 0 + ); + for name in malformed { + assert!(auto.join(name).exists(), "{name}"); + } +} From 81cdf8a795b05fecefca49d9cb6a8dc273b85f52 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Wed, 30 Sep 2026 07:34:51 +0000 Subject: [PATCH 19/30] fix(ui): keep backup retention saves reliable across navigation and busy frames Fix three issues found in QA of the retention settings form: - Results and errors of UpdateBackupRetention now go to the network chooser screen, as ClearNetworkDatabase results already do, rather than to whichever screen is visible. Navigating away while a save is in flight no longer leaves the form disabled or loses the result banner. - Ticking "Delete old upgrade backups automatically" no longer saves on its own. The user picks the period and then applies it with "Turn On Automatic Deletion". The confirmation shows the period the user chose instead of the 90-day default, so choosing 365 days never deletes backups that are 90 to 365 days old. - A save is queued and emitted only on a frame with no other screen action, and is marked in flight only once emitted. Another action on the same frame, such as the SDK reinit after DAPI discovery, can no longer overwrite the save and leave the form waiting forever. Co-Authored-By: Claude Opus 5.5 --- docs/user-stories.md | 2 +- src/app.rs | 52 +++++++- src/ui/network_chooser_screen.rs | 208 +++++++++++++++++++++---------- 3 files changed, 188 insertions(+), 74 deletions(-) diff --git a/docs/user-stories.md b/docs/user-stories.md index 47beb8551..a1a1679d2 100644 --- a/docs/user-stories.md +++ b/docs/user-stories.md @@ -332,7 +332,7 @@ As a user opening an older wallet installation, I want the app to update its sto As a user whose wallet data was copied before a storage upgrade, I want old upgrade backups to be deleted after a while so that copies of my wallet data do not stay on this device forever. - Upgrade backups older than 90 days are deleted automatically by default, covering app, wallet, upstream pre-migration and legacy `data.db` backups. -- Settings → Upgrade Backups lets the user change the period (1 to 3650 days) or keep backups forever. A shorter period asks for confirmation first, because it deletes backups immediately. +- Settings → Upgrade Backups lets the user change the period (1 to 3650 days) or keep backups forever. Turning automatic deletion on lets the user choose the period before applying it, and any stricter setting asks for confirmation first, because it deletes backups immediately. - A saved change applies right away to the shared app backups, the legacy backups, and the current network's wallet backups; other networks' wallet backups follow the next time that network is opened. - Old backups are deleted only after the app has opened the data they protect, so a backup is never removed while it may still be needed for recovery. - Removing a wallet or an identity does not delete upgrade backups; retention alone decides when they go. Clearing a network's data still deletes them. diff --git a/src/app.rs b/src/app.rs index 0804833e7..74cffc133 100644 --- a/src/app.rs +++ b/src/app.rs @@ -1328,6 +1328,15 @@ pub enum AppAction { SwitchIdentityHubTab(crate::ui::identity::IdentityHubTab), } +/// Whether results of the task behind `context` belong to the network chooser (settings) +/// screen whichever screen is visible: it tracks the task and must hear how it ended. +fn network_chooser_owns_task(context: &BackendTaskContext) -> bool { + matches!( + context, + BackendTaskContext::ClearNetworkDatabase | BackendTaskContext::UpdateBackupRetention + ) +} + impl BitOrAssign for AppAction { fn bitor_assign(&mut self, rhs: Self) { if matches!(rhs, AppAction::None) { @@ -2955,6 +2964,16 @@ impl App for AppState { ); } } + result @ BackendTaskSuccessResult::UpdatedBackupRetention { .. } => { + // The settings form waits for this answer even after the user + // navigates away, so it goes to the chooser, not the visible screen. + if let Some(screen) = self + .main_screens + .get_mut(&RootScreenType::RootScreenNetworkChooser) + { + screen.display_backend_task_result(&context, result); + } + } BackendTaskSuccessResult::DashPayIncomingDetected(outputs) => { // The EventBridge surfaced received outputs on a // freshly-seen wallet transaction. Run the owner- @@ -3184,13 +3203,13 @@ impl App for AppState { &err, ); self.route_contact_request_error_to_hidden_hub(&err); - let is_database_clear = context == BackendTaskContext::ClearNetworkDatabase; - let suppress_stale_error = !is_database_clear + let chooser_owned = network_chooser_owns_task(&context); + let suppress_stale_error = !chooser_owned && !recovery_delivered && self .visible_screen_mut() .should_suppress_backend_task_error(&context, &err); - if is_database_clear { + if chooser_owned { if let Some(screen) = self .main_screens .get_mut(&RootScreenType::RootScreenNetworkChooser) @@ -3204,7 +3223,7 @@ impl App for AppState { // Let the screen handle specific error types first. // If handled, skip the generic error banner. let handled = suppress_stale_error - || (!is_database_clear + || (!chooser_owned && !recovery_delivered && self.visible_screen_mut().display_task_error(&err)); @@ -3241,7 +3260,7 @@ impl App for AppState { } _ => {} } - if !is_database_clear && !recovery_delivered { + if !chooser_owned && !recovery_delivered { self.visible_screen_mut() .display_message(&msg, MessageType::Error); } @@ -4513,3 +4532,26 @@ mod shutdown_tests { ); } } + +#[cfg(test)] +mod network_chooser_routing_tests { + use super::*; + use crate::backend_task::system_task::SystemTask; + use crate::model::backup_retention::BackupRetention; + + /// A retention save outlives navigation: its result and error must reach the + /// chooser even when another screen is visible, or its form stays disabled. + #[test] + fn backup_retention_task_results_route_to_network_chooser() { + let task = BackendTask::SystemTask(SystemTask::UpdateBackupRetention( + BackupRetention::KeepForever, + )); + let context = BackendTaskContext::from(&task); + assert_eq!(context, BackendTaskContext::UpdateBackupRetention); + assert!(network_chooser_owns_task(&context)); + assert!(network_chooser_owns_task( + &BackendTaskContext::ClearNetworkDatabase + )); + assert!(!network_chooser_owns_task(&BackendTaskContext::Other)); + } +} diff --git a/src/ui/network_chooser_screen.rs b/src/ui/network_chooser_screen.rs index 3999fc84f..4eb0f7585 100644 --- a/src/ui/network_chooser_screen.rs +++ b/src/ui/network_chooser_screen.rs @@ -706,7 +706,7 @@ impl NetworkChooserScreen { ui.separator(); ui.add_space(12.0); - app_action |= self.backup_retention.show(ui, dark_mode); + self.backup_retention.show(ui, dark_mode); ui.add_space(12.0); ui.separator(); @@ -1418,6 +1418,9 @@ struct BackupRetentionForm { saved: Option, /// Inputs of the last settled state; only edits away from it are saved. baseline: (bool, u32), + /// A save ready to send, emitted by [`Self::take_task`] on a frame whose screen + /// action is otherwise empty, so no other action can overwrite it. + outbox: Option, /// A save sent to the backend and not yet answered. pending: Option, /// A stricter policy awaiting the user's confirmation. @@ -1433,6 +1436,7 @@ impl BackupRetentionForm { days: inputs.1, saved, baseline: inputs, + outbox: None, pending: None, confirmation: None, } @@ -1456,22 +1460,29 @@ impl BackupRetentionForm { } fn busy(&self) -> bool { - self.pending.is_some() || self.confirmation.is_some() + self.outbox.is_some() || self.pending.is_some() || self.confirmation.is_some() } - /// Handle a finished edit: save it, ask to confirm a stricter policy, or do nothing. - /// `force` saves even an unchanged form, to repair an unreadable setting. - fn settle(&mut self, force: bool) -> AppAction { + /// Whether the user is turning automatic deletion on. That waits for an explicit + /// apply, so the period can be chosen before any backup is deleted. + fn enabling(&self) -> bool { + self.enabled && !self.baseline.0 + } + + /// Handle a finished edit: queue it, ask to confirm a stricter policy, or do nothing. + /// `apply` is an explicit user request to save: it is required to turn deletion + /// on, and saves even an unchanged form to repair an unreadable setting. + fn settle(&mut self, apply: bool) { let inputs = (self.enabled, self.days); - if self.busy() || (inputs == self.baseline && !force) { - return AppAction::None; + if self.busy() || (!apply && (inputs == self.baseline || self.enabling())) { + return; } let Some(desired) = self.desired() else { - return AppAction::None; + return; }; if self.saved == Some(desired) { self.baseline = inputs; - return AppAction::None; + return; } let stricter = match self.saved { Some(saved) => desired.is_stricter_than(saved), @@ -1489,13 +1500,16 @@ impl BackupRetentionForm { .cancel_text(Some("Keep Backups")) .danger_mode(true); self.confirmation = Some((desired, dialog)); - AppAction::None } - _ => self.send(desired), + _ => self.outbox = Some(desired), } } - fn send(&mut self, policy: BackupRetention) -> AppAction { + /// The queued save task, if any; the save counts as in flight from here on. + fn take_task(&mut self) -> AppAction { + let Some(policy) = self.outbox.take() else { + return AppAction::None; + }; self.pending = Some(policy); AppAction::BackendTask(BackendTask::SystemTask(SystemTask::UpdateBackupRetention( policy, @@ -1508,16 +1522,13 @@ impl BackupRetentionForm { } /// Resolve a user's answer to the confirmation dialog. - fn resolve_confirmation(&mut self, status: ConfirmationStatus) -> AppAction { + fn resolve_confirmation(&mut self, status: ConfirmationStatus) { let Some((policy, _)) = self.confirmation.take() else { - return AppAction::None; + return; }; match status { - ConfirmationStatus::Confirmed => self.send(policy), - ConfirmationStatus::Canceled => { - self.revert(); - AppAction::None - } + ConfirmationStatus::Confirmed => self.outbox = Some(policy), + ConfirmationStatus::Canceled => self.revert(), } } @@ -1537,7 +1548,8 @@ impl BackupRetentionForm { self.revert(); } - fn show(&mut self, ui: &mut Ui, dark_mode: bool) -> AppAction { + /// Render the section. The save task it may queue is emitted by [`Self::take_task`]. + fn show(&mut self, ui: &mut Ui, dark_mode: bool) { ui.label( egui::RichText::new("Upgrade Backups") .strong() @@ -1557,7 +1569,7 @@ impl BackupRetentionForm { } let mut editing = false; - let mut force = false; + let mut apply = false; ui.add_enabled_ui(!self.busy(), |ui| { StyledCheckbox::new( &mut self.enabled, @@ -1585,23 +1597,30 @@ impl BackupRetentionForm { .color(DashColors::text_secondary(dark_mode)), ); } - if self.saved.is_none() { + if self.enabling() { ui.add_space(6.0); - force = ui.button("Save Setting").clicked(); + ui.label( + egui::RichText::new( + "Choose how many days to keep upgrade backups, then turn on automatic deletion.", + ) + .color(DashColors::text_secondary(dark_mode)), + ); + ui.add_space(6.0); + apply = ui.button("Turn On Automatic Deletion").clicked(); + } else if self.saved.is_none() { + ui.add_space(6.0); + apply = ui.button("Save Setting").clicked(); } }); - let mut action = if editing { - AppAction::None - } else { - self.settle(force) - }; + if !editing { + self.settle(apply); + } if let Some((_, dialog)) = self.confirmation.as_mut() && let Some(status) = dialog.show(ui).inner.dialog_response { - action |= self.resolve_confirmation(status); + self.resolve_confirmation(status); } - action } } @@ -1659,6 +1678,12 @@ impl ScreenLike for NetworkChooserScreen { action |= AppAction::BackendTask(BackendTask::ReinitCoreClientAndSdk); } + // Only on an otherwise idle frame, so no other action can overwrite the save; + // a busy frame leaves it queued for the next one. + if action == AppAction::None { + action = self.backup_retention.take_task(); + } + // Recheck both network status every 3 seconds let recheck_time = Duration::from_secs(3); if action == AppAction::None { @@ -1778,37 +1803,44 @@ mod tests { } } + /// Settle a finished edit and return the save task it emits, if any. + fn step(form: &mut BackupRetentionForm, apply: bool) -> Option { + form.settle(apply); + saved_retention(form.take_task()) + } + + fn answer( + form: &mut BackupRetentionForm, + status: ConfirmationStatus, + ) -> Option { + form.resolve_confirmation(status); + saved_retention(form.take_task()) + } + #[test] fn backup_retention_form_saves_looser_policies_without_confirmation() { let mut form = BackupRetentionForm::new(Some(BackupRetention::default())); assert!(form.enabled); assert_eq!(form.days, 90); - assert_eq!(saved_retention(form.settle(false)), None, "unchanged"); + assert_eq!(step(&mut form, false), None, "unchanged"); form.days = 120; assert_eq!( - saved_retention(form.settle(false)), + step(&mut form, false), Some(BackupRetention::DeleteAfterDays(120)) ); assert!(form.confirmation.is_none()); - assert_eq!( - saved_retention(form.settle(false)), - None, - "a save is in flight" - ); + assert_eq!(step(&mut form, false), None, "a save is in flight"); form.save_succeeded(BackupRetention::DeleteAfterDays(120)); form.enabled = false; - assert_eq!( - saved_retention(form.settle(false)), - Some(BackupRetention::KeepForever) - ); + assert_eq!(step(&mut form, false), Some(BackupRetention::KeepForever)); form.save_succeeded(BackupRetention::KeepForever); - assert_eq!(saved_retention(form.settle(false)), None, "settled"); + assert_eq!(step(&mut form, false), None, "settled"); form.enabled = true; form.days = 0; - assert_eq!(saved_retention(form.settle(false)), None, "invalid period"); + assert_eq!(step(&mut form, true), None, "invalid period"); } /// Shortening the period deletes backups at once, so it needs confirmation; @@ -1817,34 +1849,80 @@ mod tests { fn stricter_backup_retention_requires_confirmation() { let mut form = BackupRetentionForm::new(Some(BackupRetention::default())); form.days = 1; - assert_eq!(saved_retention(form.settle(false)), None); + assert_eq!(step(&mut form, false), None); assert!(form.confirmation.is_some()); - assert_eq!( - saved_retention(form.settle(false)), - None, - "awaiting the user" - ); + assert_eq!(step(&mut form, false), None, "awaiting the user"); - form.resolve_confirmation(ConfirmationStatus::Canceled); + assert_eq!(answer(&mut form, ConfirmationStatus::Canceled), None); assert!(form.confirmation.is_none()); assert_eq!((form.enabled, form.days), (true, 90)); form.days = 30; - form.settle(false); + assert_eq!(step(&mut form, false), None); assert_eq!( - saved_retention(form.resolve_confirmation(ConfirmationStatus::Confirmed)), + answer(&mut form, ConfirmationStatus::Confirmed), Some(BackupRetention::DeleteAfterDays(30)) ); + } + /// Turning deletion on waits for an explicit apply, so the user can choose the + /// period first; the confirmation then names the period actually chosen. + #[test] + fn enabling_backup_retention_waits_for_the_chosen_period() { let mut form = BackupRetentionForm::new(Some(BackupRetention::KeepForever)); assert!(!form.enabled); assert_eq!(form.days, u32::from(BackupRetention::DEFAULT_DAYS)); + form.enabled = true; - assert_eq!(saved_retention(form.settle(false)), None); + assert_eq!(step(&mut form, false), None); assert!( - form.confirmation.is_some(), - "enabling deletion is stricter than keeping forever" + form.confirmation.is_none(), + "ticking the box alone neither saves nor asks to delete" + ); + form.days = 365; + assert_eq!( + step(&mut form, false), + None, + "editing the period still waits" + ); + + assert_eq!(step(&mut form, true), None); + assert!(form.confirmation.is_some(), "enabling deletion is stricter"); + assert_eq!( + answer(&mut form, ConfirmationStatus::Confirmed), + Some(BackupRetention::DeleteAfterDays(365)) + ); + form.save_succeeded(BackupRetention::DeleteAfterDays(365)); + assert!(!form.enabling()); + assert_eq!((form.enabled, form.days), (true, 365)); + + // Unticking before applying simply returns to keeping backups forever. + let mut form = BackupRetentionForm::new(Some(BackupRetention::KeepForever)); + form.enabled = true; + form.days = 10; + form.settle(false); + form.enabled = false; + assert_eq!(step(&mut form, false), None); + assert!(form.confirmation.is_none()); + } + + /// The save task leaves the form only when the screen can emit it, so a busy + /// frame never drops it and the form never waits on a task that was not sent. + #[test] + fn queued_backup_retention_save_survives_a_busy_frame() { + let mut form = BackupRetentionForm::new(Some(BackupRetention::default())); + form.days = 200; + form.settle(false); + assert!(form.pending.is_none(), "not in flight until emitted"); + assert!(form.busy(), "further edits wait for the queued save"); + // A frame whose action is already taken does not call take_task; the + // save stays queued and the next idle frame emits it. + assert_eq!( + saved_retention(form.take_task()), + Some(BackupRetention::DeleteAfterDays(200)) ); + assert_eq!(form.pending, Some(BackupRetention::DeleteAfterDays(200))); + assert_eq!(saved_retention(form.take_task()), None, "emitted once"); } /// The saved policy changes only when the backend confirms it; a failed save @@ -1853,17 +1931,17 @@ mod tests { fn failed_backup_retention_save_keeps_the_saved_policy() { let mut form = BackupRetentionForm::new(Some(BackupRetention::default())); form.days = 200; - assert!(saved_retention(form.settle(false)).is_some()); + assert!(step(&mut form, false).is_some()); assert_eq!(form.saved, Some(BackupRetention::default())); form.save_failed(); assert_eq!(form.saved, Some(BackupRetention::default())); assert_eq!((form.enabled, form.days), (true, 90)); - assert_eq!(saved_retention(form.settle(false)), None, "no resend loop"); + assert_eq!(step(&mut form, false), None, "no resend loop"); form.days = 200; assert_eq!( - saved_retention(form.settle(false)), + step(&mut form, false), Some(BackupRetention::DeleteAfterDays(200)), "the user can retry" ); @@ -1875,19 +1953,13 @@ mod tests { fn unreadable_backup_retention_can_be_repaired() { let mut form = BackupRetentionForm::new(None); assert!(!form.enabled); - assert_eq!( - saved_retention(form.settle(false)), - None, - "never saved unasked" - ); - assert_eq!( - saved_retention(form.settle(true)), - Some(BackupRetention::KeepForever) - ); + assert_eq!(step(&mut form, false), None, "never saved unasked"); + assert_eq!(step(&mut form, true), Some(BackupRetention::KeepForever)); let mut form = BackupRetentionForm::new(None); form.enabled = true; - form.settle(false); + assert_eq!(step(&mut form, false), None); + assert_eq!(step(&mut form, true), None); assert!( form.confirmation.is_some(), "enabling deletion from an unknown policy needs confirmation" From 0c667646ca53defbba0b099bea1abcaaa8f71657 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Wed, 30 Sep 2026 08:37:18 +0000 Subject: [PATCH 20/30] fix(storage): route every file deletion through one protected chokepoint All production file deletions now go through a new `utils::file_deletion::delete_file` (plus `delete_tree`, which unlinks each file through it). Before anything is unlinked, the chokepoint makes sure the target is a regular file. It then checks a hard deny-list by name: det-app.sqlite, data.db, every network's wallet and shielded DB, *.pwsvault, .env and *.premigration, each with its -wal/-shm/-journal sidecars. The same deny-list is checked again by canonical location and inode, which catches `..`, symlink, case and hard-link aliases. After that come a link-count check and the scope of the caller's `DeletionIntent`. - Backup: upgrade-engine and legacy data.db backup retention (legacy pruning now gets the live-DB and hard-link checks the engine already had) - NetworkClear: confined to spv// and its lock, used by the SPV cache clear and the retired shielded-file cleanup - Log: rotated `..log` files; works before AppContext exists The deliberate exceptions (tempfile self-cleanup, atomic renames) carry a comment at the call site. A sentinel test runs retention, wallet removal, identity removal, log rotation and a full testnet network clear against a data dir holding every network's databases, and asserts that every untargeted file is byte-identical afterwards. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 1 + src/config.rs | 3 + src/context/mod.rs | 2 +- src/context/wallet_lifecycle/mod.rs | 56 +-- src/context/wallet_lifecycle/spv.rs | 5 +- src/context/wallet_lifecycle/tests.rs | 7 +- .../tests/deletion_sentinel.rs | 200 ++++++++ src/database/legacy_backups.rs | 13 +- src/logging.rs | 9 +- src/utils/file_deletion.rs | 322 +++++++++++++ src/utils/file_deletion/tests.rs | 439 ++++++++++++++++++ src/utils/mod.rs | 1 + .../platform_compatibility/engine.rs | 8 +- 13 files changed, 1029 insertions(+), 37 deletions(-) create mode 100644 src/context/wallet_lifecycle/tests/deletion_sentinel.rs create mode 100644 src/utils/file_deletion.rs create mode 100644 src/utils/file_deletion/tests.rs diff --git a/AGENTS.md b/AGENTS.md index 47199ae7b..2dc5c20fc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -114,6 +114,7 @@ Code lives by responsibility, not convenience: - **`ui/components/`** — reusable **Component-pattern widgets ONLY**: a `show()` plus a `ComponentResponse`, a display-only render widget, or component infrastructure. If it does not render egui, it is not a component. - **`ui/state/`** — non-widget UI state: per-screen view-models and async fetch-state caches (e.g. `TrackedAssetLockCache`). Owned by screens, may return `BackendTask`, render nothing. - **`src/mcp/tools/`** — MCP tool logic, one file per domain (e.g. `wallet.rs`, `shielded.rs`, `identity.rs`) with multiple tool structs per file; never in `src/bin/det_cli/`. +- **`src/utils/file_deletion.rs`** — the single file-deletion chokepoint. Every production on-disk deletion goes through `delete_file` (or `delete_tree`) with a `DeletionIntent` stating its purpose; the chokepoint hard-refuses live databases, their sidecars, the vault and `.env`. Any deliberate exception gets a code comment with the reason at its call site. - **`src/localization.rs`** — localization logic. **`src/ui/theme.rs`** — theme/alignment helpers. Discriminator for `ui/components/` vs `ui/state/`: *does it render egui (`show`/`ui`/a render fn)?* Yes → component. No → state. diff --git a/src/config.rs b/src/config.rs index 1042f4d6f..499bb4785 100644 --- a/src/config.rs +++ b/src/config.rs @@ -311,6 +311,9 @@ fn atomic_replace_env_file( "config file path has no parent directory", ), })?; + // Not routed through `delete_file`: on failure the temp file removes only + // the uniquely named file it created itself, and success is an atomic + // rename over `.env`, not a deletion. let mut env_file = NamedTempFile::new_in(parent_dir).map_err(|source| ConfigError::SaveError { source })?; write_contents(&mut env_file)?; diff --git a/src/context/mod.rs b/src/context/mod.rs index 4f1714ba0..77a4e571d 100644 --- a/src/context/mod.rs +++ b/src/context/mod.rs @@ -668,7 +668,7 @@ impl AppContext { } /// Absolute path of the shared seed vault: `/secrets/det-secrets.pwsvault`. - fn secret_store_path(data_dir: &std::path::Path) -> std::path::PathBuf { + pub(crate) fn secret_store_path(data_dir: &std::path::Path) -> std::path::PathBuf { let mut path = data_dir.to_path_buf(); path.push("secrets"); path.push("det-secrets.pwsvault"); diff --git a/src/context/wallet_lifecycle/mod.rs b/src/context/wallet_lifecycle/mod.rs index ac27d0e36..e617fd339 100644 --- a/src/context/wallet_lifecycle/mod.rs +++ b/src/context/wallet_lifecycle/mod.rs @@ -21,6 +21,7 @@ use crate::model::wallet::meta::WalletMeta; use crate::model::wallet::seed_envelope::StoredSeedEnvelope; use crate::model::wallet::single_key::SingleKeyWallet; use crate::model::wallet::{Wallet, WalletSeedHash}; +use crate::utils::file_deletion::{DeletionIntent, delete_file, delete_tree}; use crate::wallet_backend::poison::RwLockRecover; use crate::wallet_backend::{ ClearAllOutcome, DetScope, WalletBackend, WalletMetaView, WalletSeedView, network_prefix, @@ -79,29 +80,31 @@ fn spv_storage_dir(data_dir: &Path, network: Network) -> PathBuf { /// `DiskStorageManager` lock lives at `.lock` (a sibling of the /// directory); it is removed too so a stale lock cannot block the next sync. /// A missing entry is the expected fresh/never-synced state and is tolerated. -fn clear_spv_chain_storage(spv_dir: &Path) -> Result<(), TaskError> { +/// Every unlink goes through the [`delete_file`] chokepoint, confined to this +/// network's SPV data by [`DeletionIntent::NetworkClear`]. +fn clear_spv_chain_storage(data_dir: &Path, network: Network) -> Result<(), TaskError> { + let intent = DeletionIntent::NetworkClear { data_dir, network }; + let spv_dir = spv_storage_dir(data_dir, network); for entry in SPV_CHAIN_STORAGE_ENTRIES { let path = spv_dir.join(entry); - let result = if path.is_dir() { - std::fs::remove_dir_all(&path) - } else { - std::fs::remove_file(&path) + let result = match std::fs::symlink_metadata(&path) { + Ok(metadata) if metadata.is_dir() => delete_tree(&path, intent), + Ok(_) => delete_file(&path, intent), + Err(error) => Err(error), }; - if let Err(e) = result - && e.kind() != std::io::ErrorKind::NotFound - { - return Err(TaskError::FileSystem { source: e }); - } + ignore_not_found(result)?; } + ignore_not_found(delete_file(&spv_dir.with_extension("lock"), intent)) +} - let lock_path = spv_dir.with_extension("lock"); - if let Err(e) = std::fs::remove_file(&lock_path) - && e.kind() != std::io::ErrorKind::NotFound - { - return Err(TaskError::FileSystem { source: e }); +/// Treat an already-missing file as deleted; map any other failure to [`TaskError::FileSystem`]. +fn ignore_not_found(result: std::io::Result<()>) -> Result<(), TaskError> { + match result { + Err(error) if error.kind() != std::io::ErrorKind::NotFound => { + Err(TaskError::FileSystem { source: error }) + } + _ => Ok(()), } - - Ok(()) } mod bootstrap; @@ -130,8 +133,8 @@ impl AppContext { } } -/// Unlink DET's two retired legacy shielded files from `spv_dir` (the active -/// network's spv directory), tolerating a missing file. +/// Unlink DET's two retired legacy shielded files from `network`'s spv +/// directory, tolerating a missing file. /// /// These are the files DET's deleted shielded subsystem owned: /// `det-shielded.sqlite` (the plaintext note sidecar) and @@ -139,17 +142,16 @@ impl AppContext { /// upstream coordinator's store (`det--shielded.sqlite`, outside this /// directory) is a DIFFERENT file and is deliberately NOT touched here — it is /// reset via the coordinator's own `clear_shielded`. Scoped strictly to -/// `spv_dir` so a clear of one network can never reach another network's files. -fn cleanup_legacy_shielded_files(spv_dir: &Path) -> Result<(), TaskError> { +/// that directory — enforced by the [`delete_file`] chokepoint's +/// [`DeletionIntent::NetworkClear`] scope — so a clear of one network can never +/// reach another network's files. +fn cleanup_legacy_shielded_files(data_dir: &Path, network: Network) -> Result<(), TaskError> { const LEGACY_SHIELDED_FILES: [&str; 2] = ["det-shielded.sqlite", "shielded-commitment-tree.sqlite"]; + let intent = DeletionIntent::NetworkClear { data_dir, network }; + let spv_dir = spv_storage_dir(data_dir, network); for file in LEGACY_SHIELDED_FILES { - let path = spv_dir.join(file); - if let Err(e) = std::fs::remove_file(&path) - && e.kind() != std::io::ErrorKind::NotFound - { - return Err(TaskError::FileSystem { source: e }); - } + ignore_not_found(delete_file(&spv_dir.join(file), intent))?; } Ok(()) } diff --git a/src/context/wallet_lifecycle/spv.rs b/src/context/wallet_lifecycle/spv.rs index f3e855a65..eb7ddc7ac 100644 --- a/src/context/wallet_lifecycle/spv.rs +++ b/src/context/wallet_lifecycle/spv.rs @@ -16,8 +16,7 @@ impl AppContext { /// `DiskStorageManager` has released its file lock and the deletes do not /// race a live writer. A missing directory (never synced) is success. pub fn clear_spv_data(&self) -> Result<(), TaskError> { - let spv_dir = spv_storage_dir(&self.data_dir, self.network); - clear_spv_chain_storage(&spv_dir) + clear_spv_chain_storage(&self.data_dir, self.network) } pub async fn clear_network_database(self: &Arc) -> Result<(), TaskError> { @@ -109,7 +108,7 @@ impl AppContext { // strictly to THIS network's spv directory. // A failure here must not skip the steps below: upgrade backups hold the // wallet data being cleared, so record it and keep going. - if let Err(error) = cleanup_legacy_shielded_files(backend.spv_storage_dir()) { + if let Err(error) = cleanup_legacy_shielded_files(&self.data_dir, self.network) { tracing::warn!(?error, "Legacy shielded file removal failed during clear"); failures.push(error); } diff --git a/src/context/wallet_lifecycle/tests.rs b/src/context/wallet_lifecycle/tests.rs index 8cace5407..0eea579a2 100644 --- a/src/context/wallet_lifecycle/tests.rs +++ b/src/context/wallet_lifecycle/tests.rs @@ -3445,7 +3445,7 @@ fn clear_spv_chain_storage_removes_chain_cache_but_keeps_wallet_sidecars() { std::fs::write(&wallet_sqlite, b"wallet").expect("write wallet sqlite"); std::fs::write(&shielded_tree, b"tree").expect("write shielded tree"); - clear_spv_chain_storage(&spv_dir).expect("clear must succeed"); + clear_spv_chain_storage(tmp.path(), Network::Testnet).expect("clear must succeed"); for entry in SPV_CHAIN_STORAGE_ENTRIES { assert!( @@ -3477,7 +3477,8 @@ fn clear_spv_chain_storage_is_ok_when_directory_absent() { !spv_dir.exists(), "precondition: no spv dir on a fresh install" ); - clear_spv_chain_storage(&spv_dir).expect("clearing an absent cache must succeed"); + clear_spv_chain_storage(tmp.path(), Network::Testnet) + .expect("clearing an absent cache must succeed"); } /// Seed a legacy password-protected `single_key_wallet` row into the @@ -7407,3 +7408,5 @@ async fn update_backup_retention_prunes_only_after_wallet_backend_opens() { assert!(!backup.exists()); ctx.wallet_backend().unwrap().shutdown().await; } + +mod deletion_sentinel; diff --git a/src/context/wallet_lifecycle/tests/deletion_sentinel.rs b/src/context/wallet_lifecycle/tests/deletion_sentinel.rs new file mode 100644 index 000000000..22cf70500 --- /dev/null +++ b/src/context/wallet_lifecycle/tests/deletion_sentinel.rs @@ -0,0 +1,200 @@ +//! Sentinel for the file-deletion chokepoint: every production deletion path +//! runs against a realistic data directory, and every file it does not target +//! must survive byte-for-byte. + +use super::*; +use dash_sdk::dpp::identity::accessors::IdentityGettersV0; +use std::collections::{BTreeMap, BTreeSet}; +use std::time::{Duration, SystemTime}; + +const DAY: Duration = Duration::from_secs(24 * 60 * 60); + +fn plant(path: &Path, age: Duration) { + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, path.to_string_lossy().as_bytes()).unwrap(); + std::fs::File::options() + .write(true) + .open(path) + .unwrap() + .set_modified(SystemTime::now() - age) + .unwrap(); +} + +fn snapshot(root: &Path) -> BTreeMap> { + let mut files = BTreeMap::new(); + let mut pending = vec![root.to_owned()]; + while let Some(dir) = pending.pop() { + for entry in std::fs::read_dir(&dir).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + pending.push(path); + } else { + files.insert(path.clone(), std::fs::read(&path).unwrap()); + } + } + } + files +} + +/// Live files the testnet context itself writes while the operations run: +/// they must survive, but their bytes may legitimately change. +fn written_by_context(data_dir: &Path, path: &Path) -> bool { + if path.starts_with(data_dir.join("secrets")) { + return true; + } + let Some(name) = path.file_name().and_then(|name| name.to_str()) else { + return false; + }; + let live = [ + "det-app.sqlite", + "det-testnet.sqlite", + "det-testnet-shielded.sqlite", + "data.db", + ]; + path.parent() == Some(data_dir) + && live.iter().any(|base| { + name == *base + || ["-wal", "-shm", "-journal", ".platform-upgrade.lock"] + .iter() + .any(|suffix| name == format!("{base}{suffix}")) + }) +} + +/// Retention prune, wallet removal, identity removal, log rotation, SPV-cache +/// clear and a full testnet network clear run against a data directory holding +/// every network's databases. Only the files each operation targets disappear; +/// every other file keeps its exact bytes, and the testnet clear never reaches +/// another network's data. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn deletion_paths_leave_every_untargeted_file_intact() { + let (ctx, sender, tmp) = offline_testnet_context(); + ctx.ensure_wallet_backend(sender.clone()) + .await + .expect("ensure_wallet_backend should succeed offline"); + let dir = tmp.path(); + let old = DAY * 400; + let fresh = DAY; + + // Other networks' live databases and sidecars: never deletable. + let mut sentinels = Vec::new(); + for network in [Network::Mainnet, Network::Devnet, Network::Regtest] { + let wallet = wallet_database_path(dir, network); + let shielded = crate::wallet_backend::shielded_database_path(dir, network); + for base in [wallet, shielded] { + for suffix in ["", "-wal", "-shm", "-journal"] { + let mut name = base.as_os_str().to_owned(); + name.push(suffix); + sentinels.push(PathBuf::from(name)); + } + } + } + // Other networks' SPV data and backups, config, a pre-migration copy, + // current logs and a foreign-stem log. + sentinels.extend( + [ + "spv/mainnet/blocks/segment.dat", + "spv/mainnet/peers.dat", + "spv/mainnet/det-shielded.sqlite", + "spv/mainnet.lock", + "det-mainnet.sqlite.platform-67d4ef3-backup-old.sqlite", + "backups/auto/pre-migration-det-mainnet-1-to-2-20000101T000000Z.db", + "data.db.premigration", + "det.log", + "det-stderr.0000000001.log", + "notes.txt", + ] + .map(|relative| dir.join(relative)), + ); + for path in &sentinels { + plant(path, old); + } + // The current log is rotated to a name from its mtime; keep it recent so the + // rotated copy is retained rather than expiring in the same pass. + plant(&dir.join("det.log"), fresh); + + let targeted: BTreeSet = [ + // Retention prune: expired backups of the app and testnet databases. + "det-app.sqlite.platform-67d4ef3-backup-old.sqlite", + "det-testnet.sqlite.platform-67d4ef3-backup-old.sqlite", + "backups/auto/pre-migration-det-testnet-1-to-2-20000101T000000Z.db", + "backups/data_backup_20000101_000000.db", + // Network clear: every remaining app/testnet upgrade backup. + "det-app.sqlite.platform-67d4ef3-backup-new.sqlite", + "det-testnet.sqlite.platform-67d4ef3-backup-new.sqlite", + // Network clear (SPV cache + retired shielded files): testnet only. + "spv/testnet/blocks/segment.dat", + "spv/testnet/filters/nested/segment.dat", + "spv/testnet/peers.dat", + "spv/testnet.lock", + "spv/testnet/det-shielded.sqlite", + "spv/testnet/shielded-commitment-tree.sqlite", + // Log rotation: an expired rotated log of the rotated stem. + "det.0000000001.log", + ] + .map(|relative| dir.join(relative)) + .into_iter() + .collect(); + for path in &targeted { + let age = if path.to_string_lossy().contains("-new.") { + fresh + } else { + old + }; + plant(path, age); + } + + // Wallet and identity to remove. + let seed = [0xC4u8; 64]; + let wallet = crate::model::wallet::Wallet::new_from_seed(seed, Network::Testnet, None, None) + .expect("build wallet"); + let seed_hash = wallet.seed_hash(); + ctx.register_wallet(wallet, &seed, WalletOrigin::Fresh) + .expect("register wallet"); + let identity = wallet_owned_qualified_identity(None); + ctx.insert_local_qualified_identity(&identity, &None) + .expect("insert identity"); + + let before = snapshot(dir); + + let report = ctx.prune_expired_upgrade_backups(); + assert!(report.failure.is_none(), "{:?}", report.failure); + assert_eq!(report.deleted, 4, "retention prune"); + ctx.remove_wallet(&seed_hash).expect("remove wallet"); + ctx.delete_local_qualified_identity(&identity.identity.id()) + .expect("remove identity"); + crate::logging::rotate_log_in_dir(dir, "det"); + ctx.clear_spv_data().expect("clear SPV data"); + ctx.clear_network_database() + .await + .expect("clear network database"); + ctx.wallet_backend() + .expect("backend wired") + .shutdown() + .await; + + let after = snapshot(dir); + for path in &targeted { + assert!(!path.exists(), "{path:?} should have been deleted"); + } + for (path, bytes) in &before { + if targeted.contains(path) { + continue; + } + if written_by_context(dir, path) { + assert!(after.contains_key(path), "live file {path:?} was deleted"); + } else if path == &dir.join("det.log") { + // Rotation moves the current log aside rather than deleting it. + assert!( + after.values().any(|after| after == bytes), + "the current log's contents must survive rotation" + ); + } else { + assert_eq!(after.get(path), Some(bytes), "{path:?} must be unchanged"); + } + } + for path in &sentinels { + if path != &dir.join("det.log") { + assert!(path.exists(), "sentinel {path:?} missing"); + } + } +} diff --git a/src/database/legacy_backups.rs b/src/database/legacy_backups.rs index ea325a335..0d76536a6 100644 --- a/src/database/legacy_backups.rs +++ b/src/database/legacy_backups.rs @@ -8,6 +8,8 @@ use std::time::{Duration, SystemTime}; /// Directory, next to the database, that holds the legacy backups. pub(crate) const BACKUP_DIR: &str = "backups"; +/// The legacy database these backups copy. +const LEGACY_DATABASE: &str = "data.db"; const PREFIX: &str = "data_backup_"; const SUFFIX: &str = ".db"; /// UTC timestamp format embedded in the backup name. @@ -27,6 +29,8 @@ fn named_timestamp(name: &str) -> Option { /// Delete legacy backups in `/backups` older than `max_age`. /// +/// Deletion goes through the [`delete_file`](crate::utils::file_deletion::delete_file) +/// chokepoint, which refuses live databases, aliases and hard links. /// Only regular files with the exact legacy name are candidates; anything else, /// including symlinks and subdirectories, is left alone. Age is measured from the /// later of the name timestamp and the modification time, so a reset or skewed signal @@ -52,6 +56,7 @@ pub(crate) fn prune_expired( Err(error) => return Err(error), } let entries = std::fs::read_dir(&directory)?; + let legacy_database = data_dir.join(LEGACY_DATABASE); let mut removed = 0; let mut first_error = None; for entry in entries { @@ -67,7 +72,13 @@ pub(crate) fn prune_expired( if !crate::model::backup_retention::backup_expired(created, now, max_age) { return Ok(false); } - std::fs::remove_file(entry.path()).map(|()| true) + crate::utils::file_deletion::delete_file( + &entry.path(), + crate::utils::file_deletion::DeletionIntent::Backup { + database: &legacy_database, + }, + ) + .map(|()| true) }); match outcome { Ok(true) => removed += 1, diff --git a/src/logging.rs b/src/logging.rs index e45773404..422d31c4b 100644 --- a/src/logging.rs +++ b/src/logging.rs @@ -507,7 +507,7 @@ fn rotate_log_file() { /// Rotates `{stem}.log` in `dir` to a timestamped name and removes rotated /// copies of the same stem older than [`LOG_RETENTION_DAYS`]. -fn rotate_log_in_dir(dir: &Path, stem: &str) { +pub(crate) fn rotate_log_in_dir(dir: &Path, stem: &str) { let log_path = dir.join(format!("{stem}.log")); if log_path.exists() { let ts = fs::metadata(&log_path) @@ -516,6 +516,8 @@ fn rotate_log_in_dir(dir: &Path, stem: &str) { .unwrap_or_else(|_| Local::now()) .timestamp(); let rotated = dir.join(rotated_name(stem, ts)); + // Not a deletion, so not routed through `delete_file`: it moves the + // current log aside under a fresh timestamped name. let _ = fs::rename(&log_path, rotated); } @@ -531,7 +533,10 @@ fn rotate_log_in_dir(dir: &Path, stem: &str) { if let Some(ts) = parse_rotated_ts(name, stem) && ts < cutoff { - let _ = fs::remove_file(path); + let _ = crate::utils::file_deletion::delete_file( + &path, + crate::utils::file_deletion::DeletionIntent::Log { dir, stem }, + ); } } } diff --git a/src/utils/file_deletion.rs b/src/utils/file_deletion.rs new file mode 100644 index 000000000..5c6038f8a --- /dev/null +++ b/src/utils/file_deletion.rs @@ -0,0 +1,322 @@ +//! The single chokepoint for deleting files from disk. +//! +//! Every production file deletion goes through [`delete_file`] (or +//! [`delete_tree`], which unlinks each file through [`delete_file`]). A deletion +//! states its purpose as a [`DeletionIntent`], and the chokepoint enforces, in +//! this order, before anything is unlinked: +//! +//! 1. the target is a regular file — never a symlink or directory; +//! 2. a hard deny-list of live application files by **name** — `det-app.sqlite`, +//! `data.db`, every network's wallet and shielded database, `*.pwsvault`, +//! `.env`, `*.premigration`, each with its `-wal`/`-shm`/`-journal` sidecar. +//! It depends on no caller-supplied path, so a caller's matcher bug or a +//! wrong data directory can never reach these files; +//! 3. the same deny-list by **identity** — the canonical location, and on Unix +//! the device/inode, of every protected file in the intent's data directory, +//! which catches `..`, symlinked-directory and case aliases; +//! 4. on Unix, a single hard link, so no alias of another file is removed; +//! 5. the intent's scope — the directories that purpose may delete from. +//! +//! The chokepoint needs no `AppContext`: each intent carries the paths it is +//! confined to, so it also serves pre-context callers such as log rotation. +//! +//! A check and the unlink that follows it are not atomic; a concurrent rename in +//! the window is out of scope. The data directory is owner-only. + +use dash_sdk::dpp::dashcore::Network; +use std::path::{Path, PathBuf}; + +/// SQLite sidecars that belong to a database file and share its protection. +const SQLITE_SIDECAR_SUFFIXES: [&str; 3] = ["-wal", "-shm", "-journal"]; + +/// Why a deletion needs to happen, and therefore where it may reach. +#[derive(Debug, Clone, Copy)] +pub(crate) enum DeletionIntent<'a> { + /// A retained backup of `database`, inside the database's directory, its + /// `backups/` directory, or the upstream auto-backup directory. + Backup { + /// The live database the backup was taken from. + database: &'a Path, + }, + /// Resyncable per-network data removed by a network clear: anything under + /// `/spv//`, plus the `/spv/.lock` + /// storage lock. + NetworkClear { + /// The application data directory. + data_dir: &'a Path, + /// The network whose data is being cleared. + network: Network, + }, + /// A rotated log file named `..log` directly in `dir`. + Log { + /// The directory holding the logs; also the application data directory. + dir: &'a Path, + /// The log's base name, e.g. `det`. + stem: &'a str, + }, +} + +/// A deletion the chokepoint refused. Wrapped in an [`std::io::Error`], so +/// callers can downcast it from the error's inner value. +#[derive(Debug, thiserror::Error)] +pub(crate) enum DeletionRefused { + /// The target is a live application file (database, sidecar, vault, config). + #[error("A protected application file was not deleted.")] + ProtectedFile { + /// The refused target. + path: PathBuf, + }, + /// The target is a symlink, directory, or other non-regular file. + #[error("Only regular files can be deleted.")] + NotRegularFile { + /// The refused target. + path: PathBuf, + }, + /// The target has more than one hard link. + #[error("A file with more than one hard link was not deleted.")] + HardLinked { + /// The refused target. + path: PathBuf, + }, + /// The target lies outside what the deletion's intent may reach. + #[error("A file outside the permitted location was not deleted.")] + OutsideScope { + /// The refused target. + path: PathBuf, + }, +} + +impl From for std::io::Error { + fn from(refused: DeletionRefused) -> Self { + std::io::Error::other(refused) + } +} + +/// Delete the regular file at `path` if every chokepoint check passes. +/// +/// # Errors +/// +/// A refused deletion is an [`std::io::Error`] wrapping [`DeletionRefused`]. A +/// missing target is [`std::io::ErrorKind::NotFound`], so callers that tolerate +/// an already-gone file keep doing so. Other I/O failures pass through. +pub(crate) fn delete_file(path: &Path, intent: DeletionIntent<'_>) -> std::io::Result<()> { + check(path, intent)?; + std::fs::remove_file(path) +} + +/// Delete the directory tree at `dir`, unlinking every file through +/// [`delete_file`] and then removing the emptied directories. +/// +/// Symlinks are never followed; one inside the tree is refused like any other +/// non-regular file. Stops at the first failure. +/// +/// # Errors +/// +/// As [`delete_file`]; a missing `dir` is [`std::io::ErrorKind::NotFound`], and +/// a `dir` that is not a real directory is refused as [`DeletionRefused::NotRegularFile`]. +pub(crate) fn delete_tree(dir: &Path, intent: DeletionIntent<'_>) -> std::io::Result<()> { + if !std::fs::symlink_metadata(dir)?.is_dir() { + return Err(DeletionRefused::NotRegularFile { + path: dir.to_owned(), + } + .into()); + } + for entry in std::fs::read_dir(dir)? { + let path = entry?.path(); + if std::fs::symlink_metadata(&path)?.is_dir() { + delete_tree(&path, intent)?; + } else { + delete_file(&path, intent)?; + } + } + // Only an empty directory can be removed here; every file went through the checks. + std::fs::remove_dir(dir) +} + +fn check(path: &Path, intent: DeletionIntent<'_>) -> std::io::Result<()> { + let refused = + |make: fn(PathBuf) -> DeletionRefused| -> std::io::Error { make(path.to_owned()).into() }; + let metadata = std::fs::symlink_metadata(path)?; + if !metadata.is_file() { + return Err(refused(|path| DeletionRefused::NotRegularFile { path })); + } + let name = path + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| refused(|path| DeletionRefused::OutsideScope { path }))?; + if is_protected_name(name) { + return Err(refused(|path| DeletionRefused::ProtectedFile { path })); + } + let located = canonical_location(path)?; + let protected = protected_paths(intent); + if protected + .iter() + .any(|file| same_location(&located, file) || same_inode(&metadata, file)) + { + return Err(refused(|path| DeletionRefused::ProtectedFile { path })); + } + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + match metadata.nlink() { + 1 => {} + // Unlinked by a concurrent deletion between lookup and stat. + 0 => return Err(std::io::ErrorKind::NotFound.into()), + _ => return Err(refused(|path| DeletionRefused::HardLinked { path })), + } + } + if !in_scope(&located, name, intent) { + return Err(refused(|path| DeletionRefused::OutsideScope { path })); + } + Ok(()) +} + +/// Every network, via an exhaustive match: a new [`Network`] variant fails to +/// compile here until its databases are added to the deny-list. +fn all_networks() -> [Network; 4] { + let _exhaustive = |network: Network| match network { + Network::Mainnet | Network::Testnet | Network::Devnet | Network::Regtest => (), + }; + [ + Network::Mainnet, + Network::Testnet, + Network::Devnet, + Network::Regtest, + ] +} + +/// Whether `name` is a live application file, regardless of its directory. +fn is_protected_name(name: &str) -> bool { + let name = name.to_ascii_lowercase(); + let base = SQLITE_SIDECAR_SUFFIXES + .iter() + .find_map(|suffix| name.strip_suffix(suffix)) + .unwrap_or(&name); + if matches!(base, "det-app.sqlite" | "data.db" | ".env") + || base.ends_with(".pwsvault") + || base.ends_with(".premigration") + { + return true; + } + all_networks().into_iter().any(|network| { + let prefix = crate::wallet_backend::network_prefix(network); + base == format!("det-{prefix}.sqlite") || base == format!("det-{prefix}-shielded.sqlite") + }) +} + +/// The live files in the intent's data directory, with their SQLite sidecars. +fn protected_paths(intent: DeletionIntent<'_>) -> Vec { + let (data_dir, database) = match intent { + DeletionIntent::Backup { database } => (parent_of(database), Some(database)), + DeletionIntent::NetworkClear { data_dir, .. } => (data_dir, None), + DeletionIntent::Log { dir, .. } => (dir, None), + }; + let mut roots = vec![ + data_dir.join("det-app.sqlite"), + data_dir.join("data.db"), + data_dir.join(".env"), + crate::context::AppContext::secret_store_path(data_dir), + ]; + for network in all_networks() { + roots.push(crate::wallet_backend::wallet_database_path( + data_dir, network, + )); + roots.push(crate::wallet_backend::shielded_database_path( + data_dir, network, + )); + } + roots.extend(database.map(Path::to_owned)); + let mut protected = Vec::with_capacity(roots.len() * (SQLITE_SIDECAR_SUFFIXES.len() + 1)); + for root in roots { + for suffix in SQLITE_SIDECAR_SUFFIXES { + let mut name = root.as_os_str().to_owned(); + name.push(suffix); + protected.push(PathBuf::from(name)); + } + protected.push(root); + } + protected +} + +/// `path` with its parent directory resolved, keeping the final component as is +/// so a symlink target is never followed. +fn canonical_location(path: &Path) -> std::io::Result { + let absolute = std::path::absolute(path)?; + let name = absolute + .file_name() + .ok_or_else(|| std::io::Error::from(std::io::ErrorKind::InvalidInput))?; + Ok(parent_of(&absolute).canonicalize()?.join(name)) +} + +fn parent_of(path: &Path) -> &Path { + match path.parent() { + Some(parent) if !parent.as_os_str().is_empty() => parent, + _ => Path::new("."), + } +} + +/// `dir` resolved when it exists; otherwise as given (so it matches nothing real). +fn canonical_dir(dir: &Path) -> PathBuf { + dir.canonicalize().unwrap_or_else(|_| dir.to_owned()) +} + +fn same_location(located: &Path, protected: &Path) -> bool { + canonical_location(protected).is_ok_and(|protected| { + protected + .as_os_str() + .eq_ignore_ascii_case(located.as_os_str()) + }) +} + +#[cfg(unix)] +fn same_inode(target: &std::fs::Metadata, protected: &Path) -> bool { + use std::os::unix::fs::MetadataExt; + std::fs::metadata(protected) + .is_ok_and(|protected| protected.dev() == target.dev() && protected.ino() == target.ino()) +} + +#[cfg(not(unix))] +fn same_inode(_target: &std::fs::Metadata, _protected: &Path) -> bool { + false +} + +fn in_scope(located: &Path, name: &str, intent: DeletionIntent<'_>) -> bool { + let Some(parent) = located.parent() else { + return false; + }; + match intent { + DeletionIntent::Backup { database } => { + let directory = parent_of(database); + [ + directory.to_owned(), + directory.join(crate::database::legacy_backups::BACKUP_DIR), + platform_wallet_storage::default_auto_backup_dir(database), + ] + .iter() + .any(|allowed| canonical_dir(allowed) == parent) + } + DeletionIntent::NetworkClear { data_dir, network } => { + let spv = data_dir.join("spv"); + let prefix = crate::wallet_backend::network_prefix(network); + let lock = format!("{prefix}.lock"); + parent.starts_with(canonical_dir(&spv.join(prefix))) + || (parent == canonical_dir(&spv) && name == lock) + } + DeletionIntent::Log { dir, stem } => { + parent == canonical_dir(dir) && is_rotated_log_name(name, stem) + } + } +} + +/// Whether `name` is `..log`. +fn is_rotated_log_name(name: &str, stem: &str) -> bool { + !stem.is_empty() + && name + .strip_prefix(stem) + .and_then(|rest| rest.strip_prefix('.')) + .and_then(|rest| rest.strip_suffix(".log")) + .is_some_and(|digits| !digits.is_empty() && digits.bytes().all(|b| b.is_ascii_digit())) +} + +#[cfg(test)] +mod tests; diff --git a/src/utils/file_deletion/tests.rs b/src/utils/file_deletion/tests.rs new file mode 100644 index 000000000..d49814fdd --- /dev/null +++ b/src/utils/file_deletion/tests.rs @@ -0,0 +1,439 @@ +use super::*; + +/// Every protected base name in a data directory, as the app lays them out. +fn protected_files(data_dir: &Path) -> Vec { + let mut files = vec![ + data_dir.join("det-app.sqlite"), + data_dir.join("data.db"), + data_dir.join(".env"), + data_dir.join("secrets/det-secrets.pwsvault"), + data_dir.join("data.db.premigration"), + ]; + for network in all_networks() { + files.push(crate::wallet_backend::wallet_database_path( + data_dir, network, + )); + files.push(crate::wallet_backend::shielded_database_path( + data_dir, network, + )); + } + let with_sidecars: Vec = files + .iter() + .flat_map(|file| { + SQLITE_SIDECAR_SUFFIXES.iter().map(move |suffix| { + let mut name = file.as_os_str().to_owned(); + name.push(suffix); + PathBuf::from(name) + }) + }) + .collect(); + files.extend(with_sidecars); + for file in &files { + std::fs::create_dir_all(file.parent().unwrap()).unwrap(); + std::fs::write(file, b"live").unwrap(); + } + files +} + +/// Every intent, each scoped to reach `data_dir` as widely as it can. +fn intents<'a>(data_dir: &'a Path, app_db: &'a Path) -> Vec> { + vec![ + DeletionIntent::Backup { database: app_db }, + DeletionIntent::NetworkClear { + data_dir, + network: Network::Testnet, + }, + DeletionIntent::Log { + dir: data_dir, + stem: "det", + }, + ] +} + +fn refusal(error: &std::io::Error) -> &DeletionRefused { + error + .get_ref() + .and_then(|inner| inner.downcast_ref::()) + .unwrap_or_else(|| panic!("expected a refusal, got {error:?}")) +} + +fn assert_protected(result: std::io::Result<()>, what: &str) { + let error = result.expect_err(what); + assert!( + matches!(refusal(&error), DeletionRefused::ProtectedFile { .. }), + "{what}: {error:?}" + ); +} + +#[test] +fn delete_file_refuses_every_protected_file_under_every_intent() { + let dir = tempfile::tempdir().unwrap(); + let files = protected_files(dir.path()); + let app_db = dir.path().join("det-app.sqlite"); + for intent in intents(dir.path(), &app_db) { + for file in &files { + assert_protected(delete_file(file, intent), &format!("{intent:?} {file:?}")); + assert!(file.exists(), "{file:?} must survive"); + } + } +} + +#[test] +fn delete_file_refuses_protected_names_in_any_directory() { + let dir = tempfile::tempdir().unwrap(); + // A protected name inside the network-clear scope is still refused. + let spv = dir.path().join("spv/testnet"); + std::fs::create_dir_all(&spv).unwrap(); + for name in [ + "det-testnet.sqlite", + "DET-MAINNET.SQLITE-wal", + "det-app.sqlite-journal", + "wallet.db.premigration", + "other.pwsvault", + ] { + let file = spv.join(name); + std::fs::write(&file, b"live").unwrap(); + assert_protected( + delete_file( + &file, + DeletionIntent::NetworkClear { + data_dir: dir.path(), + network: Network::Testnet, + }, + ), + name, + ); + assert!(file.exists(), "{name} must survive"); + } +} + +#[test] +fn delete_file_refuses_protected_files_via_dot_dot_paths() { + let dir = tempfile::tempdir().unwrap(); + let files = protected_files(dir.path()); + std::fs::create_dir_all(dir.path().join("backups/auto")).unwrap(); + let app_db = dir.path().join("det-app.sqlite"); + for intent in intents(dir.path(), &app_db) { + for file in &files { + let relative = file.strip_prefix(dir.path()).unwrap(); + let alias = dir.path().join("backups/auto/../..").join(relative); + assert_protected(delete_file(&alias, intent), &format!("{alias:?}")); + assert!(file.exists()); + } + } +} + +#[cfg(unix)] +#[test] +fn delete_file_refuses_protected_files_via_symlinks() { + let dir = tempfile::tempdir().unwrap(); + let files = protected_files(dir.path()); + let backups = dir.path().join("backups"); + std::fs::create_dir_all(&backups).unwrap(); + // A symlink to the live file, named like an ordinary backup. + let link = backups.join("data_backup_20000101_000000.db"); + // A symlinked directory aliasing the data directory. + let dir_link = dir.path().join("spv"); + std::os::unix::fs::symlink(dir.path(), &dir_link).unwrap(); + for file in &files { + let _ = std::fs::remove_file(&link); + std::os::unix::fs::symlink(file, &link).unwrap(); + let database = dir.path().join("data.db"); + let error = delete_file( + &link, + DeletionIntent::Backup { + database: &database, + }, + ) + .expect_err("a symlink must be refused"); + assert!(matches!( + refusal(&error), + DeletionRefused::NotRegularFile { .. } + )); + + let through_dir = dir_link.join(file.strip_prefix(dir.path()).unwrap()); + for intent in intents(dir.path(), &database) { + assert_protected( + delete_file(&through_dir, intent), + &format!("{through_dir:?}"), + ); + } + assert!(file.exists()); + } +} + +#[cfg(unix)] +#[test] +fn delete_file_refuses_hard_links_to_protected_files() { + let dir = tempfile::tempdir().unwrap(); + let files = protected_files(dir.path()); + let auto = dir.path().join("backups/auto"); + std::fs::create_dir_all(&auto).unwrap(); + let alias = auto.join("pre-migration-det-app-1-to-2-20000101T000000Z.db"); + let database = dir.path().join("det-app.sqlite"); + for file in &files { + let _ = std::fs::remove_file(&alias); + std::fs::hard_link(file, &alias).unwrap(); + let error = delete_file( + &alias, + DeletionIntent::Backup { + database: &database, + }, + ) + .expect_err("a hard link to a protected file must be refused"); + // Files protected by exact location are matched by inode; the rest + // (e.g. `*.premigration`, protected by name only) by the link count. + assert!( + matches!( + refusal(&error), + DeletionRefused::ProtectedFile { .. } | DeletionRefused::HardLinked { .. } + ), + "hard link to {file:?}: {error:?}" + ); + assert!(alias.exists() && file.exists()); + } +} + +#[cfg(unix)] +#[test] +fn delete_file_refuses_any_hard_linked_file() { + let dir = tempfile::tempdir().unwrap(); + let database = dir.path().join("det-app.sqlite"); + let backup = dir + .path() + .join("det-app.sqlite.platform-67d4ef3-backup-a.sqlite"); + std::fs::write(&backup, b"copy").unwrap(); + std::fs::hard_link(&backup, dir.path().join("elsewhere")).unwrap(); + let error = delete_file( + &backup, + DeletionIntent::Backup { + database: &database, + }, + ) + .unwrap_err(); + assert!(matches!( + refusal(&error), + DeletionRefused::HardLinked { .. } + )); + assert!(backup.exists()); +} + +#[test] +fn delete_file_refuses_targets_outside_the_intent_scope() { + let dir = tempfile::tempdir().unwrap(); + let database = dir.path().join("det-app.sqlite"); + let cases = [ + ( + dir.path() + .join("elsewhere/det-app.sqlite.platform-67d4ef3-backup-a.sqlite"), + DeletionIntent::Backup { + database: &database, + }, + ), + ( + dir.path().join("spv/mainnet/peers.dat"), + DeletionIntent::NetworkClear { + data_dir: dir.path(), + network: Network::Testnet, + }, + ), + ( + dir.path().join("spv/mainnet.lock"), + DeletionIntent::NetworkClear { + data_dir: dir.path(), + network: Network::Testnet, + }, + ), + ( + dir.path().join("notes.txt"), + DeletionIntent::NetworkClear { + data_dir: dir.path(), + network: Network::Testnet, + }, + ), + ( + dir.path().join("det.log"), + DeletionIntent::Log { + dir: dir.path(), + stem: "det", + }, + ), + ( + dir.path().join("other.0000000001.log"), + DeletionIntent::Log { + dir: dir.path(), + stem: "det", + }, + ), + ( + dir.path().join("logs/det.0000000001.log"), + DeletionIntent::Log { + dir: dir.path(), + stem: "det", + }, + ), + ]; + for (file, intent) in cases { + std::fs::create_dir_all(file.parent().unwrap()).unwrap(); + std::fs::write(&file, b"keep").unwrap(); + let error = delete_file(&file, intent).unwrap_err(); + assert!( + matches!(refusal(&error), DeletionRefused::OutsideScope { .. }), + "{file:?}: {error:?}" + ); + assert!(file.exists(), "{file:?}"); + } +} + +#[test] +fn delete_file_deletes_targets_in_scope() { + let dir = tempfile::tempdir().unwrap(); + let database = dir.path().join("det-testnet.sqlite"); + let cases = [ + ( + dir.path() + .join("det-testnet.sqlite.platform-67d4ef3-backup-a.sqlite"), + DeletionIntent::Backup { + database: &database, + }, + ), + ( + dir.path().join("backups/data_backup_20000101_000000.db"), + DeletionIntent::Backup { + database: &database, + }, + ), + ( + dir.path() + .join("backups/auto/pre-migration-det-testnet-1-to-2-20000101T000000Z.db"), + DeletionIntent::Backup { + database: &database, + }, + ), + ( + dir.path().join("spv/testnet/filters/segment.dat"), + DeletionIntent::NetworkClear { + data_dir: dir.path(), + network: Network::Testnet, + }, + ), + ( + dir.path().join("spv/testnet.lock"), + DeletionIntent::NetworkClear { + data_dir: dir.path(), + network: Network::Testnet, + }, + ), + ( + dir.path().join("det.0000000001.log"), + DeletionIntent::Log { + dir: dir.path(), + stem: "det", + }, + ), + ]; + for (file, intent) in cases { + std::fs::create_dir_all(file.parent().unwrap()).unwrap(); + std::fs::write(&file, b"expendable").unwrap(); + delete_file(&file, intent).unwrap_or_else(|e| panic!("{file:?}: {e:?}")); + assert!(!file.exists(), "{file:?}"); + } +} + +#[test] +fn delete_file_reports_a_missing_target_as_not_found() { + let dir = tempfile::tempdir().unwrap(); + let error = delete_file( + &dir.path().join("spv/testnet/peers.dat"), + DeletionIntent::NetworkClear { + data_dir: dir.path(), + network: Network::Testnet, + }, + ) + .unwrap_err(); + assert_eq!(error.kind(), std::io::ErrorKind::NotFound); +} + +#[test] +fn delete_file_refuses_directories() { + let dir = tempfile::tempdir().unwrap(); + let target = dir.path().join("spv/testnet/blocks"); + std::fs::create_dir_all(&target).unwrap(); + let error = delete_file( + &target, + DeletionIntent::NetworkClear { + data_dir: dir.path(), + network: Network::Testnet, + }, + ) + .unwrap_err(); + assert!(matches!( + refusal(&error), + DeletionRefused::NotRegularFile { .. } + )); + assert!(target.is_dir()); +} + +#[test] +fn delete_tree_removes_a_cache_tree_but_stops_at_a_protected_file() { + let dir = tempfile::tempdir().unwrap(); + let intent = DeletionIntent::NetworkClear { + data_dir: dir.path(), + network: Network::Testnet, + }; + let cache = dir.path().join("spv/testnet/blocks"); + std::fs::create_dir_all(cache.join("nested")).unwrap(); + std::fs::write(cache.join("a.dat"), b"x").unwrap(); + std::fs::write(cache.join("nested/b.dat"), b"x").unwrap(); + delete_tree(&cache, intent).unwrap(); + assert!(!cache.exists()); + + std::fs::create_dir_all(&cache).unwrap(); + let planted = cache.join("det-app.sqlite"); + std::fs::write(&planted, b"live").unwrap(); + assert_protected(delete_tree(&cache, intent), "planted live database"); + assert!(planted.exists()); +} + +#[cfg(unix)] +#[test] +fn delete_tree_never_follows_a_symlinked_directory() { + let dir = tempfile::tempdir().unwrap(); + let outside = dir.path().join("outside"); + std::fs::create_dir_all(&outside).unwrap(); + std::fs::write(outside.join("keep.dat"), b"keep").unwrap(); + let cache = dir.path().join("spv/testnet/blocks"); + std::fs::create_dir_all(&cache).unwrap(); + std::os::unix::fs::symlink(&outside, cache.join("link")).unwrap(); + let intent = DeletionIntent::NetworkClear { + data_dir: dir.path(), + network: Network::Testnet, + }; + let error = delete_tree(&cache, intent).unwrap_err(); + assert!(matches!( + refusal(&error), + DeletionRefused::NotRegularFile { .. } + )); + assert!(outside.join("keep.dat").exists()); + let error = delete_tree(&cache.join("link"), intent).unwrap_err(); + assert!(matches!( + refusal(&error), + DeletionRefused::NotRegularFile { .. } + )); +} + +#[test] +fn rotated_log_names_are_exact() { + assert!(is_rotated_log_name("det.0000000001.log", "det")); + for name in [ + "det.log", + "det..log", + "det.12a.log", + "det.1.log.bak", + "detx.1.log", + "other.1.log", + ] { + assert!(!is_rotated_log_name(name, "det"), "{name}"); + } + assert!(!is_rotated_log_name(".1.log", "")); +} diff --git a/src/utils/mod.rs b/src/utils/mod.rs index d81e1a8bf..7292c1e36 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -1,3 +1,4 @@ pub mod egui_mpsc; +pub(crate) mod file_deletion; pub mod parsers; pub mod tasks; diff --git a/src/wallet_backend/platform_compatibility/engine.rs b/src/wallet_backend/platform_compatibility/engine.rs index 0c833d255..17fcf262d 100644 --- a/src/wallet_backend/platform_compatibility/engine.rs +++ b/src/wallet_backend/platform_compatibility/engine.rs @@ -380,7 +380,10 @@ fn validate_backup_file(backup: &Path, database: &Path) -> std::io::Result<()> { fn remove_backup(backup: &Path, database: &Path) -> std::io::Result<()> { validate_backup_file(backup, database)?; - std::fs::remove_file(backup) + crate::utils::file_deletion::delete_file( + backup, + crate::utils::file_deletion::DeletionIntent::Backup { database }, + ) } pub(super) struct BackupGuard { @@ -681,6 +684,9 @@ fn backup_with_hook_locked( let path = &guard.path; let parent = path.parent().ok_or(UpgradeError::Unrecognized)?; let prefix = backup_prefix(path).ok_or(UpgradeError::Unrecognized)?; + // Not routed through `delete_file`: on failure the temp file removes only + // the uniquely named `.pending` file it created itself, which keeps the + // cleanup tied to drop on every error path. let file = tempfile::Builder::new() .prefix(&prefix) .suffix(".pending") From fbbb7f7482c06e684ca306ed5526408f0a9b2e07 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Wed, 30 Sep 2026 09:30:08 +0000 Subject: [PATCH 21/30] fix(storage): honor backup retention globally and harden the deletion chokepoint Published upgrade snapshots are now deleted only by the user's time-based retention: opening a database removes just crash-left `.pending` copies and byte-identical duplicates of a newer snapshot of the same migration (size check, then a streamed comparison), so "Keep forever" keeps every distinct snapshot while repeated failed upgrades no longer pile up copies. Retention is one global policy for every database in the data directory (app database and every network's wallet database, plus legacy data.db backups) and always keeps the newest backup of each database, whatever its age or the clock. A database whose lifecycle lock is held is skipped, and one without backups is not locked at all. A network clear removes only that network's wallet backups; det-app.sqlite backups stay under retention. Deletion chokepoint: Backup intents allow only the exact backup name grammar of their database; delete_tree scope-checks every directory and walks iteratively; a network clear unlinks symlinked SPV entries without following them; Windows refuses alias-prone names; clippy.toml disallows raw fs remove calls. One shared prune helper, one storage-failure classifier, shared file-name constants, a single rotated-log grammar. UI: out-of-range retention days stay visible with an error instead of being clamped and saved, singular confirmation copy, the form re-syncs on arrival so a lost result cannot wedge it, and cleanup failures carry details. Docs (CHANGELOG, WAL-034, upgrade notes, AGENTS.md) describe the new behaviour. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- CHANGELOG.md | 64 ++- Cargo.lock | 1 - Cargo.toml | 2 - clippy.toml | 9 + .../2026-09-10-platform-pin/upgrade-notes.md | 33 +- docs/user-stories.md | 23 +- src/app.rs | 3 +- src/app_dir.rs | 5 +- src/backend_task/error.rs | 51 +-- src/backend_task/migration/finish_unwire.rs | 4 + src/backend_task/mod.rs | 4 +- src/backend_task/system_task/mod.rs | 2 +- src/config.rs | 15 +- src/context/mod.rs | 2 +- src/context/wallet_lifecycle/removal.rs | 33 +- src/context/wallet_lifecycle/retention.rs | 134 +++++- src/context/wallet_lifecycle/tests.rs | 83 +++- .../tests/deletion_sentinel.rs | 70 ++- src/database/legacy_backups.rs | 102 +++-- src/logging.rs | 19 +- src/mcp/server.rs | 2 +- src/ui/network_chooser_screen.rs | 94 +++- src/ui/tokens/tokens_screen/mod.rs | 12 + src/utils/backup_prune.rs | 81 ++++ src/utils/file_deletion.rs | 221 ++++++--- src/utils/file_deletion/tests.rs | 187 +++++++- src/utils/mod.rs | 1 + src/wallet_backend/mod.rs | 22 + .../platform_compatibility/engine.rs | 432 +++++++++--------- .../platform_compatibility/engine/tests.rs | 288 ++++++------ .../platform_compatibility/mod.rs | 165 +++---- .../platform_compatibility/storage_failure.rs | 101 ++++ tests/backend-e2e/spv_reconnect.rs | 4 + 34 files changed, 1522 insertions(+), 749 deletions(-) create mode 100644 clippy.toml create mode 100644 src/utils/backup_prune.rs create mode 100644 src/wallet_backend/platform_compatibility/storage_failure.rs diff --git a/AGENTS.md b/AGENTS.md index 2dc5c20fc..3f5ce5989 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -114,7 +114,7 @@ Code lives by responsibility, not convenience: - **`ui/components/`** — reusable **Component-pattern widgets ONLY**: a `show()` plus a `ComponentResponse`, a display-only render widget, or component infrastructure. If it does not render egui, it is not a component. - **`ui/state/`** — non-widget UI state: per-screen view-models and async fetch-state caches (e.g. `TrackedAssetLockCache`). Owned by screens, may return `BackendTask`, render nothing. - **`src/mcp/tools/`** — MCP tool logic, one file per domain (e.g. `wallet.rs`, `shielded.rs`, `identity.rs`) with multiple tool structs per file; never in `src/bin/det_cli/`. -- **`src/utils/file_deletion.rs`** — the single file-deletion chokepoint. Every production on-disk deletion goes through `delete_file` (or `delete_tree`) with a `DeletionIntent` stating its purpose; the chokepoint hard-refuses live databases, their sidecars, the vault and `.env`. Any deliberate exception gets a code comment with the reason at its call site. +- **`src/utils/file_deletion.rs`** — the single file-deletion chokepoint. Every production on-disk deletion goes through `delete_file` (or `delete_tree`) with a `DeletionIntent` stating its purpose; the chokepoint hard-refuses live databases, their sidecars, the vault and `.env`. `clippy.toml` disallows raw `std::fs`/`tokio::fs` remove calls; any deliberate exception gets `#[expect(clippy::disallowed_methods, reason = "...")]` at its call site. - **`src/localization.rs`** — localization logic. **`src/ui/theme.rs`** — theme/alignment helpers. Discriminator for `ui/components/` vs `ui/state/`: *does it render egui (`show`/`ui`/a render fn)?* Yes → component. No → state. diff --git a/CHANGELOG.md b/CHANGELOG.md index dc4855603..047074f88 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,43 +46,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). Allowing 3.x needs an upstream change in `dashpay/rust-dashcore` first. A TODO in `Cargo.toml` marks the re-check. -- **Safer guidance for wallet data this version cannot open**: the message no - longer tells you to remove your local wallet data, which could have deleted - the `secrets` folder along with keys no recovery phrase can restore. It now - asks you to write down your recovery phrases and imported keys in the version - you used before, close every running instance, then set aside the `.sqlite` - files together with their matching `-wal` and `-shm` files and keep the - `secrets` folder. Imported private keys are listed separately from recovery phrases. - A database temporarily held by another session now asks you to close that - session and try again instead of reporting incompatible data. - -- **Upgrade backups no longer pile up, and old ones expire automatically**: a - failed database upgrade no longer leaves a new backup on every attempt. Each - database keeps at most one upgrade backup, and it is deleted when you clear - a network's data. Upgrade backups older than 90 days are - deleted automatically once the app has opened your data successfully; change - the period or keep backups forever under Settings → Upgrade Backups (a shorter - period asks for confirmation, since it deletes backups at once). The same - retention covers upstream pre-migration snapshots and legacy `data.db` - migration backups. Removing a wallet or an identity does not touch upgrade - backups, since they also hold other wallets' data. - An upgrade that can never succeed now shows its message instead of being - retried silently. Cleanup rejects links and non-files, reports pruning - failures, and removes unfinished snapshots left by a crash. Temporary storage and memory - failures remain retryable; permission and invalid-input failures surface directly. - Concurrent sessions now protect active snapshots from cleanup, and staged - validation preserves retry options and guidance for temporary resource failures. - One lock now covers the complete open, upgrade, backup, and retention cycle. - A backup cleanup problem no longer stops up-to-date wallet data from opening; - it blocks only an upgrade that would create a new backup. A new upgrade backup - replaces the previous one only after it is saved. The retained backup - is always one that opens and passes an integrity check, so an unfinished copy - never replaces a complete one. Clearing a network's data still removes upgrade - backups when an old shielded file cannot be deleted, and reports the clear as - incomplete. - ### Added +- **Upgrade backup retention setting**: Settings → Upgrade Backups controls how + long the backup copies taken before a storage upgrade are kept. By default, + backups older than 90 days are now deleted automatically; choose another + period (1 to 3650 days) or keep them forever. The newest backup of each + database is always kept, and a shorter period asks for confirmation because + it deletes backups at once. + - **Add wallet-created identity keys**: for identities loaded from a wallet on this device, the Add Key screen defaults to "Create from wallet", which creates an ECDSA_SECP256K1 or ECDSA_HASH160 key from that wallet so it can be @@ -182,6 +154,28 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Fixed +- **Safer guidance for wallet data this version cannot open**: the message no + longer tells you to remove your local wallet data, which could have deleted + the `secrets` folder along with keys no recovery phrase can restore. It asks + you to write down your recovery phrases and imported keys in the version you + used before, close every running instance, and set aside the `.sqlite` files + while keeping the `secrets` folder. A database held by another session asks + you to close that session and try again. + +- **Upgrade backups no longer pile up**: repeated failed upgrades no longer + leave an identical backup on every attempt, and a failed upgrade that can + never succeed shows its message instead of retrying silently. Clearing a + network's data removes that network's upgrade backups; backups of the shared + app data are left to the retention setting. Removing a wallet or an identity + keeps upgrade backups. + +- **Clearing a network's data works with moved chain data**: chain-sync data + that was moved to another disk and linked back no longer stops the clear. + +- **Clearer identity funding errors**: when the wallet cannot save the data a + payment needs, the message now distinguishes a busy or full store from other + failures and says how to retry. + - **Identity keys added by hand no longer disappear**: unlocking a wallet, starting the app or loading an identity from a wallet refreshed the identity with only the keys the wallet can recreate, so keys you had pasted or diff --git a/Cargo.lock b/Cargo.lock index 1243cc6a6..00d31d4ed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2150,7 +2150,6 @@ dependencies = [ "tz-rs", "unicode-general-category", "unicode-segmentation", - "url", "urlencoding", "which 8.0.6", "winres", diff --git a/Cargo.toml b/Cargo.toml index 0998112f0..9a93d0c0a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -119,8 +119,6 @@ tz-rs = { version = "0.7.0" } tempfile = "3.20.0" arc-swap = "1" urlencoding = "2" -# Builds SQLite `file:` URIs (percent-encoded, cross-platform); already a transitive dependency. -url = "2.5" rmcp = { version = "3.3", optional = true, default-features = false } axum = { version = "0.8", default-features = false, features = ["http1", "tokio", "json"], optional = true } # Constant-time equality for `model::secret::Secret` (unconditional) and the diff --git a/clippy.toml b/clippy.toml new file mode 100644 index 000000000..257b2af12 --- /dev/null +++ b/clippy.toml @@ -0,0 +1,9 @@ +# Every production file deletion goes through `src/utils/file_deletion.rs`. +disallowed-methods = [ + { path = "std::fs::remove_file", reason = "delete through utils::file_deletion::delete_file" }, + { path = "std::fs::remove_dir", reason = "delete through utils::file_deletion::delete_tree" }, + { path = "std::fs::remove_dir_all", reason = "delete through utils::file_deletion::delete_tree" }, + { path = "tokio::fs::remove_file", reason = "delete through utils::file_deletion::delete_file" }, + { path = "tokio::fs::remove_dir", reason = "delete through utils::file_deletion::delete_tree" }, + { path = "tokio::fs::remove_dir_all", reason = "delete through utils::file_deletion::delete_tree" }, +] diff --git a/docs/ai-design/2026-09-10-platform-pin/upgrade-notes.md b/docs/ai-design/2026-09-10-platform-pin/upgrade-notes.md index 4d51b51de..514691fa8 100644 --- a/docs/ai-design/2026-09-10-platform-pin/upgrade-notes.md +++ b/docs/ai-design/2026-09-10-platform-pin/upgrade-notes.md @@ -61,19 +61,26 @@ preserved. An ambiguous or malformed saved identity roster fails closed. The encrypted seed vault is not migrated or rewritten. The bridge takes its backup only after the converted copy validates, just before -the rebuild, and keeps at most one backup per database. Retention includes upstream -`backups/auto/pre-migration--*.db` snapshots and runs before and after -storage opens, including failed opens. Pruning errors stop the operation; backup -candidates must have the exact database-specific name format and be regular files, -not symbolic links (or hard links on Unix). Clearing a network's data also -deletes these backups, because they copy wallet and identity history. Removing a -single wallet or identity does not: the backups also hold other wallets' data. -Instead, backups older than a user-configurable retention period (default 90 days, -or never) are deleted once the databases they protect have opened successfully, -covering bridge, upstream pre-migration, and legacy `data.db` backups. Downgrading does not automatically reverse the -database conversion; recovery requires the corresponding backup and the prior -application version. Validation uses synthetic upstream fixtures, not a user's -real profile. +the rebuild. Opening a database never deletes a published backup except an exact +duplicate of a newer backup of the same migration (each failed attempt copies the +unchanged original again); it also removes unfinished `.pending` copies left by a +crash. Upstream `backups/auto/pre-migration--*.db` snapshots are covered +too. A housekeeping error blocks a new upgrade backup, but not opening a database +that is already current. Backup candidates must have the exact database-specific +name format and be regular files, not symbolic links (or hard links on Unix). + +Backups older than a user-configurable retention period (default 90 days, or +never) are deleted by one global policy covering every database in the data +directory: bridge, upstream pre-migration, and legacy `data.db` backups. The +newest backup of each database is always kept, whatever its age or the clock. +Clearing a network's data deletes that network's wallet database backups, because +they copy its wallet and identity history; backups of the shared `det-app.sqlite` +also hold other networks' data and stay under retention. Removing a single wallet +or identity deletes no backups. + +Downgrading does not automatically reverse the database conversion; recovery +requires the corresponding backup and the prior application version. Validation +uses synthetic upstream fixtures, not a user's real profile. ## Validation and limits diff --git a/docs/user-stories.md b/docs/user-stories.md index a1a1679d2..56dc3b9a4 100644 --- a/docs/user-stories.md +++ b/docs/user-stories.md @@ -326,17 +326,6 @@ As a user opening an older wallet installation, I want the app to update its sto - Standalone command-line and MCP use never wait for a window that is not present. Without a password they ask the user to open the desktop app once, then try again. - An operator without the desktop app can supply the wallet password non-interactively (`app_storage_update`; det-cli `--password-stdin` or `--password-file`, never as a command-line argument or environment variable). The update then finishes for every wallet sharing that password, or fails at once without skipping any wallet. -### WAL-034: Control how long upgrade backups are kept [Implemented] -**Persona:** Alex, Priya, Jordan - -As a user whose wallet data was copied before a storage upgrade, I want old upgrade backups to be deleted after a while so that copies of my wallet data do not stay on this device forever. - -- Upgrade backups older than 90 days are deleted automatically by default, covering app, wallet, upstream pre-migration and legacy `data.db` backups. -- Settings → Upgrade Backups lets the user change the period (1 to 3650 days) or keep backups forever. Turning automatic deletion on lets the user choose the period before applying it, and any stricter setting asks for confirmation first, because it deletes backups immediately. -- A saved change applies right away to the shared app backups, the legacy backups, and the current network's wallet backups; other networks' wallet backups follow the next time that network is opened. -- Old backups are deleted only after the app has opened the data they protect, so a backup is never removed while it may still be needed for recovery. -- Removing a wallet or an identity does not delete upgrade backups; retention alone decides when they go. Clearing a network's data still deletes them. - ### WAL-033: Wallets, addresses and identities survive an app upgrade [Gap] **Persona:** Alex, Priya, Jordan @@ -348,6 +337,18 @@ As a user updating from any earlier version I still have installed, I want every - Starting the updated app a second time changes nothing further, and a user whose data was already current keeps an untouched database rather than a needlessly rewritten one. - Currently a gap: each migration mechanism is unit-tested on synthetic data, but no automated check upgrades a data directory a *released* build actually wrote and confirms the result. The design for closing it is `docs/ai-design/2026-09-10-migration-matrix/design.md`; related implemented behaviour is covered by WAL-032, IDN-016 and IDN-020. +### WAL-034: Control how long upgrade backups are kept [Implemented] +**Persona:** Alex, Priya, Jordan + +As a user whose wallet data was copied before a storage upgrade, I want old upgrade backups to be deleted after a while so that copies of my wallet data do not stay on this device forever. + +- Upgrade backups older than 90 days are deleted automatically by default, covering app, wallet, upstream pre-migration and legacy `data.db` backups. +- Settings → Upgrade Backups lets the user change the period (1 to 3650 days) or keep backups forever. Turning automatic deletion on lets the user choose the period before applying it, and any stricter setting asks for confirmation first, because it deletes backups immediately. A period outside the range is shown with an error and never saved. +- The setting is one policy for every backup in the data directory, whichever network is open; a saved change applies to every network's backups as soon as the wallet data has opened. +- The newest backup of each database is always kept, whatever its age or the system clock, so the last recovery copy is never deleted automatically. +- With "keep forever", no published backup is deleted automatically. Opening the data removes only unfinished copies left by a crash and exact duplicates of a newer backup. +- Removing a wallet or an identity does not delete upgrade backups. Clearing a network's data deletes that network's wallet backups; backups of the shared app data stay under the retention setting. + --- ## Send and Receive (SND) diff --git a/src/app.rs b/src/app.rs index 74cffc133..7b233af5c 100644 --- a/src/app.rs +++ b/src/app.rs @@ -1455,7 +1455,8 @@ impl AppState { pub(crate) fn boot_inputs() -> Result<(PathBuf, Arc), Box> { let data_dir = crate::boot::prepare_environment()?; - let db_file_path = data_file_path(&data_dir, "data.db")?; + let db_file_path = + data_file_path(&data_dir, crate::database::legacy_backups::LEGACY_DATABASE)?; let db = if db_file_path.exists() { Arc::new(Database::open_legacy_read_only(&db_file_path)?) } else { diff --git a/src/app_dir.rs b/src/app_dir.rs index 414ef778d..c91cc6ce0 100644 --- a/src/app_dir.rs +++ b/src/app_dir.rs @@ -152,10 +152,13 @@ pub fn copy_env_file_if_not_exists() { ensure_env_file(&app_data_dir); } +/// File name of the environment config in the data directory. +pub const ENV_FILE: &str = ".env"; + /// If .env file does not exist in the given data directory, /// copy the bundled `.env.example` file there. pub fn ensure_env_file(data_dir: &Path) { - let env_file_in_app_dir = data_dir.join(".env"); + let env_file_in_app_dir = data_dir.join(ENV_FILE); BundledResource::DotEnvExample .write_to_file(&env_file_in_app_dir, false) .expect("Failed to write bundled .env.example file"); diff --git a/src/backend_task/error.rs b/src/backend_task/error.rs index 8d238cc2c..87a83413f 100644 --- a/src/backend_task/error.rs +++ b/src/backend_task/error.rs @@ -4,6 +4,7 @@ //! `Debug` → variant name + fields (logged and shown in collapsible details). use crate::model::fee_estimation::format_credits_as_dash; +use crate::wallet_backend::platform_compatibility::StorageFailure; use dash_sdk::Error as SdkError; use dash_sdk::dapi_client::DapiClientError; use dash_sdk::dapi_client::transport::TransportError; @@ -108,6 +109,9 @@ impl std::error::Error for BackendTaskJoinError {} /// Dash Core RPC error code: wallet file not specified (multi-wallet node). const RPC_WALLET_NOT_SPECIFIED: i32 = -19; +/// Shown whenever another window or session holds the wallet database. +pub(crate) const WALLET_DATA_IN_USE: &str = "Your wallet data is open in another Dash Evo Tool window or command-line session. Close it and try again."; + /// App-level error envelope for backend tasks. #[derive(Debug, Error)] pub enum TaskError { @@ -436,9 +440,7 @@ pub enum TaskError { }, /// Another process currently owns the wallet database write lock. - #[error( - "Your wallet data is open in another Dash Evo Tool window or command-line session. Close it and try again." - )] + #[error("{}", WALLET_DATA_IN_USE)] WalletStorageInUse { #[source] source: platform_wallet_storage::WalletStorageError, @@ -1188,7 +1190,7 @@ pub enum TaskError { /// The upgrade-backup retention setting could not be read, so no backup was /// deleted. #[error( - "Could not read your backup retention setting, so no old backups were deleted. Restart the application to try again." + "Your backup retention setting could not be read, so no old upgrade backups were deleted. Open Settings, choose a backup retention setting and save it." )] BackupRetentionRead { #[source] @@ -3067,50 +3069,17 @@ impl TaskError { fn wallet_storage_error_is_in_use( source: &platform_wallet_storage::WalletStorageError, ) -> bool { - Self::wallet_storage_sqlite_cause(source, |code| { - matches!( - code, - rusqlite::ErrorCode::DatabaseBusy | rusqlite::ErrorCode::DatabaseLocked - ) - }) + StorageFailure::in_chain(source) == Some(StorageFailure::InUse) } /// A migration that ran out of disk space, hit an OS I/O failure or ran /// out of memory is recoverable by freeing the resource and retrying, so - /// it must not get the terminal incompatible-data guidance. Access - /// failures (`PermissionDenied`, `ReadOnly`) are deliberately excluded: - /// retrying never fixes them. + /// it must not get the terminal incompatible-data guidance. fn wallet_storage_error_is_resource_exhausted( source: &platform_wallet_storage::WalletStorageError, ) -> bool { - Self::wallet_storage_sqlite_cause(source, |code| { - matches!( - code, - rusqlite::ErrorCode::DiskFull - | rusqlite::ErrorCode::SystemIoFailure - | rusqlite::ErrorCode::OutOfMemory - ) - }) - } - - /// Whether any `rusqlite::Error` in the typed source chain carries a - /// SQLite code accepted by `is_match`. - fn wallet_storage_sqlite_cause( - source: &platform_wallet_storage::WalletStorageError, - is_match: impl Fn(rusqlite::ErrorCode) -> bool, - ) -> bool { - let mut cause: Option<&(dyn std::error::Error + 'static)> = Some(source); - while let Some(error) = cause { - if error - .downcast_ref::() - .and_then(rusqlite::Error::sqlite_error_code) - .is_some_and(&is_match) - { - return true; - } - cause = error.source(); - } - false + StorageFailure::in_chain(source) + .is_some_and(|failure| failure.is_retryable() && failure != StorageFailure::InUse) } /// Returns `true` when this is a [`Self::SecretStore`] open failure caused diff --git a/src/backend_task/migration/finish_unwire.rs b/src/backend_task/migration/finish_unwire.rs index 0fab71dfd..31612efef 100644 --- a/src/backend_task/migration/finish_unwire.rs +++ b/src/backend_task/migration/finish_unwire.rs @@ -3125,6 +3125,10 @@ mod tests { } #[tokio::test] + #[expect( + clippy::disallowed_methods, + reason = "test fixture setup/teardown outside any production deletion path" + )] async fn dapi_config_persistence_reports_save_failure() { let _env_guard = CONFIG_ENV_LOCK.lock().await; let tmp = tempfile::tempdir().expect("tempdir"); diff --git a/src/backend_task/mod.rs b/src/backend_task/mod.rs index e90b7d502..2a536f277 100644 --- a/src/backend_task/mod.rs +++ b/src/backend_task/mod.rs @@ -686,8 +686,8 @@ pub enum BackendTaskSuccessResult { retention: crate::model::backup_retention::BackupRetention, /// Expired backups deleted under the new policy. deleted: usize, - /// Whether some expired backups could not be deleted (retried at next start). - cleanup_incomplete: bool, + /// Why some expired backups could not be deleted; they are retried at the next start. + cleanup_failure: Option>, }, PlatformInfo(PlatformInfoTaskResult), diff --git a/src/backend_task/system_task/mod.rs b/src/backend_task/system_task/mod.rs index 8b8812799..e6b754e5f 100644 --- a/src/backend_task/system_task/mod.rs +++ b/src/backend_task/system_task/mod.rs @@ -85,7 +85,7 @@ impl AppContext { Ok(BackendTaskSuccessResult::UpdatedBackupRetention { retention, deleted: report.deleted, - cleanup_incomplete: report.failure.is_some(), + cleanup_failure: report.failure.map(Arc::new), }) } } diff --git a/src/config.rs b/src/config.rs index 499bb4785..9e4c01221 100644 --- a/src/config.rs +++ b/src/config.rs @@ -142,8 +142,8 @@ impl Config { /// Preserves lines not modeled by `Config` and replaces the modeled fields. /// Uses a temporary file and atomic rename to prevent partial writes. pub fn save(&self, data_dir: &Path) -> Result<(), ConfigError> { - let env_file_path = - data_file_path(data_dir, ".env").map_err(|e| ConfigError::SaveError { source: e })?; + let env_file_path = data_file_path(data_dir, crate::app_dir::ENV_FILE) + .map_err(|e| ConfigError::SaveError { source: e })?; let existing_contents = read_env_contents(&env_file_path)?; validate_env_contents(&existing_contents)?; let modeled_entries = self.modeled_entries(); @@ -185,8 +185,8 @@ impl Config { network: Network, addresses: &str, ) -> Result<(), ConfigError> { - let env_file_path = - data_file_path(data_dir, ".env").map_err(|source| ConfigError::SaveError { source })?; + let env_file_path = data_file_path(data_dir, crate::app_dir::ENV_FILE) + .map_err(|source| ConfigError::SaveError { source })?; let existing_contents = read_env_contents(&env_file_path)?; let key = format!("{}dapi_addresses", dotenv_network_prefix(network)); atomic_replace_env_file(&env_file_path, |env_file| { @@ -203,15 +203,16 @@ impl Config { /// Loads the configuration for all networks from environment variables and `.env` file /// located in the default app data directory. pub fn load() -> Result { - let env_file_path = app_user_data_file_path(".env").expect("should create .env file path"); + let env_file_path = app_user_data_file_path(crate::app_dir::ENV_FILE) + .expect("should create .env file path"); Self::load_from_env_path(env_file_path) } /// Loads the configuration for all networks from environment variables and `.env` file /// located in the given data directory. pub fn load_from(data_dir: &Path) -> Result { - let env_file_path = - data_file_path(data_dir, ".env").map_err(|source| ConfigError::LoadError { source })?; + let env_file_path = data_file_path(data_dir, crate::app_dir::ENV_FILE) + .map_err(|source| ConfigError::LoadError { source })?; Self::load_from_env_path(env_file_path) } diff --git a/src/context/mod.rs b/src/context/mod.rs index 77a4e571d..5cf75bec5 100644 --- a/src/context/mod.rs +++ b/src/context/mod.rs @@ -618,7 +618,7 @@ impl AppContext { use platform_wallet_storage::SqlitePersisterConfig; crate::app_dir::ensure_data_dir_exists(data_dir) .map_err(|source| TaskError::FileSystem { source })?; - let path = data_dir.join("det-app.sqlite"); + let path = crate::wallet_backend::app_database_path(data_dir); let config = SqlitePersisterConfig::new(path); let persister = Arc::new(crate::wallet_backend::platform_compatibility::open(config)?); Ok(Arc::new(DetKv::new(persister))) diff --git a/src/context/wallet_lifecycle/removal.rs b/src/context/wallet_lifecycle/removal.rs index c1c42ff64..312fd9536 100644 --- a/src/context/wallet_lifecycle/removal.rs +++ b/src/context/wallet_lifecycle/removal.rs @@ -78,31 +78,18 @@ impl AppContext { Ok(()) } - /// Delete the retained compatibility-upgrade backups of the app and this network's - /// wallet databases, which copy the wallet and identity history a network clear wipes. + /// Delete the retained compatibility-upgrade backups of this network's wallet + /// database, which copy the wallet and identity history a network clear wipes. /// - /// Filesystem cleanup is available even when the wallet backend could not be opened. - /// Removing a single wallet or identity never calls this: those backups also hold - /// other wallets' data and expire through time-based retention instead. + /// The shared app database's backups also hold other networks' data, so they are + /// left to time-based retention. Filesystem cleanup is available even when the + /// wallet backend could not be opened. Removing a single wallet or identity never + /// calls this: those backups also hold other wallets' data. pub(crate) fn remove_upgrade_backups(&self) -> Result<(), TaskError> { - let mut first_error = None; - for database in self.upgrade_backup_databases() { - if let Err(source) = - crate::wallet_backend::platform_compatibility::remove_backups(&database) - { - first_error.get_or_insert(TaskError::FileSystem { source }); - } - } - first_error.map_or(Ok(()), Err) - } - - /// The app database and this network's wallet database, whose upgrade backups - /// copy wallet and identity history. - pub(super) fn upgrade_backup_databases(&self) -> [std::path::PathBuf; 2] { - [ - self.data_dir().join("det-app.sqlite"), - crate::wallet_backend::wallet_database_path(self.data_dir(), self.network), - ] + crate::wallet_backend::platform_compatibility::remove_backups( + &crate::wallet_backend::wallet_database_path(self.data_dir(), self.network), + ) + .map_err(|source| TaskError::FileSystem { source }) } } diff --git a/src/context/wallet_lifecycle/retention.rs b/src/context/wallet_lifecycle/retention.rs index cc1ff0418..21bd0f51a 100644 --- a/src/context/wallet_lifecycle/retention.rs +++ b/src/context/wallet_lifecycle/retention.rs @@ -12,17 +12,36 @@ pub struct BackupPruneReport { pub failure: Option, } +/// Every database in `data_dir` whose upgrade backups the global retention covers: +/// the shared app database and each network's wallet database. +pub(crate) fn retained_backup_databases(data_dir: &std::path::Path) -> Vec { + std::iter::once(crate::wallet_backend::app_database_path(data_dir)) + .chain( + crate::wallet_backend::all_networks() + .into_iter() + .map(|network| crate::wallet_backend::wallet_database_path(data_dir, network)), + ) + .collect() +} + impl AppContext { /// Delete upgrade and legacy migration backups older than the configured retention period. /// - /// Covers the app database and this network's wallet database (bridge and upstream - /// `pre-migration-*` snapshots), plus legacy `data.db` backups. Call only after those - /// databases opened successfully: an expired snapshot may be the last one, which is - /// expendable only once the database it protects is known to work. + /// The policy is global: it covers every database in the data directory (the app + /// database and every network's wallet database, bridge and upstream + /// `pre-migration-*` snapshots), plus legacy `data.db` backups, whichever network + /// this context serves. The newest backup of each database is always kept, so the + /// pass is safe for databases this process has not opened. A database another + /// context is opening or upgrading is skipped until the next pass. /// /// Attempts every location and reports the first failure. Nothing is deleted when the /// policy cannot be read. pub(crate) fn prune_expired_upgrade_backups(&self) -> BackupPruneReport { + self.prune_expired_upgrade_backups_at(SystemTime::now()) + } + + /// [`Self::prune_expired_upgrade_backups`] as if the clock read `now`. + pub(crate) fn prune_expired_upgrade_backups_at(&self, now: SystemTime) -> BackupPruneReport { let mut report = BackupPruneReport::default(); let retention = match self.backup_retention() { Ok(retention) => retention, @@ -34,9 +53,7 @@ impl AppContext { let Some(max_age) = retention.max_age() else { return report; }; - let now = SystemTime::now(); - let outcomes = self - .upgrade_backup_databases() + let outcomes = retained_backup_databases(self.data_dir()) .into_iter() .map(|database| { crate::wallet_backend::platform_compatibility::prune_expired_backups( @@ -66,15 +83,20 @@ impl AppContext { report } - /// [`Self::prune_expired_upgrade_backups`] with any failure logged; it is retried on - /// the next start and never fails the surrounding operation. + /// [`Self::prune_expired_upgrade_backups`] with any failure logged; it never fails + /// the surrounding operation. pub(crate) fn prune_expired_upgrade_backups_best_effort(&self) -> BackupPruneReport { let report = self.prune_expired_upgrade_backups(); - if let Some(error) = &report.failure { - tracing::warn!( + match &report.failure { + Some(error @ TaskError::BackupRetentionRead { .. }) => tracing::warn!( ?error, - "Expired upgrade backups remain; retrying on next start" - ); + "Backup retention setting unreadable; no upgrade backups are deleted until the setting is saved again" + ), + Some(error) => tracing::warn!( + ?error, + "Expired upgrade backups remain; retrying when wallet data next opens" + ), + None => {} } report } @@ -113,7 +135,10 @@ mod tests { fn backups(ctx: &crate::context::AppContext) -> Fixture { let dir = ctx.data_dir(); - let [app, wallet] = ctx.upgrade_backup_databases(); + let [app, wallet] = [ + crate::wallet_backend::app_database_path(dir), + crate::wallet_backend::wallet_database_path(dir, ctx.network), + ]; let bridge = |database: &Path, suffix: &str| { database.with_file_name(format!( "{}.platform-67d4ef3-backup-{suffix}.sqlite", @@ -125,7 +150,14 @@ mod tests { bridge(&wallet, "old"), dir.join("backups/data_backup_20000101_000000.db"), ]; - let fresh = vec![bridge(&app, "new"), bridge(&wallet, "new")]; + let fresh = vec![ + bridge(&app, "new"), + bridge(&wallet, "new"), + dir.join("backups") + .join(crate::database::legacy_backups::backup_file_name( + chrono::Utc::now(), + )), + ]; for path in &expired { write_aged(path, DAY * 120); } @@ -174,4 +206,76 @@ mod tests { assert_eq!(pruned(&ctx), 3); assert!(fixture.fresh.iter().all(|path| path.exists())); } + + #[test] + fn retention_is_global_across_networks() { + let tmp = tempfile::tempdir().unwrap(); + let ctx = test_app_context(tmp.path()); + let other = dash_sdk::dpp::dashcore::Network::Mainnet; + assert_ne!(ctx.network, other); + let database = crate::wallet_backend::wallet_database_path(ctx.data_dir(), other); + let auto = platform_wallet_storage::default_auto_backup_dir(&database); + let expired = auto.join("pre-migration-det-mainnet-1-to-2-20000101T000000Z.db"); + let newest = auto.join("pre-migration-det-mainnet-2-to-3-20000102T000000Z.db"); + write_aged(&expired, DAY * 400); + write_aged(&newest, DAY * 300); + + assert_eq!(pruned(&ctx), 1); + assert!( + !expired.exists(), + "another network's expired backup is pruned" + ); + assert!( + newest.exists(), + "the newest backup of each database is kept" + ); + assert!( + !crate::wallet_backend::wallet_database_path( + ctx.data_dir(), + dash_sdk::dpp::dashcore::Network::Devnet + ) + .with_extension("sqlite.platform-upgrade.lock") + .exists(), + "a database without backups is not locked or touched" + ); + } + + #[test] + fn newest_backup_of_each_database_survives_a_far_future_clock() { + let tmp = tempfile::tempdir().unwrap(); + let ctx = test_app_context(tmp.path()); + let fixture = backups(&ctx); + let far_future = SystemTime::now() + DAY * 365 * 100; + + let report = ctx.prune_expired_upgrade_backups_at(far_future); + assert!(report.failure.is_none(), "{:?}", report.failure); + assert_eq!(report.deleted, 3); + assert!(fixture.fresh.iter().all(|path| path.exists())); + assert!(fixture.expired.iter().all(|path| !path.exists())); + let again = ctx.prune_expired_upgrade_backups_at(far_future); + assert_eq!(again.deleted, 0, "the last backups are never deleted"); + assert!(fixture.fresh.iter().all(|path| path.exists())); + } + + #[test] + fn database_in_use_is_skipped_not_failed() { + let tmp = tempfile::tempdir().unwrap(); + let ctx = test_app_context(tmp.path()); + let fixture = backups(&ctx); + let wallet = crate::wallet_backend::wallet_database_path(ctx.data_dir(), ctx.network); + let lock = wallet.with_file_name(format!( + "{}.platform-upgrade.lock", + wallet.file_name().unwrap().to_string_lossy() + )); + let held = std::fs::File::create(&lock).unwrap(); + held.try_lock().unwrap(); + + let report = ctx.prune_expired_upgrade_backups(); + assert!(report.failure.is_none(), "{:?}", report.failure); + assert_eq!( + report.deleted, 2, + "app and legacy backups; the locked wallet is skipped" + ); + assert!(fixture.expired[1].exists()); + } } diff --git a/src/context/wallet_lifecycle/tests.rs b/src/context/wallet_lifecycle/tests.rs index 0eea579a2..edcbbbc12 100644 --- a/src/context/wallet_lifecycle/tests.rs +++ b/src/context/wallet_lifecycle/tests.rs @@ -2562,13 +2562,21 @@ async fn clear_network_database_removes_backups_when_legacy_shielded_cleanup_fai std::fs::create_dir_all(blocker.join("occupied")).unwrap(); let backup = ctx .data_dir() - .join("det-app.sqlite.platform-67d4ef3-backup-fixture.sqlite"); + .join("det-testnet.sqlite.platform-67d4ef3-backup-fixture.sqlite"); std::fs::write(&backup, b"old wallet history").unwrap(); + let shared = ctx + .data_dir() + .join("det-app.sqlite.platform-67d4ef3-backup-fixture.sqlite"); + std::fs::write(&shared, b"all networks' app data").unwrap(); let result = ctx.clear_network_database().await; backend.shutdown().await; assert!(!backup.exists(), "upgrade backups must still be removed"); + assert!( + shared.exists(), + "the shared app database's backups are left to retention" + ); match result { Err(TaskError::WalletDataClearIncomplete { failed, @@ -2677,6 +2685,10 @@ async fn lock_wipes_session_cached_seed() { /// with a directory: the store's atomic `persist` rename onto a directory /// path fails deterministically (root cannot bypass this). #[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[expect( + clippy::disallowed_methods, + reason = "test fixture setup/teardown outside any production deletion path" +)] async fn register_wallet_fails_closed_when_seed_envelope_write_fails() { let temp_dir = tempfile::tempdir().expect("tempdir"); let (ctx, _sender) = offline_testnet_context_at(temp_dir.path()); @@ -3481,6 +3493,32 @@ fn clear_spv_chain_storage_is_ok_when_directory_absent() { .expect("clearing an absent cache must succeed"); } +/// Chain storage moved to another disk and symlinked back is cleared by +/// unlinking the links; the data they point at is never followed or touched. +#[cfg(unix)] +#[test] +fn clear_spv_chain_storage_unlinks_symlinked_entries() { + let tmp = tempfile::tempdir().expect("tempdir"); + let other_disk = tempfile::tempdir().expect("tempdir"); + std::fs::write(other_disk.path().join("segment.dat"), b"elsewhere").unwrap(); + let spv_dir = spv_storage_dir(tmp.path(), Network::Testnet); + std::fs::create_dir_all(&spv_dir).unwrap(); + std::os::unix::fs::symlink(other_disk.path(), spv_dir.join("block_headers")).unwrap(); + std::os::unix::fs::symlink( + other_disk.path().join("segment.dat"), + spv_dir.join("peers.dat"), + ) + .unwrap(); + + clear_spv_chain_storage(tmp.path(), Network::Testnet) + .expect("symlinked chain storage must not block the clear"); + + for entry in ["block_headers", "peers.dat"] { + assert!(spv_dir.join(entry).symlink_metadata().is_err(), "{entry}"); + } + assert!(other_disk.path().join("segment.dat").exists()); +} + /// Seed a legacy password-protected `single_key_wallet` row into the /// context's `data.db`, encrypted under `password`. Returns the /// derived address. The default test DB created `single_key_wallet` @@ -7306,8 +7344,22 @@ async fn remove_wallet_keeps_upgrade_backups_without_backend() { /// An upgrade-backup fixture next to `database`, last modified `days` ago. fn aged_upgrade_backup(database: &std::path::Path, days: u64) -> std::path::PathBuf { + named_aged_upgrade_backup(database, "aged", days) +} + +/// A newer backup of `database`: retention always keeps the newest one, so it lets +/// the aged backup expire. +fn recent_upgrade_backup(database: &std::path::Path) -> std::path::PathBuf { + named_aged_upgrade_backup(database, "recent", 1) +} + +fn named_aged_upgrade_backup( + database: &std::path::Path, + suffix: &str, + days: u64, +) -> std::path::PathBuf { let backup = database.with_file_name(format!( - "{}.platform-67d4ef3-backup-aged.sqlite", + "{}.platform-67d4ef3-backup-{suffix}.sqlite", database.file_name().unwrap().to_string_lossy() )); std::fs::write(&backup, b"old wallet history").unwrap(); @@ -7326,13 +7378,16 @@ fn aged_upgrade_backup(database: &std::path::Path, days: u64) -> std::path::Path #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn ensure_wallet_backend_prunes_expired_upgrade_backups() { let (ctx, sender, _tmp) = offline_testnet_context(); - let expired = [ - aged_upgrade_backup(&ctx.data_dir().join("det-app.sqlite"), 120), - aged_upgrade_backup( - &crate::wallet_backend::wallet_database_path(ctx.data_dir(), Network::Testnet), - 120, - ), + let databases = [ + ctx.data_dir().join("det-app.sqlite"), + crate::wallet_backend::wallet_database_path(ctx.data_dir(), Network::Testnet), ]; + let expired = databases + .each_ref() + .map(|database| aged_upgrade_backup(database, 120)); + let newest = databases + .each_ref() + .map(|database| recent_upgrade_backup(database)); ctx.ensure_wallet_backend(sender) .await @@ -7341,6 +7396,13 @@ async fn ensure_wallet_backend_prunes_expired_upgrade_backups() { for backup in &expired { assert!(!backup.exists(), "{} must expire", backup.display()); } + for backup in &newest { + assert!( + backup.exists(), + "{} is the newest and stays", + backup.display() + ); + } ctx.wallet_backend().unwrap().shutdown().await; } @@ -7374,12 +7436,13 @@ async fn update_backup_retention_prunes_only_after_wallet_backend_opens() { let (ctx, sender, _tmp) = offline_testnet_context(); let backup = aged_upgrade_backup(&ctx.data_dir().join("det-app.sqlite"), 60); + recent_upgrade_backup(&ctx.data_dir().join("det-app.sqlite")); let saved = |result| match result { Ok(BackendTaskSuccessResult::UpdatedBackupRetention { retention, deleted, - cleanup_incomplete, - }) => (retention, deleted, cleanup_incomplete), + cleanup_failure, + }) => (retention, deleted, cleanup_failure.is_some()), other => panic!("unexpected result: {other:?}"), }; diff --git a/src/context/wallet_lifecycle/tests/deletion_sentinel.rs b/src/context/wallet_lifecycle/tests/deletion_sentinel.rs index 22cf70500..228781bfa 100644 --- a/src/context/wallet_lifecycle/tests/deletion_sentinel.rs +++ b/src/context/wallet_lifecycle/tests/deletion_sentinel.rs @@ -88,16 +88,14 @@ async fn deletion_paths_leave_every_untargeted_file_intact() { } } } - // Other networks' SPV data and backups, config, a pre-migration copy, - // current logs and a foreign-stem log. + // Other networks' SPV data, config, a pre-migration copy, current logs and + // a foreign-stem log. sentinels.extend( [ "spv/mainnet/blocks/segment.dat", "spv/mainnet/peers.dat", "spv/mainnet/det-shielded.sqlite", "spv/mainnet.lock", - "det-mainnet.sqlite.platform-67d4ef3-backup-old.sqlite", - "backups/auto/pre-migration-det-mainnet-1-to-2-20000101T000000Z.db", "data.db.premigration", "det.log", "det-stderr.0000000001.log", @@ -108,18 +106,31 @@ async fn deletion_paths_leave_every_untargeted_file_intact() { for path in &sentinels { plant(path, old); } + // The newest backup of each database is kept by retention whatever its age, and + // the network clear keeps the shared app database's backups for retention too. + let newest: Vec = [ + "backups/auto/pre-migration-det-mainnet-1-to-2-20000101T000000Z.db", + "det-app.sqlite.platform-67d4ef3-backup-new.sqlite", + "backups/data_backup_20000102_000000.db", + ] + .map(|relative| dir.join(relative)) + .into(); + for path in &newest { + plant(path, fresh); + } + sentinels.extend(newest); // The current log is rotated to a name from its mtime; keep it recent so the // rotated copy is retained rather than expiring in the same pass. plant(&dir.join("det.log"), fresh); let targeted: BTreeSet = [ - // Retention prune: expired backups of the app and testnet databases. + // Retention prune (global): expired backups of every network's database. "det-app.sqlite.platform-67d4ef3-backup-old.sqlite", + "det-mainnet.sqlite.platform-67d4ef3-backup-old.sqlite", "det-testnet.sqlite.platform-67d4ef3-backup-old.sqlite", "backups/auto/pre-migration-det-testnet-1-to-2-20000101T000000Z.db", "backups/data_backup_20000101_000000.db", - // Network clear: every remaining app/testnet upgrade backup. - "det-app.sqlite.platform-67d4ef3-backup-new.sqlite", + // Network clear: every remaining testnet upgrade backup. "det-testnet.sqlite.platform-67d4ef3-backup-new.sqlite", // Network clear (SPV cache + retired shielded files): testnet only. "spv/testnet/blocks/segment.dat", @@ -158,7 +169,7 @@ async fn deletion_paths_leave_every_untargeted_file_intact() { let report = ctx.prune_expired_upgrade_backups(); assert!(report.failure.is_none(), "{:?}", report.failure); - assert_eq!(report.deleted, 4, "retention prune"); + assert_eq!(report.deleted, 5, "retention prune"); ctx.remove_wallet(&seed_hash).expect("remove wallet"); ctx.delete_local_qualified_identity(&identity.identity.id()) .expect("remove identity"); @@ -198,3 +209,46 @@ async fn deletion_paths_leave_every_untargeted_file_intact() { } } } + +/// "Keep forever" keeps every published snapshot through the databases' opens and +/// repeated retention passes; only crash-left partials go. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn keep_forever_keeps_every_published_snapshot() { + let (ctx, sender, tmp) = offline_testnet_context(); + let dir = tmp.path(); + ctx.set_backup_retention(crate::model::backup_retention::BackupRetention::KeepForever) + .unwrap(); + let published: Vec = [ + "det-app.sqlite.platform-67d4ef3-backup-a1.sqlite", + "det-testnet.sqlite.platform-67d4ef3-backup-a1.sqlite", + "backups/auto/pre-migration-det-testnet-1-to-2-20000101T000000Z.db", + "backups/auto/pre-migration-det-testnet-2-to-3-20010101T000000Z.db", + "backups/auto/pre-migration-det-mainnet-1-to-2-20000101T000000Z.db", + "backups/data_backup_20000101_000000.db", + ] + .map(|relative| dir.join(relative)) + .into(); + for path in &published { + plant(path, DAY * 4000); + } + let partial = dir.join("det-testnet.sqlite.platform-67d4ef3-backup-crash.pending"); + plant(&partial, DAY * 4000); + + ctx.ensure_wallet_backend(sender) + .await + .expect("ensure_wallet_backend should succeed offline"); + for _ in 0..3 { + let report = ctx.prune_expired_upgrade_backups(); + assert!(report.failure.is_none(), "{:?}", report.failure); + assert_eq!(report.deleted, 0); + } + ctx.wallet_backend() + .expect("backend wired") + .shutdown() + .await; + + for path in &published { + assert!(path.exists(), "{path:?} must be kept forever"); + } + assert!(!partial.exists(), "a crash-left partial is cleaned up"); +} diff --git a/src/database/legacy_backups.rs b/src/database/legacy_backups.rs index 0d76536a6..c11a3d291 100644 --- a/src/database/legacy_backups.rs +++ b/src/database/legacy_backups.rs @@ -9,7 +9,7 @@ use std::time::{Duration, SystemTime}; /// Directory, next to the database, that holds the legacy backups. pub(crate) const BACKUP_DIR: &str = "backups"; /// The legacy database these backups copy. -const LEGACY_DATABASE: &str = "data.db"; +pub(crate) const LEGACY_DATABASE: &str = "data.db"; const PREFIX: &str = "data_backup_"; const SUFFIX: &str = ".db"; /// UTC timestamp format embedded in the backup name. @@ -20,14 +20,16 @@ pub(crate) fn backup_file_name(timestamp: chrono::DateTime) -> Stri format!("{PREFIX}{}{SUFFIX}", timestamp.format(TIMESTAMP_FORMAT)) } -/// The UTC creation time embedded in a legacy backup name, or `None` for any other file. -fn named_timestamp(name: &str) -> Option { +/// The UTC creation time embedded in a legacy backup name, or `None` for any other +/// file. Also the deletion chokepoint's definition of a legacy backup name. +pub(crate) fn backup_timestamp(name: &str) -> Option { let timestamp = name.strip_prefix(PREFIX)?.strip_suffix(SUFFIX)?; let parsed = chrono::NaiveDateTime::parse_from_str(timestamp, TIMESTAMP_FORMAT).ok()?; Some(parsed.and_utc().into()) } -/// Delete legacy backups in `/backups` older than `max_age`. +/// Delete legacy backups in `/backups` older than `max_age`, always +/// keeping the newest one. /// /// Deletion goes through the [`delete_file`](crate::utils::file_deletion::delete_file) /// chokepoint, which refuses live databases, aliases and hard links. @@ -35,10 +37,8 @@ fn named_timestamp(name: &str) -> Option { /// including symlinks and subdirectories, is left alone. Age is measured from the /// later of the name timestamp and the modification time, so a reset or skewed signal /// only delays deletion. A `backups` path that is a symlink or not a directory is -/// refused. A candidate that vanishes mid-pass (e.g. a concurrent prune from another -/// network's context) counts as already handled, not as a failure. Attempts every -/// candidate and returns the first failure; `Ok` carries the number of backups this -/// call deleted. +/// refused. Expiry itself is [`prune_expired`](crate::utils::backup_prune::prune_expired); +/// `Ok` carries the number of backups this call deleted. pub(crate) fn prune_expired( data_dir: &Path, max_age: Duration, @@ -55,41 +55,27 @@ pub(crate) fn prune_expired( Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(0), Err(error) => return Err(error), } - let entries = std::fs::read_dir(&directory)?; - let legacy_database = data_dir.join(LEGACY_DATABASE); - let mut removed = 0; - let mut first_error = None; - for entry in entries { - let outcome = entry.and_then(|entry| { - let Some(named) = entry.file_name().to_str().and_then(named_timestamp) else { - return Ok(false); - }; - let metadata = std::fs::symlink_metadata(entry.path())?; - if !metadata.is_file() { - return Ok(false); - } - let created = named.max(metadata.modified()?); - if !crate::model::backup_retention::backup_expired(created, now, max_age) { - return Ok(false); - } - crate::utils::file_deletion::delete_file( - &entry.path(), - crate::utils::file_deletion::DeletionIntent::Backup { - database: &legacy_database, - }, - ) - .map(|()| true) - }); - match outcome { - Ok(true) => removed += 1, - Ok(false) => {} - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(error) => { - first_error.get_or_insert(error); - } - } + let mut candidates = Vec::new(); + for entry in std::fs::read_dir(&directory)? { + let entry = entry?; + let Some(named) = entry.file_name().to_str().and_then(backup_timestamp) else { + continue; + }; + let metadata = match std::fs::symlink_metadata(entry.path()) { + Ok(metadata) if !metadata.is_file() => continue, + other => other, + }; + let created = metadata.and_then(|metadata| Ok(named.max(metadata.modified()?))); + candidates.push((entry.path(), created)); } - first_error.map_or(Ok(removed), Err) + crate::utils::backup_prune::prune_expired( + candidates, + now, + max_age, + crate::utils::file_deletion::DeletionIntent::Backup { + database: &data_dir.join(LEGACY_DATABASE), + }, + ) } #[cfg(test)] @@ -114,13 +100,13 @@ mod tests { .to_utc(); let name = backup_file_name(taken); assert_eq!(name, "data_backup_20240305_060708.db"); - assert_eq!(named_timestamp(&name), Some(taken.into())); + assert_eq!(backup_timestamp(&name), Some(taken.into())); for other in [ "data_backup_20240305_060708.db-journal", "data_backup_latest.db", "data.db", ] { - assert_eq!(named_timestamp(other), None, "{other}"); + assert_eq!(backup_timestamp(other), None, "{other}"); } } @@ -220,8 +206,32 @@ mod tests { .into_iter() .map(|result| result.expect("a concurrently deleted backup is not a failure")) .sum(); - assert_eq!(deleted, 200, "each backup is counted once"); - assert_eq!(std::fs::read_dir(&backups).unwrap().count(), 0); + assert_eq!( + deleted, 199, + "each backup is counted once; the newest stays" + ); + assert_eq!(std::fs::read_dir(&backups).unwrap().count(), 1); + } + + /// A clock far in the future must never delete the last legacy backup. + #[test] + fn newest_legacy_backup_survives_a_far_future_clock() { + let dir = tempfile::tempdir().unwrap(); + let backups = dir.path().join(BACKUP_DIR); + std::fs::create_dir_all(&backups).unwrap(); + let older = backups.join("data_backup_20200101_000000.db"); + let newest = backups.join("data_backup_20210101_000000.db"); + for (path, age) in [(&older, DAY * 30), (&newest, DAY * 2)] { + std::fs::write(path, b"backup").unwrap(); + set_mtime(path, SystemTime::now() - age); + } + let far_future = SystemTime::now() + DAY * 365 * 100; + + assert_eq!(prune_expired(dir.path(), DAY, far_future).unwrap(), 1); + assert!(!older.exists()); + assert!(newest.exists(), "the newest backup is the floor"); + assert_eq!(prune_expired(dir.path(), DAY, far_future).unwrap(), 0); + assert!(newest.exists()); } #[cfg(unix)] diff --git a/src/logging.rs b/src/logging.rs index 422d31c4b..83ed14365 100644 --- a/src/logging.rs +++ b/src/logging.rs @@ -548,10 +548,18 @@ fn rotated_name(stem: &str, ts: i64) -> String { /// Parses the timestamp out of a rotated log file name produced by /// [`rotated_name`], returning `None` for names that don't match the stem. -fn parse_rotated_ts(name: &str, stem: &str) -> Option { - name.strip_prefix(&format!("{stem}.")) - .and_then(|s| s.strip_suffix(".log")) - .and_then(|s| s.parse::().ok()) +/// The timestamp in a rotated log name `..log`, or `None` for any +/// other name. Also the deletion chokepoint's definition of a rotated log. +pub(crate) fn parse_rotated_ts(name: &str, stem: &str) -> Option { + if stem.is_empty() { + return None; + } + name.strip_prefix(stem)? + .strip_prefix('.')? + .strip_suffix(".log") + .filter(|digits| !digits.is_empty() && digits.bytes().all(|b| b.is_ascii_digit()))? + .parse() + .ok() } #[cfg(test)] @@ -664,6 +672,9 @@ mod tests { assert_eq!(parse_rotated_ts("det.log", "det"), None); assert_eq!(parse_rotated_ts("det.notanumber.log", "det"), None); assert_eq!(parse_rotated_ts("unrelated.txt", "det"), None); + assert_eq!(parse_rotated_ts("det.-5.log", "det"), None); + assert_eq!(parse_rotated_ts("det.+5.log", "det"), None); + assert_eq!(parse_rotated_ts(".5.log", ""), None); } #[test] diff --git a/src/mcp/server.rs b/src/mcp/server.rs index b088483d2..760a3e1e6 100644 --- a/src/mcp/server.rs +++ b/src/mcp/server.rs @@ -371,7 +371,7 @@ pub async fn init_app_context() -> Result, McpError> { tracing::warn!("Failed to load .env file at {}: {e}", env_path.display()); } - let db_file_path = data_file_path(&data_dir, "data.db") + let db_file_path = data_file_path(&data_dir, crate::database::legacy_backups::LEGACY_DATABASE) .map_err(|e| McpError::internal_error(format!("db path: {e}"), None))?; let db = if db_file_path.exists() { Arc::new( diff --git a/src/ui/network_chooser_screen.rs b/src/ui/network_chooser_screen.rs index 4eb0f7585..2a4b053ff 100644 --- a/src/ui/network_chooser_screen.rs +++ b/src/ui/network_chooser_screen.rs @@ -1380,7 +1380,7 @@ impl NetworkChooserScreen { } /// Shown when the retention setting cannot be read; the backend then deletes nothing. -const BACKUP_RETENTION_UNREADABLE: &str = "Your backup retention setting could not be loaded, so no upgrade backups are deleted automatically. Choose a setting and save it to fix this."; +const BACKUP_RETENTION_UNREADABLE: &str = "Your backup retention setting could not be read, so no old upgrade backups are deleted. Choose a backup retention setting and save it."; /// Shown when the new policy was saved but some expired backups could not be deleted. const BACKUP_RETENTION_CLEANUP_INCOMPLETE: &str = "Your backup retention setting was saved, but some old upgrade backups could not be deleted. The app will try again the next time it starts."; @@ -1390,6 +1390,7 @@ fn backup_retention_saved_banner( deleted: usize, cleanup_incomplete: bool, ) -> (String, MessageType) { + // The wording matches `TaskError::UpgradeBackupCleanup`, prefixed by the save outcome. if cleanup_incomplete { return ( BACKUP_RETENTION_CLEANUP_INCOMPLETE.to_owned(), @@ -1404,6 +1405,17 @@ fn backup_retention_saved_banner( (message, MessageType::Success) } +/// Confirmation text for switching to a stricter retention of `days` days. +fn stricter_retention_confirmation(days: u16) -> String { + if days == 1 { + "Upgrade backups older than one day will be deleted now, and from then on automatically. Deleted backups cannot be restored. Continue?".to_owned() + } else { + format!( + "Upgrade backups older than {days} days will be deleted now, and from then on automatically. Deleted backups cannot be restored. Continue?" + ) + } +} + /// Settings-screen state for the upgrade-backup retention policy. /// /// Edits are saved only once finished, and only after the backend confirms is the @@ -1492,9 +1504,7 @@ impl BackupRetentionForm { BackupRetention::DeleteAfterDays(days) if stricter => { let dialog = ConfirmationDialog::new( "Delete old upgrade backups?", - format!( - "Upgrade backups older than {days} days will be deleted now, and from then on automatically. Deleted backups cannot be restored. Continue?" - ), + stricter_retention_confirmation(days), ) .confirm_text(Some("Delete Old Backups")) .cancel_text(Some("Keep Backups")) @@ -1548,6 +1558,17 @@ impl BackupRetentionForm { self.revert(); } + /// Re-sync with the persisted policy when the screen is shown again. + /// + /// A save whose answer never arrived no longer blocks the form: its result, if it + /// still comes, is applied as usual. A queued save or an open confirmation is kept. + fn reload(&mut self, saved: Option) { + self.pending = None; + if !self.busy() { + *self = Self::new(saved); + } + } + /// Render the section. The save task it may queue is emitted by [`Self::take_task`]. fn show(&mut self, ui: &mut Ui, dark_mode: bool) { ui.label( @@ -1580,11 +1601,9 @@ impl BackupRetentionForm { ui.add_space(6.0); ui.horizontal(|ui| { ui.label("Keep upgrade backups for this many days:"); - let response = ui.add( - egui::DragValue::new(&mut self.days) - .range(BackupRetention::MIN_DAYS..=BackupRetention::MAX_DAYS) - .speed(1), - ); + // Not clamped: an out-of-range entry stays visible with the error + // below and is never saved, instead of silently becoming the limit. + let response = ui.add(egui::DragValue::new(&mut self.days).speed(1)); // Save only a finished edit, not every intermediate drag or keystroke. editing = response.dragged() || response.has_focus(); }); @@ -1649,6 +1668,8 @@ impl ScreenLike for NetworkChooserScreen { // changed on another surface (e.g. the onboarding row) since this cached // root screen was constructed. self.selected_role = self.current_app_context().user_role(); + self.backup_retention + .reload(self.current_app_context().backup_retention().ok()); } fn ui(&mut self, ui: &mut egui::Ui) -> AppAction { @@ -1718,13 +1739,20 @@ impl ScreenLike for NetworkChooserScreen { if let BackendTaskSuccessResult::UpdatedBackupRetention { retention, deleted, - cleanup_incomplete, + cleanup_failure, } = backend_task_success_result { self.backup_retention.save_succeeded(retention); let (message, message_type) = - backup_retention_saved_banner(deleted, cleanup_incomplete); - MessageBanner::set_global(self.current_app_context().egui_ctx(), message, message_type); + backup_retention_saved_banner(deleted, cleanup_failure.is_some()); + let banner = MessageBanner::set_global( + self.current_app_context().egui_ctx(), + message, + message_type, + ); + if let Some(failure) = cleanup_failure { + banner.with_details(failure); + } } else if let BackendTaskSuccessResult::NetworkDatabaseCleared { .. } = &backend_task_success_result { @@ -1817,6 +1845,48 @@ mod tests { saved_retention(form.take_task()) } + #[test] + fn out_of_range_retention_days_are_never_saved() { + let mut form = BackupRetentionForm::new(Some(BackupRetention::default())); + for days in [0, 3651, 4000] { + form.days = days; + assert_eq!(step(&mut form, false), None, "{days} must not be saved"); + assert_eq!(step(&mut form, true), None, "{days} must not be saved"); + assert_eq!(form.days, days, "the entry stays visible, not clamped"); + } + } + + #[test] + fn stricter_retention_confirmation_reads_naturally() { + assert!(stricter_retention_confirmation(1).contains("older than one day will")); + assert!(stricter_retention_confirmation(30).contains("older than 30 days will")); + } + + #[test] + fn lost_backup_retention_result_does_not_wedge_the_form() { + let mut form = BackupRetentionForm::new(Some(BackupRetention::default())); + form.days = 200; + assert!(step(&mut form, false).is_some()); + assert!(form.busy()); + + form.reload(Some(BackupRetention::default())); + assert!(!form.busy(), "arriving again unblocks the form"); + assert_eq!((form.enabled, form.days), (true, 90)); + + // A late answer is still applied. + form.save_succeeded(BackupRetention::DeleteAfterDays(200)); + assert_eq!(form.saved, Some(BackupRetention::DeleteAfterDays(200))); + assert_eq!(form.days, 200); + } + + #[test] + fn reload_picks_up_a_setting_that_became_readable() { + let mut form = BackupRetentionForm::new(None); + form.reload(Some(BackupRetention::KeepForever)); + assert_eq!(form.saved, Some(BackupRetention::KeepForever)); + assert!(!form.enabled); + } + #[test] fn backup_retention_form_saves_looser_policies_without_confirmation() { let mut form = BackupRetentionForm::new(Some(BackupRetention::default())); diff --git a/src/ui/tokens/tokens_screen/mod.rs b/src/ui/tokens/tokens_screen/mod.rs index ff20837c7..6bfea6ced 100644 --- a/src/ui/tokens/tokens_screen/mod.rs +++ b/src/ui/tokens/tokens_screen/mod.rs @@ -3245,6 +3245,10 @@ mod tests { } #[test] + #[expect( + clippy::disallowed_methods, + reason = "test fixture setup/teardown outside any production deletion path" + )] fn test_token_creator_ui_builds_correct_contract() { let db_file_path = "test_db_token_creator"; let _ = std::fs::remove_file(db_file_path); // Clean up from previous runs @@ -3547,6 +3551,10 @@ mod tests { } #[test] + #[expect( + clippy::disallowed_methods, + reason = "test fixture setup/teardown outside any production deletion path" + )] fn test_distribution_function_random() { let db_file_path = "test_db_distribution_random"; let _ = std::fs::remove_file(db_file_path); // Clean up from previous runs @@ -3663,6 +3671,10 @@ mod tests { } #[test] + #[expect( + clippy::disallowed_methods, + reason = "test fixture setup/teardown outside any production deletion path" + )] fn test_parse_token_build_args_fails_with_empty_token_name() { let db_file_path = "test_db_empty_token_name"; let _ = std::fs::remove_file(db_file_path); // Clean up from previous runs diff --git a/src/utils/backup_prune.rs b/src/utils/backup_prune.rs new file mode 100644 index 000000000..d99d396f9 --- /dev/null +++ b/src/utils/backup_prune.rs @@ -0,0 +1,81 @@ +//! Time-based expiry shared by every upgrade-backup location. +//! +//! One algorithm for the bridge, upstream `pre-migration-*` and legacy `data.db` +//! backups: delete those older than the retention period, except the newest backup +//! of the database, which is kept whatever its age. A wrong-forward clock can +//! therefore never delete the last recovery copy. + +use super::file_deletion::{DeletionIntent, delete_file}; +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; +use std::time::{Duration, SystemTime}; + +/// A published backup of one database, with its creation time or the error that +/// prevented reading it. +pub(crate) type Candidate = (PathBuf, std::io::Result); + +/// Delete every candidate older than `max_age` at `now`, except the newest one. +/// +/// All candidates must be backups of the database named by `intent`. A candidate +/// whose creation time cannot be read is kept and reported. A candidate that is +/// already gone (e.g. pruned concurrently from another network's context) counts +/// as handled. Attempts every candidate, syncs the directories it deleted from, +/// and returns the first failure; `Ok` carries the number of backups deleted. +pub(crate) fn prune_expired( + candidates: Vec, + now: SystemTime, + max_age: Duration, + intent: DeletionIntent<'_>, +) -> std::io::Result { + let mut first_error = None; + let mut dated = Vec::with_capacity(candidates.len()); + for (path, created) in candidates { + match created { + Ok(created) => dated.push((created, path)), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + first_error.get_or_insert(error); + } + } + } + dated.sort(); + // The newest backup is the floor: it survives any age and any clock. + dated.pop(); + let mut removed = 0; + let mut touched = BTreeSet::new(); + for (created, path) in dated { + if !crate::model::backup_retention::backup_expired(created, now, max_age) { + continue; + } + match delete_file(&path, intent) { + Ok(()) => { + removed += 1; + if let Some(directory) = path.parent() { + touched.insert(directory.to_owned()); + } + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + first_error.get_or_insert(error); + } + } + } + for directory in touched { + if let Err(error) = sync_directory(&directory) { + first_error.get_or_insert(error); + } + } + first_error.map_or(Ok(removed), Err) +} + +/// Persist directory-entry changes (unlinks) in `directory`. +/// +/// Windows cannot open a directory as a `File` without extra flags and NTFS +/// journals metadata changes, so this is a no-op there. +pub(crate) fn sync_directory(directory: &Path) -> std::io::Result<()> { + #[cfg(unix)] + std::fs::File::open(directory)?.sync_all()?; + #[cfg(not(unix))] + let _ = directory; + Ok(()) +} diff --git a/src/utils/file_deletion.rs b/src/utils/file_deletion.rs index 5c6038f8a..d46340b16 100644 --- a/src/utils/file_deletion.rs +++ b/src/utils/file_deletion.rs @@ -5,7 +5,9 @@ //! states its purpose as a [`DeletionIntent`], and the chokepoint enforces, in //! this order, before anything is unlinked: //! -//! 1. the target is a regular file — never a symlink or directory; +//! 1. the target is a regular file — never a directory, and a symlink only under +//! [`DeletionIntent::NetworkClear`], where the link itself is unlinked and its +//! target is never followed; //! 2. a hard deny-list of live application files by **name** — `det-app.sqlite`, //! `data.db`, every network's wallet and shielded database, `*.pwsvault`, //! `.env`, `*.premigration`, each with its `-wal`/`-shm`/`-journal` sidecar. @@ -15,7 +17,12 @@ //! the device/inode, of every protected file in the intent's data directory, //! which catches `..`, symlinked-directory and case aliases; //! 4. on Unix, a single hard link, so no alias of another file is removed; -//! 5. the intent's scope — the directories that purpose may delete from. +//! 5. the intent's scope — the directories that purpose may delete from and, +//! for backups and logs, the exact file-name grammar of that purpose. +//! +//! Per platform: Unix gets every layer. Windows has no device/inode or hard-link +//! check, so it instead refuses names that can alias another file there (8.3 +//! short names with `~`, `:` stream suffixes, a trailing `.` or space). //! //! The chokepoint needs no `AppContext`: each intent carries the paths it is //! confined to, so it also serves pre-context callers such as log rotation. @@ -32,14 +39,16 @@ const SQLITE_SIDECAR_SUFFIXES: [&str; 3] = ["-wal", "-shm", "-journal"]; /// Why a deletion needs to happen, and therefore where it may reach. #[derive(Debug, Clone, Copy)] pub(crate) enum DeletionIntent<'a> { - /// A retained backup of `database`, inside the database's directory, its - /// `backups/` directory, or the upstream auto-backup directory. + /// A retained backup of `database`: a bridge snapshot of it next to it, an + /// upstream `pre-migration-*` snapshot of it in the auto-backup directory, or, + /// for the legacy `data.db`, a `backups/data_backup_*.db` copy. Backup { /// The live database the backup was taken from. database: &'a Path, }, /// Resyncable per-network data removed by a network clear: anything under - /// `/spv//`, plus the `/spv/.lock` + /// `/spv//` (including emptied directories and symlinks, + /// which are unlinked, never followed), plus the `/spv/.lock` /// storage lock. NetworkClear { /// The application data directory. @@ -66,7 +75,8 @@ pub(crate) enum DeletionRefused { /// The refused target. path: PathBuf, }, - /// The target is a symlink, directory, or other non-regular file. + /// The target is a directory, a symlink outside a network clear, or another + /// non-regular file. #[error("Only regular files can be deleted.")] NotRegularFile { /// The refused target. @@ -100,20 +110,24 @@ impl From for std::io::Error { /// missing target is [`std::io::ErrorKind::NotFound`], so callers that tolerate /// an already-gone file keep doing so. Other I/O failures pass through. pub(crate) fn delete_file(path: &Path, intent: DeletionIntent<'_>) -> std::io::Result<()> { - check(path, intent)?; - std::fs::remove_file(path) + let metadata = check(path, intent)?; + unlink(path, &metadata) } -/// Delete the directory tree at `dir`, unlinking every file through -/// [`delete_file`] and then removing the emptied directories. +/// Delete the directory tree at `dir`, unlinking every file through the same +/// checks as [`delete_file`] and then removing the emptied directories. /// -/// Symlinks are never followed; one inside the tree is refused like any other -/// non-regular file. Stops at the first failure. +/// Only [`DeletionIntent::NetworkClear`] may remove directories, and only strictly +/// inside its network's SPV directory; each directory is scope-checked before it is +/// read and again before it is removed. Symlinks are unlinked, never followed. The +/// walk uses an explicit stack, so depth cannot overflow the call stack. Stops at +/// the first failure. /// /// # Errors /// -/// As [`delete_file`]; a missing `dir` is [`std::io::ErrorKind::NotFound`], and -/// a `dir` that is not a real directory is refused as [`DeletionRefused::NotRegularFile`]. +/// As [`delete_file`]; a missing `dir` is [`std::io::ErrorKind::NotFound`], a `dir` +/// that is not a real directory is refused as [`DeletionRefused::NotRegularFile`], +/// and one outside the intent's scope as [`DeletionRefused::OutsideScope`]. pub(crate) fn delete_tree(dir: &Path, intent: DeletionIntent<'_>) -> std::io::Result<()> { if !std::fs::symlink_metadata(dir)?.is_dir() { return Err(DeletionRefused::NotRegularFile { @@ -121,30 +135,99 @@ pub(crate) fn delete_tree(dir: &Path, intent: DeletionIntent<'_>) -> std::io::Re } .into()); } - for entry in std::fs::read_dir(dir)? { - let path = entry?.path(); - if std::fs::symlink_metadata(&path)?.is_dir() { - delete_tree(&path, intent)?; - } else { - delete_file(&path, intent)?; + check_dir_scope(dir, intent)?; + // Post-order walk: a directory is removed once all its entries are gone. + let mut stack = vec![(dir.to_owned(), false)]; + while let Some((directory, emptied)) = stack.pop() { + if emptied { + check_dir_scope(&directory, intent)?; + remove_empty_dir(&directory)?; + continue; + } + stack.push((directory.clone(), true)); + for entry in std::fs::read_dir(&directory)? { + let path = entry?.path(); + if std::fs::symlink_metadata(&path)?.is_dir() { + check_dir_scope(&path, intent)?; + stack.push((path, false)); + } else { + delete_file(&path, intent)?; + } + } + } + Ok(()) +} + +/// Refuse a directory outside what `intent` may remove. +fn check_dir_scope(dir: &Path, intent: DeletionIntent<'_>) -> std::io::Result<()> { + let located = canonical_location(dir)?; + let allowed = match intent { + DeletionIntent::NetworkClear { data_dir, network } => { + let root = canonical_dir(&network_spv_dir(data_dir, network)); + located.starts_with(&root) && located != root } + DeletionIntent::Backup { .. } | DeletionIntent::Log { .. } => false, + }; + if allowed { + Ok(()) + } else { + Err(DeletionRefused::OutsideScope { + path: dir.to_owned(), + } + .into()) } - // Only an empty directory can be removed here; every file went through the checks. +} + +/// The only raw directory removal; its caller has emptied and scope-checked `dir`. +#[expect( + clippy::disallowed_methods, + reason = "the deletion chokepoint itself; `dir` is empty and scope-checked" +)] +fn remove_empty_dir(dir: &Path) -> std::io::Result<()> { std::fs::remove_dir(dir) } -fn check(path: &Path, intent: DeletionIntent<'_>) -> std::io::Result<()> { +/// The only raw file unlink; its caller has passed every chokepoint check. +#[expect( + clippy::disallowed_methods, + reason = "the deletion chokepoint itself; every check has passed" +)] +fn unlink(path: &Path, metadata: &std::fs::Metadata) -> std::io::Result<()> { + // Windows removes a symlink to a directory as a directory; the link is still + // unlinked without following it. + #[cfg(windows)] + { + use std::os::windows::fs::FileTypeExt; + if metadata.file_type().is_symlink_dir() { + return std::fs::remove_dir(path); + } + } + #[cfg(not(windows))] + let _ = metadata; + std::fs::remove_file(path) +} + +fn network_spv_dir(data_dir: &Path, network: Network) -> PathBuf { + data_dir + .join("spv") + .join(crate::wallet_backend::network_prefix(network)) +} + +/// Every check before an unlink; returns the target's own (unfollowed) metadata. +fn check(path: &Path, intent: DeletionIntent<'_>) -> std::io::Result { let refused = |make: fn(PathBuf) -> DeletionRefused| -> std::io::Error { make(path.to_owned()).into() }; let metadata = std::fs::symlink_metadata(path)?; - if !metadata.is_file() { + let unlinkable_link = + metadata.is_symlink() && matches!(intent, DeletionIntent::NetworkClear { .. }); + if !metadata.is_file() && !unlinkable_link { return Err(refused(|path| DeletionRefused::NotRegularFile { path })); } let name = path .file_name() .and_then(|name| name.to_str()) .ok_or_else(|| refused(|path| DeletionRefused::OutsideScope { path }))?; - if is_protected_name(name) { + if is_protected_name(name) || (cfg!(windows) && is_windows_alias_name(name)) { return Err(refused(|path| DeletionRefused::ProtectedFile { path })); } let located = canonical_location(path)?; @@ -168,21 +251,7 @@ fn check(path: &Path, intent: DeletionIntent<'_>) -> std::io::Result<()> { if !in_scope(&located, name, intent) { return Err(refused(|path| DeletionRefused::OutsideScope { path })); } - Ok(()) -} - -/// Every network, via an exhaustive match: a new [`Network`] variant fails to -/// compile here until its databases are added to the deny-list. -fn all_networks() -> [Network; 4] { - let _exhaustive = |network: Network| match network { - Network::Mainnet | Network::Testnet | Network::Devnet | Network::Regtest => (), - }; - [ - Network::Mainnet, - Network::Testnet, - Network::Devnet, - Network::Regtest, - ] + Ok(metadata) } /// Whether `name` is a live application file, regardless of its directory. @@ -192,16 +261,24 @@ fn is_protected_name(name: &str) -> bool { .iter() .find_map(|suffix| name.strip_suffix(suffix)) .unwrap_or(&name); - if matches!(base, "det-app.sqlite" | "data.db" | ".env") + if [ + crate::wallet_backend::APP_DATABASE_FILE, + crate::database::legacy_backups::LEGACY_DATABASE, + crate::app_dir::ENV_FILE, + ] + .contains(&base) || base.ends_with(".pwsvault") || base.ends_with(".premigration") { return true; } - all_networks().into_iter().any(|network| { - let prefix = crate::wallet_backend::network_prefix(network); - base == format!("det-{prefix}.sqlite") || base == format!("det-{prefix}-shielded.sqlite") - }) + crate::wallet_backend::all_networks() + .into_iter() + .any(|network| { + let prefix = crate::wallet_backend::network_prefix(network); + base == format!("det-{prefix}.sqlite") + || base == format!("det-{prefix}-shielded.sqlite") + }) } /// The live files in the intent's data directory, with their SQLite sidecars. @@ -212,12 +289,12 @@ fn protected_paths(intent: DeletionIntent<'_>) -> Vec { DeletionIntent::Log { dir, .. } => (dir, None), }; let mut roots = vec![ - data_dir.join("det-app.sqlite"), - data_dir.join("data.db"), - data_dir.join(".env"), + crate::wallet_backend::app_database_path(data_dir), + data_dir.join(crate::database::legacy_backups::LEGACY_DATABASE), + data_dir.join(crate::app_dir::ENV_FILE), crate::context::AppContext::secret_store_path(data_dir), ]; - for network in all_networks() { + for network in crate::wallet_backend::all_networks() { roots.push(crate::wallet_backend::wallet_database_path( data_dir, network, )); @@ -280,43 +357,47 @@ fn same_inode(_target: &std::fs::Metadata, _protected: &Path) -> bool { false } +/// Whether `name` could alias another file on Windows: an 8.3 short name (`~`), an +/// alternate data stream (`:`), or a trailing `.` or space that Win32 strips. +fn is_windows_alias_name(name: &str) -> bool { + name.contains(['~', ':']) || name.ends_with(['.', ' ']) +} + fn in_scope(located: &Path, name: &str, intent: DeletionIntent<'_>) -> bool { let Some(parent) = located.parent() else { return false; }; + let is = |dir: &Path| canonical_dir(dir) == parent; match intent { DeletionIntent::Backup { database } => { let directory = parent_of(database); - [ - directory.to_owned(), - directory.join(crate::database::legacy_backups::BACKUP_DIR), - platform_wallet_storage::default_auto_backup_dir(database), - ] - .iter() - .any(|allowed| canonical_dir(allowed) == parent) + let legacy = database.file_name().is_some_and(|file| { + file.eq_ignore_ascii_case(crate::database::legacy_backups::LEGACY_DATABASE) + }); + (is(directory) + && crate::wallet_backend::platform_compatibility::is_bridge_backup_name( + database, name, + )) + || (is(&platform_wallet_storage::default_auto_backup_dir(database)) + && crate::wallet_backend::platform_compatibility::upstream_backup_timestamp( + database, name, + ) + .is_some()) + || (legacy + && is(&directory.join(crate::database::legacy_backups::BACKUP_DIR)) + && crate::database::legacy_backups::backup_timestamp(name).is_some()) } DeletionIntent::NetworkClear { data_dir, network } => { let spv = data_dir.join("spv"); - let prefix = crate::wallet_backend::network_prefix(network); - let lock = format!("{prefix}.lock"); - parent.starts_with(canonical_dir(&spv.join(prefix))) - || (parent == canonical_dir(&spv) && name == lock) + let lock = format!("{}.lock", crate::wallet_backend::network_prefix(network)); + parent.starts_with(canonical_dir(&network_spv_dir(data_dir, network))) + || (is(&spv) && name == lock) } DeletionIntent::Log { dir, stem } => { - parent == canonical_dir(dir) && is_rotated_log_name(name, stem) + is(dir) && crate::logging::parse_rotated_ts(name, stem).is_some() } } } -/// Whether `name` is `..log`. -fn is_rotated_log_name(name: &str, stem: &str) -> bool { - !stem.is_empty() - && name - .strip_prefix(stem) - .and_then(|rest| rest.strip_prefix('.')) - .and_then(|rest| rest.strip_suffix(".log")) - .is_some_and(|digits| !digits.is_empty() && digits.bytes().all(|b| b.is_ascii_digit())) -} - #[cfg(test)] mod tests; diff --git a/src/utils/file_deletion/tests.rs b/src/utils/file_deletion/tests.rs index d49814fdd..820b05a05 100644 --- a/src/utils/file_deletion/tests.rs +++ b/src/utils/file_deletion/tests.rs @@ -1,3 +1,8 @@ +#![expect( + clippy::disallowed_methods, + reason = "test fixture setup/teardown outside any production deletion path" +)] + use super::*; /// Every protected base name in a data directory, as the app lays them out. @@ -9,7 +14,7 @@ fn protected_files(data_dir: &Path) -> Vec { data_dir.join("secrets/det-secrets.pwsvault"), data_dir.join("data.db.premigration"), ]; - for network in all_networks() { + for network in crate::wallet_backend::all_networks() { files.push(crate::wallet_backend::wallet_database_path( data_dir, network, )); @@ -251,6 +256,38 @@ fn delete_file_refuses_targets_outside_the_intent_scope() { network: Network::Testnet, }, ), + ( + dir.path().join("app.ron"), + DeletionIntent::Backup { + database: &database, + }, + ), + ( + dir.path().join("det-app.sqlite.platform-upgrade.lock"), + DeletionIntent::Backup { + database: &database, + }, + ), + ( + dir.path() + .join("det-mainnet.sqlite.platform-67d4ef3-backup-a.sqlite"), + DeletionIntent::Backup { + database: &database, + }, + ), + ( + dir.path().join("backups/data_backup_20000101_000000.db"), + DeletionIntent::Backup { + database: &database, + }, + ), + ( + dir.path() + .join("backups/auto/pre-migration-det-mainnet-1-to-2-20000101T000000Z.db"), + DeletionIntent::Backup { + database: &database, + }, + ), ( dir.path().join("det.log"), DeletionIntent::Log { @@ -289,6 +326,7 @@ fn delete_file_refuses_targets_outside_the_intent_scope() { fn delete_file_deletes_targets_in_scope() { let dir = tempfile::tempdir().unwrap(); let database = dir.path().join("det-testnet.sqlite"); + let legacy = dir.path().join("data.db"); let cases = [ ( dir.path() @@ -298,11 +336,16 @@ fn delete_file_deletes_targets_in_scope() { }, ), ( - dir.path().join("backups/data_backup_20000101_000000.db"), + dir.path() + .join("det-testnet.sqlite.platform-67d4ef3-backup-b.pending"), DeletionIntent::Backup { database: &database, }, ), + ( + dir.path().join("backups/data_backup_20000101_000000.db"), + DeletionIntent::Backup { database: &legacy }, + ), ( dir.path() .join("backups/auto/pre-migration-det-testnet-1-to-2-20000101T000000Z.db"), @@ -395,6 +438,128 @@ fn delete_tree_removes_a_cache_tree_but_stops_at_a_protected_file() { assert!(planted.exists()); } +/// A network clear unlinks a symlinked cache entry itself (e.g. chain data moved to +/// another disk) without following it; other intents still refuse symlinks. +#[cfg(unix)] +#[test] +fn network_clear_unlinks_symlinks_without_following_them() { + let dir = tempfile::tempdir().unwrap(); + let outside = tempfile::tempdir().unwrap(); + std::fs::write(outside.path().join("keep.dat"), b"keep").unwrap(); + let spv = dir.path().join("spv/testnet"); + std::fs::create_dir_all(&spv).unwrap(); + let intent = DeletionIntent::NetworkClear { + data_dir: dir.path(), + network: Network::Testnet, + }; + let dir_link = spv.join("block_headers"); + std::os::unix::fs::symlink(outside.path(), &dir_link).unwrap(); + let file_link = spv.join("peers.dat"); + std::os::unix::fs::symlink(outside.path().join("keep.dat"), &file_link).unwrap(); + for link in [&dir_link, &file_link] { + delete_file(link, intent).unwrap_or_else(|e| panic!("{link:?}: {e:?}")); + assert!( + link.symlink_metadata().is_err(), + "{link:?} must be unlinked" + ); + } + assert!(outside.path().join("keep.dat").exists()); + + let backup_link = dir + .path() + .join("det-app.sqlite.platform-67d4ef3-backup-a.sqlite"); + std::os::unix::fs::symlink(outside.path().join("keep.dat"), &backup_link).unwrap(); + let database = dir.path().join("det-app.sqlite"); + let error = delete_file( + &backup_link, + DeletionIntent::Backup { + database: &database, + }, + ) + .unwrap_err(); + assert!(matches!( + refusal(&error), + DeletionRefused::NotRegularFile { .. } + )); +} + +#[test] +fn delete_tree_refuses_directories_outside_its_scope() { + let dir = tempfile::tempdir().unwrap(); + let database = dir.path().join("det-app.sqlite"); + let clear = DeletionIntent::NetworkClear { + data_dir: dir.path(), + network: Network::Testnet, + }; + let cases = [ + (dir.path().join("spv/testnet"), clear), + (dir.path().join("spv/mainnet/blocks"), clear), + ( + dir.path().join("backups/auto"), + DeletionIntent::Backup { + database: &database, + }, + ), + ( + dir.path().join("logs"), + DeletionIntent::Log { + dir: dir.path(), + stem: "det", + }, + ), + ]; + for (target, intent) in cases { + std::fs::create_dir_all(&target).unwrap(); + let error = delete_tree(&target, intent).unwrap_err(); + assert!( + matches!(refusal(&error), DeletionRefused::OutsideScope { .. }), + "{target:?}: {error:?}" + ); + assert!(target.is_dir(), "{target:?}"); + } +} + +#[test] +fn delete_tree_handles_deep_trees_without_recursion() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("spv/testnet/filters"); + // Short components keep the full path within OS limits. + let mut deepest = root.clone(); + for _ in 0..1000 { + deepest.push("d"); + } + std::fs::create_dir_all(&deepest).unwrap(); + std::fs::write(deepest.join("x.dat"), b"x").unwrap(); + delete_tree( + &root, + DeletionIntent::NetworkClear { + data_dir: dir.path(), + network: Network::Testnet, + }, + ) + .unwrap(); + assert!(!root.exists()); +} + +#[test] +fn windows_alias_names_are_recognised() { + for name in [ + "DET-AP~1.SQL", + "det-app.sqlite::$DATA", + "det-app.sqlite.", + "det-app.sqlite ", + ] { + assert!(is_windows_alias_name(name), "{name}"); + } + for name in [ + "det-app.sqlite.platform-67d4ef3-backup-a1.sqlite", + "pre-migration-det-app-1-to-2-20000101T000000Z.db", + "det.0000000001.log", + ] { + assert!(!is_windows_alias_name(name), "{name}"); + } +} + #[cfg(unix)] #[test] fn delete_tree_never_follows_a_symlinked_directory() { @@ -409,31 +574,31 @@ fn delete_tree_never_follows_a_symlinked_directory() { data_dir: dir.path(), network: Network::Testnet, }; - let error = delete_tree(&cache, intent).unwrap_err(); - assert!(matches!( - refusal(&error), - DeletionRefused::NotRegularFile { .. } - )); - assert!(outside.join("keep.dat").exists()); let error = delete_tree(&cache.join("link"), intent).unwrap_err(); assert!(matches!( refusal(&error), DeletionRefused::NotRegularFile { .. } )); + // Inside the tree the link itself is unlinked; its target is never entered. + delete_tree(&cache, intent).unwrap(); + assert!(!cache.exists()); + assert!(outside.join("keep.dat").exists()); } #[test] fn rotated_log_names_are_exact() { - assert!(is_rotated_log_name("det.0000000001.log", "det")); + let rotated = |name: &str, stem: &str| crate::logging::parse_rotated_ts(name, stem).is_some(); + assert!(rotated("det.0000000001.log", "det")); for name in [ "det.log", "det..log", "det.12a.log", + "det.-5.log", "det.1.log.bak", "detx.1.log", "other.1.log", ] { - assert!(!is_rotated_log_name(name, "det"), "{name}"); + assert!(!rotated(name, "det"), "{name}"); } - assert!(!is_rotated_log_name(".1.log", "")); + assert!(!rotated(".1.log", "")); } diff --git a/src/utils/mod.rs b/src/utils/mod.rs index 7292c1e36..574b3edf9 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod backup_prune; pub mod egui_mpsc; pub(crate) mod file_deletion; pub mod parsers; diff --git a/src/wallet_backend/mod.rs b/src/wallet_backend/mod.rs index 0f8815d36..6dde5805e 100644 --- a/src/wallet_backend/mod.rs +++ b/src/wallet_backend/mod.rs @@ -529,6 +529,28 @@ impl std::fmt::Debug for WalletBackend { } } +/// File name of the cross-network app database in the data directory. +pub(crate) const APP_DATABASE_FILE: &str = "det-app.sqlite"; + +/// The cross-network app k/v database: `/det-app.sqlite`. +pub(crate) fn app_database_path(data_dir: &Path) -> std::path::PathBuf { + data_dir.join(APP_DATABASE_FILE) +} + +/// Every network, via an exhaustive match: a new [`Network`] variant fails to +/// compile here until each caller (e.g. the deletion deny-list) is reviewed. +pub(crate) fn all_networks() -> [Network; 4] { + let _exhaustive = |network: Network| match network { + Network::Mainnet | Network::Testnet | Network::Devnet | Network::Regtest => (), + }; + [ + Network::Mainnet, + Network::Testnet, + Network::Devnet, + Network::Regtest, + ] +} + /// The durable per-network wallet database: `/det-.sqlite`. /// /// A sibling of `det-app.sqlite`, deliberately outside `/spv//` diff --git a/src/wallet_backend/platform_compatibility/engine.rs b/src/wallet_backend/platform_compatibility/engine.rs index 17fcf262d..2c8382785 100644 --- a/src/wallet_backend/platform_compatibility/engine.rs +++ b/src/wallet_backend/platform_compatibility/engine.rs @@ -3,6 +3,8 @@ use std::path::{Path, PathBuf}; use rusqlite::{Connection, OpenFlags, OptionalExtension, TransactionBehavior}; +use super::storage_failure::StorageFailure; + const OLD_SCHEMA: &str = include_str!("fixtures/67d4ef3.sql"); const TARGET_SCHEMA: &str = include_str!("fixtures/e3cd7cf.sql"); const HISTORY: &str = "refinery_schema_history"; @@ -12,9 +14,7 @@ const IDENTITY_INDEX_KEY: &str = "det:identity_index:v1"; #[derive(Debug, thiserror::Error)] pub enum UpgradeError { /// Another process holds the database (`SQLITE_BUSY` / `SQLITE_LOCKED`). - #[error( - "Your wallet data is open in another Dash Evo Tool window or command-line session. Close it and try again." - )] + #[error("{}", crate::backend_task::error::WALLET_DATA_IN_USE)] InUse(#[source] Box), #[error( "Could not upgrade wallet data because the disk is full. Free up disk space and try again." @@ -75,37 +75,36 @@ impl UpgradeError { } } +impl UpgradeError { + /// The variant for an environmental storage failure, keeping `error` as the source. + pub(super) fn from_failure( + failure: StorageFailure, + error: Box, + ) -> Self { + match failure { + StorageFailure::InUse => Self::InUse(error), + StorageFailure::Full => Self::StorageFull(error), + StorageFailure::Unavailable => Self::StorageUnavailable(error), + StorageFailure::OutOfMemory => Self::OutOfMemory(error), + StorageFailure::AccessDenied => Self::AccessDenied(error), + } + } +} + impl From for UpgradeError { fn from(error: rusqlite::Error) -> Self { - match error.sqlite_error_code() { - Some(rusqlite::ErrorCode::DatabaseBusy | rusqlite::ErrorCode::DatabaseLocked) => { - Self::InUse(Box::new(error)) - } - Some(rusqlite::ErrorCode::DiskFull) => Self::StorageFull(Box::new(error)), - Some(rusqlite::ErrorCode::SystemIoFailure) => Self::StorageUnavailable(Box::new(error)), - Some(rusqlite::ErrorCode::OutOfMemory) => Self::OutOfMemory(Box::new(error)), - Some(rusqlite::ErrorCode::PermissionDenied | rusqlite::ErrorCode::ReadOnly) => { - Self::AccessDenied(Box::new(error)) - } - _ => Self::Sqlite(error), + match StorageFailure::of_sqlite(&error) { + Some(failure) => Self::from_failure(failure, Box::new(error)), + None => Self::Sqlite(error), } } } impl From for UpgradeError { fn from(error: std::io::Error) -> Self { - use std::io::ErrorKind; - match error.kind() { - ErrorKind::StorageFull => Self::StorageFull(Box::new(error)), - ErrorKind::OutOfMemory => Self::OutOfMemory(Box::new(error)), - ErrorKind::Interrupted - | ErrorKind::WouldBlock - | ErrorKind::TimedOut - | ErrorKind::ResourceBusy => Self::StorageUnavailable(Box::new(error)), - ErrorKind::PermissionDenied | ErrorKind::ReadOnlyFilesystem => { - Self::AccessDenied(Box::new(error)) - } - _ => Self::Io(error), + match StorageFailure::of_io(&error) { + Some(failure) => Self::from_failure(failure, Box::new(error)), + None => Self::Io(error), } } } @@ -247,7 +246,74 @@ fn backup_prefix(path: &Path) -> Option { )) } +/// Upstream snapshot names: `pre-migration---to--.db`. +const UPSTREAM_PREFIX: &str = "pre-migration-"; +const UPSTREAM_SUFFIX: &str = ".db"; +/// UTC timestamp format in an upstream snapshot name. +const UPSTREAM_TIMESTAMP_FORMAT: &str = "%Y%m%dT%H%M%SZ"; + +/// Whether `name` is a bridge snapshot of `database` (published `.sqlite` or +/// unpublished `.pending`), which lives next to the database. +pub(crate) fn is_bridge_backup_name(database: &Path, name: &str) -> bool { + let Some(prefix) = backup_prefix(database) else { + return false; + }; + name.strip_prefix(&prefix) + .and_then(|suffix| { + suffix + .strip_suffix(".sqlite") + .or_else(|| suffix.strip_suffix(".pending")) + }) + .is_some_and(|suffix| { + !suffix.is_empty() && suffix.bytes().all(|b| b.is_ascii_alphanumeric()) + }) +} + +/// The migration an upstream snapshot of `database` precedes, and when it was taken. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct UpstreamSnapshot { + from: u32, + to: u32, + taken: std::time::SystemTime, +} + +fn parse_upstream_name(database: &Path, name: &str) -> Option { + let stem = database.file_stem()?.to_str()?; + // DET database names are already valid upstream stems; reject lossy/ambiguous names. + if stem.is_empty() + || stem.len() > 32 + || !stem + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_') + { + return None; + } + let suffix = name + .strip_prefix(UPSTREAM_PREFIX)? + .strip_prefix(stem)? + .strip_prefix('-')? + .strip_suffix(UPSTREAM_SUFFIX)?; + let (versions, timestamp) = suffix.rsplit_once('-')?; + let (from, to) = versions.split_once("-to-")?; + let taken = chrono::NaiveDateTime::parse_from_str(timestamp, UPSTREAM_TIMESTAMP_FORMAT).ok()?; + Some(UpstreamSnapshot { + from: from.parse().ok()?, + to: to.parse().ok()?, + taken: taken.and_utc().into(), + }) +} + +/// When the upstream snapshot `name` of `database` was taken, or `None` when +/// `name` is not one. +pub(crate) fn upstream_backup_timestamp( + database: &Path, + name: &str, +) -> Option { + parse_upstream_name(database, name).map(|snapshot| snapshot.taken) +} + /// Retained upgrade backups of the database at `path`. +#[cfg(test)] fn backups(path: &Path) -> std::io::Result> { backups_in(path, Some(&default_auto_dir(path))) } @@ -279,7 +345,7 @@ fn backups_in(path: &Path, auto_dir: Option<&Path>) -> std::io::Result) -> std::io::Result { let mut found = Vec::new(); let mut rejected = Vec::new(); - let (Some(parent), Some(prefix)) = (path.parent(), backup_prefix(path)) else { + let Some(parent) = path.parent() else { return Ok(BackupScan { found, rejected }); }; for directory in [Some(parent), auto_dir].into_iter().flatten() { @@ -297,18 +363,8 @@ fn scan_backups_in(path: &Path, auto_dir: Option<&Path>) -> std::io::Result found.push(entry.path()), @@ -322,36 +378,8 @@ fn scan_backups_in(path: &Path, auto_dir: Option<&Path>) -> std::io::Result bool { - let Some(stem) = path.file_stem().and_then(|stem| stem.to_str()) else { - return false; - }; - // DET database names are already valid upstream stems; reject lossy/ambiguous names. - if stem.is_empty() - || stem.len() > 32 - || !stem - .bytes() - .all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_') - { - return false; - } - let Some(suffix) = name - .strip_prefix(&format!("pre-migration-{stem}-")) - .and_then(|suffix| suffix.strip_suffix(".db")) - else { - return false; - }; - let Some((versions, timestamp)) = suffix.rsplit_once('-') else { - return false; - }; - let Some((from, to)) = versions.split_once("-to-") else { - return false; - }; - from.parse::().is_ok() - && to.parse::().is_ok() - && chrono::NaiveDateTime::parse_from_str(timestamp, "%Y%m%dT%H%M%SZ").is_ok() -} - +/// Scan-time classification of a candidate; the deletion itself is policed by the +/// [`delete_file`](crate::utils::file_deletion::delete_file) chokepoint. fn validate_backup_file(backup: &Path, database: &Path) -> std::io::Result<()> { let metadata = std::fs::symlink_metadata(backup)?; if !metadata.is_file() || backup == database { @@ -379,7 +407,6 @@ fn validate_backup_file(backup: &Path, database: &Path) -> std::io::Result<()> { } fn remove_backup(backup: &Path, database: &Path) -> std::io::Result<()> { - validate_backup_file(backup, database)?; crate::utils::file_deletion::delete_file( backup, crate::utils::file_deletion::DeletionIntent::Backup { database }, @@ -432,99 +459,86 @@ pub(super) fn backup_lock(path: &Path) -> std::io::Result { }) } -/// Retain the newest usable snapshot across both upgrade-backup formats. +/// [`tidy_backups_locked`] under a freshly taken lifecycle lock. #[cfg(test)] -pub(super) fn retain_one_backup(path: &Path, auto_dir: Option<&Path>) -> std::io::Result<()> { +pub(super) fn tidy_backups(path: &Path, auto_dir: Option<&Path>) -> std::io::Result<()> { let guard = backup_lock(path)?; - retain_one_backup_locked(&guard, auto_dir) + tidy_backups_locked(&guard, auto_dir) } -pub(super) fn retain_one_backup_locked( +/// Housekeeping that loses no recovery data: delete unpublished `.pending` copies left +/// by a crash, and published snapshots that are byte-identical to a newer retained +/// snapshot of the same migration (each failed open writes another copy of the +/// unchanged database). Every other snapshot is left to time-based retention. +/// +/// The scan is strict, so an unexpected candidate fails the call before anything is +/// deleted. +pub(super) fn tidy_backups_locked( guard: &BackupGuard, auto_dir: Option<&Path>, ) -> std::io::Result<()> { let path = &guard.path; - let mut snapshots = Vec::new(); + // Group published snapshots by the migration they precede; newest first. + let mut groups: std::collections::BTreeMap, Vec<_>> = + std::collections::BTreeMap::new(); for backup in backups_in(path, auto_dir)? { - // Unpublished copies may be incomplete and must never supersede a recovery snapshot. + let name = backup + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or_default(); if backup .extension() .is_some_and(|extension| extension == "pending") { remove_backup(&backup, path)?; - } else { - snapshots.push((std::fs::metadata(&backup)?.modified()?, backup)); + continue; } - } - snapshots.sort(); - // Keep the newest snapshot that proves usable. An interrupted copy (e.g. a killed - // bridge from an older build that wrote straight to the final name) can be the newest - // file; it must never supersede an older complete snapshot. - let Some(survivor) = snapshots - .iter() - .rposition(|(_, backup)| usable_snapshot(backup)) - else { - if !snapshots.is_empty() { - // Nothing is proven usable, so nothing is proven expendable either. - tracing::warn!( - count = snapshots.len(), - "No upgrade backup passed validation; keeping all of them" - ); + // Bridge snapshots all precede the same pinned-profile upgrade. + let migration = + parse_upstream_name(path, name).map(|snapshot| (snapshot.from, snapshot.to)); + let created = backup_created(path, &backup)?; + groups.entry(migration).or_default().push((created, backup)); + } + for mut group in groups.into_values() { + group.sort_by(|a, b| b.cmp(a)); + let mut kept: Vec = Vec::with_capacity(group.len()); + for (_, backup) in group { + let mut duplicate = false; + for newer in &kept { + if same_contents(newer, &backup)? { + duplicate = true; + break; + } + } + if duplicate { + remove_backup(&backup, path)?; + } else { + kept.push(backup); + } } - return Ok(()); - }; - snapshots.remove(survivor); - for (_, backup) in snapshots { - remove_backup(&backup, path)?; } Ok(()) } -/// Whether `backup` is a complete, readable wallet database snapshot. -/// -/// Opens the file as immutable and read-only, then requires a clean `PRAGMA quick_check` -/// plus a non-empty migration history — an empty or truncated file fails one of these. -/// Any error while checking counts as unusable: retention then keeps an older proven copy. -fn usable_snapshot(backup: &Path) -> bool { - let check = || -> Result> { - // `immutable=1`: snapshots copied from a WAL database carry a WAL header, and a plain - // read-only open would leave `-wal`/`-shm` sidecars next to them. The caller holds - // the backup lock, so no DET writer changes the file while it is checked. - // `SQLITE_OPEN_NOFOLLOW` rejects a symlink anywhere in the path, so resolve the - // parent directory (e.g. macOS `/var` -> `/private/var`) and keep the entry name: - // a symlink at the snapshot itself is still refused. - let invalid = || std::io::Error::from(std::io::ErrorKind::InvalidInput); - let absolute = std::path::absolute(backup)?; - let parent = absolute.parent().ok_or_else(invalid)?.canonicalize()?; - let name = absolute.file_name().ok_or_else(invalid)?; - let mut uri = url::Url::from_file_path(parent.join(name)).map_err(|()| invalid())?; - uri.set_query(Some("immutable=1")); - let conn = Connection::open_with_flags( - uri.as_str(), - OpenFlags::SQLITE_OPEN_READ_ONLY - | OpenFlags::SQLITE_OPEN_URI - | OpenFlags::SQLITE_OPEN_NOFOLLOW, - )?; - let mut rows = conn.prepare("PRAGMA quick_check")?; - let report = rows - .query_map([], |r| r.get::<_, String>(0))? - .collect::, _>>()?; - if report != ["ok"] { - return Ok(false); +/// Whether two files hold the same bytes. Sizes are compared first, then the +/// contents are streamed in fixed-size chunks, stopping at the first difference. +fn same_contents(a: &Path, b: &Path) -> std::io::Result { + use std::io::Read; + const CHUNK: usize = 64 * 1024; + if std::fs::symlink_metadata(a)?.len() != std::fs::symlink_metadata(b)?.len() { + return Ok(false); + } + let (mut a, mut b) = (std::fs::File::open(a)?, std::fs::File::open(b)?); + let (mut left, mut right) = (vec![0u8; CHUNK], vec![0u8; CHUNK]); + loop { + let read = a.read(&mut left)?; + if read == 0 { + // Equal lengths: `b` must be exhausted too, unless it grew meanwhile. + return Ok(b.read(&mut right[..1])? == 0); } - let has_history = conn - .prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?1")? - .exists([HISTORY])?; - Ok(has_history - && conn - .prepare(&format!("SELECT 1 FROM {}", quote(HISTORY)))? - .exists([])?) - }; - match check() { - Ok(usable) => usable, - Err(error) => { - tracing::debug!(backup = %backup.display(), ?error, "Upgrade backup is not usable"); - false + b.read_exact(&mut right[..read])?; + if left[..read] != right[..read] { + return Ok(false); } } } @@ -536,14 +550,16 @@ fn usable_snapshot(backup: &Path) -> bool { /// synced on every call (including a retry that finds nothing left to delete), /// so callers may retire their retry state afterwards. pub(crate) fn remove_backups(path: &Path) -> std::io::Result<()> { - remove_backups_with_sync(path, |_| true, sync_directory) + remove_backups_with_sync(path, |_| true, crate::utils::backup_prune::sync_directory) } /// Delete the upgrade backups of the database at `path` that are older than `max_age`. /// /// Covers both the bridge snapshots next to the database and upstream `pre-migration-*` -/// snapshots. Call it only once the database has opened successfully: it may remove the -/// last snapshot, which is safe only when the database no longer needs recovery. +/// snapshots, and always keeps the newest one, so it is safe whether or not the database +/// currently opens. A database with no backups is skipped without taking its lock, and +/// one whose lock is held (another context is opening or upgrading it) is skipped too; +/// the next pass covers it. /// /// Attempts every candidate and returns the first failure; a candidate that fails /// validation is reported but never blocks removing the others. `Ok` carries the number @@ -553,61 +569,58 @@ pub(crate) fn prune_expired_backups( max_age: std::time::Duration, now: std::time::SystemTime, ) -> std::io::Result { - let _guard = backup_lock(path)?; - let scan = scan_backups_in(path, Some(&default_auto_dir(path)))?; - let mut first_error = scan.rejected.into_iter().next().map(|(_, error)| error); - let mut removed = 0; - let mut touched = BTreeSet::new(); - for backup in scan.found { - let outcome = backup_created(&backup).and_then(|created| { - if !crate::model::backup_retention::backup_expired(created, now, max_age) { - return Ok(false); - } - remove_backup(&backup, path).map(|()| true) - }); - match outcome { - Ok(true) => { - removed += 1; - if let Some(directory) = backup.parent() { - touched.insert(directory.to_owned()); - } - } - Ok(false) => {} - Err(error) => { - first_error.get_or_insert(error); - } - } - } - for directory in touched { - if let Err(error) = sync_directory(&directory) { - first_error.get_or_insert(error); + let auto_dir = default_auto_dir(path); + let unlocked = scan_backups_in(path, Some(&auto_dir))?; + if unlocked.found.is_empty() && unlocked.rejected.is_empty() { + return Ok(0); + } + let _guard = match backup_lock(path) { + Ok(guard) => guard, + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + tracing::debug!(database = %path.display(), "Upgrade backup retention skipped a database in use"); + return Ok(0); } - } - first_error.map_or(Ok(removed), Err) -} - -/// When `backup` was created: the later of its modification time and the timestamp in -/// its name, if any. Taking the later one means a reset or skewed signal only delays -/// deletion, never hastens it. -fn backup_created(backup: &Path) -> std::io::Result { + Err(error) => return Err(error), + }; + let scan = scan_backups_in(path, Some(&auto_dir))?; + let rejected = scan.rejected.into_iter().next().map(|(_, error)| error); + let candidates = scan + .found + .into_iter() + .filter(|backup| { + backup + .extension() + .is_none_or(|extension| extension != "pending") + }) + .map(|backup| { + let created = backup_created(path, &backup); + (backup, created) + }) + .collect(); + let pruned = crate::utils::backup_prune::prune_expired( + candidates, + now, + max_age, + crate::utils::file_deletion::DeletionIntent::Backup { database: path }, + ); + match (pruned, rejected) { + (Err(error), _) | (Ok(_), Some(error)) => Err(error), + (Ok(removed), None) => Ok(removed), + } +} + +/// When `backup` of `database` was created: the later of its modification time and +/// the timestamp in its name, if any. Taking the later one means a reset or skewed +/// signal only delays deletion, never hastens it. +fn backup_created(database: &Path, backup: &Path) -> std::io::Result { let modified = std::fs::symlink_metadata(backup)?.modified()?; let named = backup .file_name() .and_then(|name| name.to_str()) - .and_then(upstream_backup_timestamp); + .and_then(|name| upstream_backup_timestamp(database, name)); Ok(named.map_or(modified, |named| named.max(modified))) } -/// The UTC creation timestamp embedded in an upstream `pre-migration-*` snapshot name. -fn upstream_backup_timestamp(name: &str) -> Option { - let (_, timestamp) = name - .strip_prefix("pre-migration-")? - .strip_suffix(".db")? - .rsplit_once('-')?; - let parsed = chrono::NaiveDateTime::parse_from_str(timestamp, "%Y%m%dT%H%M%SZ").ok()?; - Some(parsed.and_utc().into()) -} - fn remove_backups_with_sync( path: &Path, selected: impl Fn(&Path) -> bool, @@ -651,18 +664,6 @@ fn remove_backups_with_sync( first_error.map_or(Ok(()), Err) } -/// Persist directory-entry changes (unlinks) in `directory`. -/// -/// Windows cannot open a directory as a `File` without extra flags and NTFS -/// journals metadata changes, so this is a no-op there. -fn sync_directory(directory: &Path) -> std::io::Result<()> { - #[cfg(unix)] - std::fs::File::open(directory)?.sync_all()?; - #[cfg(not(unix))] - let _ = directory; - Ok(()) -} - #[cfg(test)] fn backup(path: &Path) -> Result { backup_with_hook(path, |_| Ok(())) @@ -706,27 +707,10 @@ fn backup_with_hook_locked( verify(&dest)?; drop(dest); file.as_file().sync_all()?; - // Strict scan before publication: an unexpected candidate aborts before anything changes. - let pending = file.path().to_owned(); - let superseded: Vec = backups(path)? - .into_iter() - .filter(|old| *old != pending) - .collect(); - let kept = pending.with_extension("sqlite"); + // Earlier snapshots stay: only time-based retention and open-time tidying remove them. + let kept = file.path().with_extension("sqlite"); file.persist_noclobber(&kept).map_err(|e| e.error)?; - sync_directory(parent)?; - // Prune only once the replacement is durably published, so a failed or interrupted - // publication never leaves zero recovery snapshots. The original is still untouched. - let mut touched = BTreeSet::new(); - for old in superseded { - remove_backup(&old, path)?; - if let Some(directory) = old.parent() { - touched.insert(directory.to_owned()); - } - } - for directory in touched { - sync_directory(&directory)?; - } + crate::utils::backup_prune::sync_directory(parent)?; Ok(kept) } diff --git a/src/wallet_backend/platform_compatibility/engine/tests.rs b/src/wallet_backend/platform_compatibility/engine/tests.rs index d6a9d2f7b..cc9433636 100644 --- a/src/wallet_backend/platform_compatibility/engine/tests.rs +++ b/src/wallet_backend/platform_compatibility/engine/tests.rs @@ -1,3 +1,8 @@ +#![expect( + clippy::disallowed_methods, + reason = "test fixture setup/teardown outside any production deletion path" +)] + use super::*; fn fixture() -> (tempfile::TempDir, PathBuf, PathBuf) { @@ -223,7 +228,7 @@ fn platform_compatibility_typed_validation_failure_never_rewrites_original() { } #[test] -fn platform_compatibility_repeated_failed_rebuilds_keep_one_backup() { +fn platform_compatibility_repeated_failed_rebuilds_are_deduplicated() { let (dir, path, _target) = fixture(); let before = snapshot(&path); for attempt in 0..3 { @@ -241,11 +246,17 @@ fn platform_compatibility_repeated_failed_rebuilds_keep_one_backup() { .unwrap_err(); assert!(matches!(error, UpgradeError::Verification)); } + assert_eq!( + backup_files(dir.path()).len(), + 3, + "publication never deletes an earlier snapshot" + ); + tidy_backups(&path, None).unwrap(); let backups = backup_files(dir.path()); assert_eq!( backups.len(), 1, - "Retried upgrades must not accumulate backups." + "identical copies of the unchanged original collapse to one" ); assert_eq!(snapshot(&backups[0]), before); } @@ -396,15 +407,66 @@ fn platform_compatibility_failed_publication_keeps_previous_snapshot() { assert_eq!(snapshot(&previous), snapshot(&path)); } -/// A successful publication still leaves exactly one snapshot. +/// Publication keeps earlier snapshots: only retention and tidying remove them. #[test] -fn platform_compatibility_publication_prunes_superseded_snapshots() { +fn platform_compatibility_publication_keeps_earlier_snapshots() { let (dir, path, _target) = fixture(); let previous = backup(&path).unwrap(); let replacement = backup(&path).unwrap(); assert_ne!(previous, replacement); - assert!(!previous.exists()); - assert_eq!(backup_files(dir.path()), vec![replacement]); + assert!(previous.exists()); + assert!(replacement.exists()); + assert_eq!(backup_files(dir.path()).len(), 2); +} + +/// Tidying removes a snapshot only when a newer one of the same migration has the +/// same bytes; a snapshot that differs by even one byte, or precedes another +/// migration, is never removed. +#[test] +fn platform_compatibility_tidy_removes_only_identical_copies() { + let (dir, path, _target) = fixture(); + let auto = dir.path().join("backups/auto"); + std::fs::create_dir_all(&auto).unwrap(); + let original = backup(&path).unwrap(); + let copy = backup(&path).unwrap(); + set_mtime(&original, -3600); + let bytes = std::fs::read(©).unwrap(); + let mut differing = bytes.clone(); + *differing.last_mut().unwrap() ^= 0xFF; + let upstream = |versions: &str, second: u32| { + auto.join(format!( + "pre-migration-wallet-{versions}-20260915T12000{second}Z.db" + )) + }; + let (same_a, same_b, other_migration, changed) = ( + upstream("1-to-2", 1), + upstream("1-to-2", 2), + upstream("2-to-3", 3), + upstream("1-to-2", 4), + ); + for (file, contents) in [ + (&same_a, &bytes), + (&same_b, &bytes), + (&other_migration, &bytes), + (&changed, &differing), + ] { + std::fs::write(file, contents).unwrap(); + } + set_mtime(&same_a, -3600); + + tidy_backups(&path, Some(&auto)).unwrap(); + + assert!(!original.exists(), "the older identical bridge copy goes"); + assert!(copy.exists(), "the newest identical copy stays"); + assert!(!same_a.exists(), "the older identical upstream copy goes"); + assert!(same_b.exists()); + assert!( + other_migration.exists(), + "another migration is never merged" + ); + assert!(changed.exists(), "a differing snapshot is never removed"); + tidy_backups(&path, Some(&auto)).unwrap(); + assert_eq!(backup_files(dir.path()), vec![copy]); } /// One invalid candidate must not shield the valid snapshots from deletion: @@ -442,15 +504,20 @@ fn platform_compatibility_invalid_candidate_does_not_block_removal_of_valid_back } #[test] -fn platform_compatibility_prune_failure_preserves_original() { - let (dir, path, target) = fixture(); +fn platform_compatibility_tidy_rejects_unexpected_candidate_without_changes() { + let (dir, path, _target) = fixture(); let before = snapshot(&path); + let pending = dir + .path() + .join("wallet.sqlite.platform-67d4ef3-backup-crash.pending"); + std::fs::copy(&path, &pending).unwrap(); std::fs::create_dir( dir.path() .join("wallet.sqlite.platform-67d4ef3-backup-blocked.sqlite"), ) .unwrap(); - assert!(upgrade(&path, &target, |_| Ok(())).is_err()); + assert!(tidy_backups(&path, None).is_err()); + assert!(pending.exists(), "a strict scan failure deletes nothing"); assert_eq!(snapshot(&path), before); } @@ -677,7 +744,7 @@ fn platform_compatibility_pending_cleanup_preserves_published_snapshot() { .path() .join("wallet.sqlite.platform-67d4ef3-backup-crash.pending"); std::fs::copy(&path, &pending).unwrap(); - retain_one_backup(&path, None).unwrap(); + tidy_backups(&path, None).unwrap(); assert!(!pending.exists()); assert!(published.exists()); assert!(path.exists()); @@ -781,7 +848,7 @@ fn platform_compatibility_active_snapshot_survives_concurrent_cleanup() { break PathBuf::from(path); } }; - let retained = retain_one_backup(&path, None); + let retained = tidy_backups(&path, None); let removed = remove_backups(&path); let exists = pending.exists(); if crash { @@ -796,14 +863,14 @@ fn platform_compatibility_active_snapshot_survives_concurrent_cleanup() { if crash { assert!(!status.success()); assert!(pending.exists()); - retain_one_backup(&path, None).unwrap(); + tidy_backups(&path, None).unwrap(); assert!(!pending.exists()); continue; } assert!(status.success()); let published = pending.with_extension("sqlite"); assert_eq!(snapshot(&published), snapshot(&path)); - retain_one_backup(&path, None).unwrap(); + tidy_backups(&path, None).unwrap(); assert!(published.exists()); remove_backups(&path).unwrap(); assert!(!published.exists()); @@ -815,11 +882,11 @@ fn platform_compatibility_one_guard_covers_retention_and_upgrade() { let (_dir, path, target) = fixture(); let before = snapshot(&path); let guard = backup_lock(&path).unwrap(); - retain_one_backup_locked(&guard, None).unwrap(); + tidy_backups_locked(&guard, None).unwrap(); let backup = upgrade_locked(&guard, &target, |_| Ok(())) .unwrap() .unwrap(); - retain_one_backup_locked(&guard, None).unwrap(); + tidy_backups_locked(&guard, None).unwrap(); assert_eq!(snapshot(&backup), before); } @@ -839,115 +906,35 @@ fn set_mtime(path: &Path, seconds_from_now: i64) { .unwrap(); } -/// An interrupted copy left under a final snapshot name (empty or truncated) -/// must never replace an older, complete recovery snapshot — in either format. +/// An interrupted copy left under a final snapshot name (empty or truncated) differs +/// from the complete snapshot, so tidying keeps both, in either format. #[test] -fn platform_compatibility_retention_keeps_valid_snapshot_over_newer_incomplete_one() { +fn platform_compatibility_tidy_keeps_differing_snapshots() { let (dir, path, _target) = fixture(); let before = snapshot(&path); - let full = std::fs::read(backup(&path).unwrap()).unwrap(); + let valid = backup(&path).unwrap(); + let full = std::fs::read(&valid).unwrap(); let auto = dir.path().join("backups/auto"); std::fs::create_dir_all(&auto).unwrap(); - let cases: [(&str, &[u8], PathBuf); 4] = [ - ( - "empty bridge", - b"", - dir.path() - .join("wallet.sqlite.platform-67d4ef3-backup-killed.sqlite"), - ), - ( - "truncated bridge", - &full[..full.len() / 2], - dir.path() - .join("wallet.sqlite.platform-67d4ef3-backup-killed.sqlite"), - ), - ( - "header-only bridge", - &full[..100], - dir.path() - .join("wallet.sqlite.platform-67d4ef3-backup-killed.sqlite"), - ), - ( - "empty upstream", - b"", - auto.join("pre-migration-wallet-1-to-2-20260915T120000Z.db"), - ), + let bridge = dir + .path() + .join("wallet.sqlite.platform-67d4ef3-backup-killed.sqlite"); + let upstream = auto.join("pre-migration-wallet-1-to-2-20260915T120000Z.db"); + let cases: [(&str, &[u8], &PathBuf); 4] = [ + ("empty bridge", b"", &bridge), + ("truncated bridge", &full[..full.len() / 2], &bridge), + ("header-only bridge", &full[..100], &bridge), + ("empty upstream", b"", &upstream), ]; + set_mtime(&valid, -3600); for (case, bytes, incomplete) in cases { - let valid = backup_files(dir.path()) - .into_iter() - .find(|p| *p != incomplete) - .unwrap(); - set_mtime(&valid, -3600); - std::fs::write(&incomplete, bytes).unwrap(); - retain_one_backup(&path, Some(&auto)).unwrap(); - assert!(valid.exists(), "{case}: the valid snapshot must survive"); + std::fs::write(incomplete, bytes).unwrap(); + tidy_backups(&path, Some(&auto)).unwrap(); assert_eq!(snapshot(&valid), before, "{case}"); - assert!( - !incomplete.exists(), - "{case}: the incomplete copy is pruned" - ); + assert!(incomplete.exists(), "{case}: a differing copy is kept"); } } -/// With no snapshot proven usable, retention deletes nothing. -#[test] -fn platform_compatibility_retention_keeps_all_when_none_is_usable() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("wallet.sqlite"); - let older = dir - .path() - .join("wallet.sqlite.platform-67d4ef3-backup-a1.sqlite"); - let newer = dir - .path() - .join("wallet.sqlite.platform-67d4ef3-backup-b2.sqlite"); - std::fs::write(&older, b"not a database").unwrap(); - std::fs::write(&newer, b"").unwrap(); - set_mtime(&older, -3600); - retain_one_backup(&path, None).unwrap(); - assert!(older.exists()); - assert!(newer.exists()); -} - -/// Validating a snapshot copied from a WAL-mode database opens it read-only -/// without leaving journal sidecars next to it. -#[test] -fn platform_compatibility_snapshot_validation_leaves_no_sidecars() { - let (dir, path, _target) = fixture(); - Connection::open(&path) - .unwrap() - .query_row("PRAGMA journal_mode=WAL", [], |_| Ok(())) - .unwrap(); - let kept = backup(&path).unwrap(); - let before: BTreeSet<_> = std::fs::read_dir(dir.path()) - .unwrap() - .map(|e| e.unwrap().file_name()) - .collect(); - assert!(usable_snapshot(&kept)); - let after: BTreeSet<_> = std::fs::read_dir(dir.path()) - .unwrap() - .map(|e| e.unwrap().file_name()) - .collect(); - assert_eq!(after, before); -} - -/// A symlinked ancestor directory (e.g. macOS `/var` -> `/private/var`) does not make a -/// complete snapshot unusable, while a symlink at the snapshot entry itself still does. -#[cfg(unix)] -#[test] -fn platform_compatibility_snapshot_validation_follows_ancestor_symlinks() { - let (dir, path, _target) = fixture(); - let kept = backup(&path).unwrap(); - let links = tempfile::tempdir().unwrap(); - let alias = links.path().join("alias"); - std::os::unix::fs::symlink(dir.path(), &alias).unwrap(); - let name = kept.file_name().unwrap(); - assert!(usable_snapshot(&alias.join(name))); - let entry_link = links.path().join(name); - std::os::unix::fs::symlink(&kept, &entry_link).unwrap(); - assert!(!usable_snapshot(&entry_link)); -} - const RETENTION_DAY: std::time::Duration = std::time::Duration::from_secs(24 * 60 * 60); /// Expiry covers both backup formats, measures age from the later of the file's @@ -1006,19 +993,27 @@ fn platform_compatibility_prune_expired_backups_by_age() { assert!(path.exists(), "the live database is never a candidate"); } -/// Expiry also removes the only remaining snapshot: callers run it after a successful -/// open, when the database no longer needs recovery. +/// The newest snapshot is the floor: it survives any age, even with the clock far in +/// the future, so the last recovery copy is never deleted. #[test] -fn platform_compatibility_prune_removes_expired_sole_snapshot() { +fn platform_compatibility_prune_keeps_newest_snapshot_for_any_clock() { let (dir, path, _target) = fixture(); - let snapshot = backup(&path).unwrap(); - set_mtime(&snapshot, -2 * 24 * 60 * 60); + let older = backup(&path).unwrap(); + let newest = backup(&path).unwrap(); + set_mtime(&older, -3 * 24 * 60 * 60); + set_mtime(&newest, -2 * 24 * 60 * 60); + let far_future = std::time::SystemTime::now() + RETENTION_DAY * 365 * 100; assert_eq!( - prune_expired_backups(&path, RETENTION_DAY, std::time::SystemTime::now()).unwrap(), + prune_expired_backups(&path, RETENTION_DAY, far_future).unwrap(), 1 ); - assert!(!snapshot.exists()); - assert!(backups(&path).unwrap().is_empty()); + assert!(!older.exists()); + assert!(newest.exists()); + assert_eq!( + prune_expired_backups(&path, RETENTION_DAY, far_future).unwrap(), + 0 + ); + assert_eq!(backups(&path).unwrap(), vec![newest]); drop(dir); } @@ -1033,6 +1028,10 @@ fn platform_compatibility_prune_reports_rejected_candidate_after_removing_others .join("det-testnet.sqlite.platform-67d4ef3-backup-old.sqlite"); std::fs::write(&expired, b"backup").unwrap(); set_mtime(&expired, -10 * 24 * 60 * 60); + let newest = dir + .path() + .join("det-testnet.sqlite.platform-67d4ef3-backup-new.sqlite"); + std::fs::write(&newest, b"newer backup").unwrap(); let blocked = dir .path() .join("det-testnet.sqlite.platform-67d4ef3-backup-blocked.sqlite"); @@ -1041,10 +1040,12 @@ fn platform_compatibility_prune_reports_rejected_candidate_after_removing_others prune_expired_backups(&path, RETENTION_DAY, std::time::SystemTime::now()).unwrap_err(); assert!(!expired.exists()); + assert!(newest.exists()); assert!(blocked.is_dir()); } -/// A held lifecycle lock means an open or upgrade is running: pruning backs off. +/// A held lifecycle lock means an open or upgrade is running: pruning skips the +/// database without failing, and a later pass covers it. #[test] fn platform_compatibility_prune_backs_off_while_lock_is_held() { let dir = tempfile::tempdir().unwrap(); @@ -1054,11 +1055,35 @@ fn platform_compatibility_prune_backs_off_while_lock_is_held() { .join("det-testnet.sqlite.platform-67d4ef3-backup-old.sqlite"); std::fs::write(&expired, b"backup").unwrap(); set_mtime(&expired, -10 * 24 * 60 * 60); - let _guard = backup_lock(&path).unwrap(); - let error = - prune_expired_backups(&path, RETENTION_DAY, std::time::SystemTime::now()).unwrap_err(); - assert_eq!(error.kind(), std::io::ErrorKind::WouldBlock); + std::fs::write( + dir.path() + .join("det-testnet.sqlite.platform-67d4ef3-backup-new.sqlite"), + b"newer backup", + ) + .unwrap(); + let guard = backup_lock(&path).unwrap(); + assert_eq!( + prune_expired_backups(&path, RETENTION_DAY, std::time::SystemTime::now()).unwrap(), + 0 + ); assert!(expired.exists()); + drop(guard); + assert_eq!( + prune_expired_backups(&path, RETENTION_DAY, std::time::SystemTime::now()).unwrap(), + 1 + ); +} + +/// A database with no backups is skipped before its lock file would be created. +#[test] +fn platform_compatibility_prune_skips_database_without_backups() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("det-devnet.sqlite"); + assert_eq!( + prune_expired_backups(&path, RETENTION_DAY, std::time::SystemTime::now()).unwrap(), + 0 + ); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 0); } /// Only a well-formed UTC timestamp counts as a creation time. A malformed name is @@ -1066,7 +1091,10 @@ fn platform_compatibility_prune_backs_off_while_lock_is_held() { #[test] fn platform_compatibility_prune_rejects_malformed_upstream_timestamps() { assert_eq!( - upstream_backup_timestamp("pre-migration-det-testnet-1-to-2-20200101T000000Z.db"), + upstream_backup_timestamp( + Path::new("det-testnet.sqlite"), + "pre-migration-det-testnet-1-to-2-20200101T000000Z.db" + ), Some( chrono::NaiveDate::from_ymd_opt(2020, 1, 1) .unwrap() @@ -1083,7 +1111,11 @@ fn platform_compatibility_prune_rejects_malformed_upstream_timestamps() { "pre-migration-det-testnet-1-to-2-.db", ]; for name in malformed { - assert_eq!(upstream_backup_timestamp(name), None, "{name}"); + assert_eq!( + upstream_backup_timestamp(Path::new("det-testnet.sqlite"), name), + None, + "{name}" + ); } let dir = tempfile::tempdir().unwrap(); diff --git a/src/wallet_backend/platform_compatibility/mod.rs b/src/wallet_backend/platform_compatibility/mod.rs index 53124f41f..8147cbc6b 100644 --- a/src/wallet_backend/platform_compatibility/mod.rs +++ b/src/wallet_backend/platform_compatibility/mod.rs @@ -1,8 +1,12 @@ //! Preserve databases created by the pinned Platform PR when opening the development release. mod engine; +mod storage_failure; pub use engine::UpgradeError; -pub(crate) use engine::{prune_expired_backups, remove_backups}; +pub(crate) use engine::{ + is_bridge_backup_name, prune_expired_backups, remove_backups, upstream_backup_timestamp, +}; +pub(crate) use storage_failure::StorageFailure; use platform_wallet::changeset::PlatformWalletPersistence; use platform_wallet_storage::{SqlitePersister, SqlitePersisterConfig, WalletStorageError}; @@ -10,34 +14,36 @@ use platform_wallet_storage::{SqlitePersister, SqlitePersisterConfig, WalletStor use crate::backend_task::error::TaskError; pub(crate) fn open(config: SqlitePersisterConfig) -> Result { - open_with_retention(config, engine::retain_one_backup_locked) + open_with_housekeeping(config, engine::tidy_backups_locked) } -fn open_with_retention( +fn open_with_housekeeping( config: SqlitePersisterConfig, - mut retain: impl FnMut(&engine::BackupGuard, Option<&std::path::Path>) -> std::io::Result<()>, + mut tidy: impl FnMut(&engine::BackupGuard, Option<&std::path::Path>) -> std::io::Result<()>, ) -> Result { let guard = engine::backup_lock(&config.path).map_err(lock_error)?; let auto_dir = config.auto_backup_dir.as_deref(); - // Check retention before another attempt can create a snapshot, including after a restart. - // Retention stays strict only when a snapshot could follow: a database that is already - // current still opens, since failed housekeeping does not endanger it. - if let Err(retention) = retain(&guard, auto_dir) { + // Tidy (strict scan, crash leftovers, identical copies) before another attempt can create + // a snapshot, including after a restart. It stays strict only when a snapshot could + // follow: a database that is already current still opens, since failed housekeeping + // does not endanger it. + if let Err(housekeeping) = tidy(&guard, auto_dir) { return open_current_only(&config).map_err(|probe| { tracing::debug!( error = ?probe, - "Wallet database is not openable without a snapshot; backup retention failure stands" + "Wallet database is not openable without a snapshot; backup housekeeping failure stands" ); - lock_error(retention) + lock_error(housekeeping) }); } let result = open_inner(&config, &guard); - // Post-open retention is best-effort housekeeping: it must neither discard a successful - // open nor mask the open's own error. The next open retries it before any new snapshot. - if let Err(error) = retain(&guard, auto_dir) { + // Post-open tidying is best-effort: it must neither discard a successful open nor mask + // the open's own error. It also drops a copy a failed open just duplicated. The next + // open retries it before any new snapshot. + if let Err(error) = tidy(&guard, auto_dir) { tracing::warn!( ?error, - "Upgrade backup retention failed after opening the wallet database" + "Upgrade backup housekeeping failed after opening the wallet database" ); } result @@ -58,7 +64,7 @@ fn open_current_only(config: &SqlitePersisterConfig) -> Result TaskError { fn validation_error(error: impl std::error::Error + Send + Sync + 'static) -> UpgradeError { use platform_wallet::changeset::PersistenceError; - use rusqlite::ErrorCode; - use std::io::ErrorKind; - type WrapError = fn(Box) -> UpgradeError; - - let mut cause: &dyn std::error::Error = &error; + if let Some(failure) = StorageFailure::in_chain(&error) { + return UpgradeError::from_failure(failure, Box::new(error)); + } let mut transient = false; - loop { - let category: Option = - if let Some(sqlite) = cause.downcast_ref::() { - match sqlite.sqlite_error_code() { - Some(ErrorCode::DatabaseBusy | ErrorCode::DatabaseLocked) => { - Some(UpgradeError::InUse) - } - Some(ErrorCode::DiskFull) => Some(UpgradeError::StorageFull), - Some(ErrorCode::SystemIoFailure) => Some(UpgradeError::StorageUnavailable), - Some(ErrorCode::OutOfMemory) => Some(UpgradeError::OutOfMemory), - Some(ErrorCode::PermissionDenied | ErrorCode::ReadOnly) => { - Some(UpgradeError::AccessDenied) - } - _ => None, - } - } else if let Some(io) = cause.downcast_ref::() { - match io.kind() { - ErrorKind::StorageFull => Some(UpgradeError::StorageFull), - ErrorKind::OutOfMemory => Some(UpgradeError::OutOfMemory), - ErrorKind::Interrupted - | ErrorKind::WouldBlock - | ErrorKind::TimedOut - | ErrorKind::ResourceBusy => Some(UpgradeError::StorageUnavailable), - ErrorKind::PermissionDenied | ErrorKind::ReadOnlyFilesystem => { - Some(UpgradeError::AccessDenied) - } - _ => None, - } - } else { - None - }; - if let Some(wrap) = category { - return wrap(Box::new(error)); - } - if let Some(persistence) = cause.downcast_ref::() { - transient |= persistence.is_transient(); - } - let Some(source) = cause.source() else { break }; - cause = source; + let mut cause: Option<&(dyn std::error::Error + 'static)> = Some(&error); + while let Some(current) = cause { + transient |= current + .downcast_ref::() + .is_some_and(PersistenceError::is_transient); + cause = current.source(); } if transient { UpgradeError::StorageUnavailable(Box::new(error)) @@ -182,6 +153,8 @@ mod tests { use super::*; use dash_sdk::dpp::identity::accessors::IdentityGettersV0; + /// Repeated failed opens can leave identical copies of the unchanged database; + /// the open's own tidying collapses them to the newest, even when the open fails. #[test] fn platform_compatibility_bounds_upstream_backups_after_failed_opens() { let dir = tempfile::tempdir().unwrap(); @@ -196,26 +169,46 @@ mod tests { .unwrap(); let auto = platform_wallet_storage::default_auto_backup_dir(&path); std::fs::create_dir_all(&auto).unwrap(); + let copy = std::fs::read(&path).unwrap(); for attempt in 1..=3 { - let old = auto.join(format!( + let snapshot = auto.join(format!( "pre-migration-wallet-1-to-2-20260915T12000{attempt}Z.db" )); - // A complete snapshot: retention only prunes in favour of a proven-usable copy. - rusqlite::Connection::open(&old) - .unwrap() - .execute_batch(include_str!("fixtures/67d4ef3.sql")) - .unwrap(); + std::fs::write(&snapshot, ©).unwrap(); assert!(open(SqlitePersisterConfig::new(&path)).is_err()); - assert_eq!(std::fs::read_dir(&auto).unwrap().count(), 1); - let retained = std::fs::read_dir(&auto) - .unwrap() - .next() - .unwrap() + let retained: Vec<_> = std::fs::read_dir(&auto) .unwrap() - .path(); - if retained != old { - std::fs::rename(retained, old).unwrap(); - } + .map(|entry| entry.unwrap().path()) + .collect(); + assert_eq!(retained, vec![snapshot], "attempt {attempt}"); + } + } + + /// Opening never deletes a published snapshot that differs from the others: only + /// the retention setting removes those, so "Keep forever" keeps every one. + #[test] + fn platform_compatibility_repeated_opens_keep_every_distinct_snapshot() { + let dir = tempfile::tempdir().unwrap(); + crate::app_dir::ensure_data_dir_exists(dir.path()).unwrap(); + let path = dir.path().join("wallet.sqlite"); + drop(open(SqlitePersisterConfig::new(&path)).unwrap()); + let auto = platform_wallet_storage::default_auto_backup_dir(&path); + std::fs::create_dir_all(&auto).unwrap(); + let snapshots = [ + dir.path() + .join("wallet.sqlite.platform-67d4ef3-backup-first.sqlite"), + auto.join("pre-migration-wallet-1-to-2-20200101T000000Z.db"), + auto.join("pre-migration-wallet-2-to-3-20210101T000000Z.db"), + auto.join("pre-migration-wallet-2-to-3-20220101T000000Z.db"), + ]; + for (index, snapshot) in snapshots.iter().enumerate() { + std::fs::write(snapshot, format!("snapshot {index}")).unwrap(); + } + for _ in 0..3 { + drop(open(SqlitePersisterConfig::new(&path)).unwrap()); + } + for snapshot in &snapshots { + assert!(snapshot.exists(), "{} must be kept", snapshot.display()); } } @@ -225,7 +218,7 @@ mod tests { crate::app_dir::ensure_data_dir_exists(dir.path()).unwrap(); let path = dir.path().join("wallet.sqlite"); let mut calls = 0; - let persister = open_with_retention(SqlitePersisterConfig::new(&path), |_, _| { + let persister = open_with_housekeeping(SqlitePersisterConfig::new(&path), |_, _| { calls += 1; if calls == 1 { Ok(()) @@ -240,7 +233,7 @@ mod tests { let bad = dir.path().join("corrupt.sqlite"); std::fs::write(&bad, b"not a sqlite database, just candy wrappers").unwrap(); let mut calls = 0; - let error = match open_with_retention(SqlitePersisterConfig::new(&bad), |_, _| { + let error = match open_with_housekeeping(SqlitePersisterConfig::new(&bad), |_, _| { calls += 1; if calls == 1 { Ok(()) @@ -286,7 +279,7 @@ mod tests { // First a brand-new database, then the same database once it is current. for _ in 0..2 { let mut calls = 0; - let persister = open_with_retention( + let persister = open_with_housekeeping( SqlitePersisterConfig::new(&path), failing_retention(&mut calls), ) @@ -339,7 +332,7 @@ mod tests { let before = schema(); let entries_before = dir_entries(dir.path()); let mut calls = 0; - let error = match open_with_retention( + let error = match open_with_housekeeping( SqlitePersisterConfig::new(&path), failing_retention(&mut calls), ) { @@ -437,9 +430,21 @@ mod tests { 1 ); drop(persister); - open(SqlitePersisterConfig::new(&path)).unwrap(); let upstream = platform_wallet_storage::default_auto_backup_dir(&path); - assert_eq!(std::fs::read_dir(upstream).unwrap().count(), 0); + let upstream_snapshots = || std::fs::read_dir(&upstream).unwrap().count(); + let before_reopen = upstream_snapshots(); + drop(open(SqlitePersisterConfig::new(&path)).unwrap()); + // Both recovery points stay: the bridge copy of the pinned profile and any + // upstream copy taken before its own migration. A current database adds none. + assert_eq!(upstream_snapshots(), before_reopen); + assert!( + std::fs::read_dir(dir.path()).unwrap().any(|entry| entry + .unwrap() + .file_name() + .to_string_lossy() + .contains(".platform-67d4ef3-backup-")), + "the bridge snapshot is kept" + ); } } diff --git a/src/wallet_backend/platform_compatibility/storage_failure.rs b/src/wallet_backend/platform_compatibility/storage_failure.rs new file mode 100644 index 000000000..ec08aee83 --- /dev/null +++ b/src/wallet_backend/platform_compatibility/storage_failure.rs @@ -0,0 +1,101 @@ +//! The one policy for which storage failures are environmental (and which of +//! those retrying can fix), shared by the upgrade engine and `TaskError`. + +use std::error::Error; + +/// An environmental storage failure, as opposed to a defect in the data itself. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum StorageFailure { + /// Another connection or process holds the database. + InUse, + /// The disk is full. + Full, + /// The OS reported an I/O failure or a transient interruption. + Unavailable, + /// Memory ran out. + OutOfMemory, + /// Permissions or a read-only medium prevent writing. + AccessDenied, +} + +impl StorageFailure { + /// Classify one error, ignoring its sources. + pub(crate) fn of(error: &(dyn Error + 'static)) -> Option { + if let Some(sqlite) = error.downcast_ref::() { + return Self::of_sqlite(sqlite); + } + error.downcast_ref::().and_then(Self::of_io) + } + + /// Classify the first error in `error`'s source chain that has a category. + pub(crate) fn in_chain(error: &(dyn Error + 'static)) -> Option { + let mut cause = Some(error); + while let Some(current) = cause { + if let Some(failure) = Self::of(current) { + return Some(failure); + } + cause = current.source(); + } + None + } + + pub(crate) fn of_sqlite(error: &rusqlite::Error) -> Option { + use rusqlite::ErrorCode; + match error.sqlite_error_code()? { + ErrorCode::DatabaseBusy | ErrorCode::DatabaseLocked => Some(Self::InUse), + ErrorCode::DiskFull => Some(Self::Full), + ErrorCode::SystemIoFailure => Some(Self::Unavailable), + ErrorCode::OutOfMemory => Some(Self::OutOfMemory), + ErrorCode::PermissionDenied | ErrorCode::ReadOnly => Some(Self::AccessDenied), + _ => None, + } + } + + pub(crate) fn of_io(error: &std::io::Error) -> Option { + use std::io::ErrorKind; + match error.kind() { + ErrorKind::StorageFull => Some(Self::Full), + ErrorKind::OutOfMemory => Some(Self::OutOfMemory), + ErrorKind::Interrupted + | ErrorKind::WouldBlock + | ErrorKind::TimedOut + | ErrorKind::ResourceBusy => Some(Self::Unavailable), + ErrorKind::PermissionDenied | ErrorKind::ReadOnlyFilesystem => Some(Self::AccessDenied), + _ => None, + } + } + + /// Whether freeing the resource and retrying can recover. Access failures + /// never fix themselves. + pub(crate) fn is_retryable(self) -> bool { + !matches!(self, Self::AccessDenied) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn storage_failure_walks_the_source_chain() { + let sqlite = rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error::new(rusqlite::ffi::SQLITE_FULL), + None, + ); + let wrapped = platform_wallet_storage::WalletStorageError::Sqlite(sqlite); + assert_eq!( + StorageFailure::in_chain(&wrapped), + Some(StorageFailure::Full) + ); + let io = std::io::Error::from(std::io::ErrorKind::ReadOnlyFilesystem); + assert_eq!( + StorageFailure::in_chain(&io), + Some(StorageFailure::AccessDenied) + ); + assert!(!StorageFailure::AccessDenied.is_retryable()); + assert_eq!( + StorageFailure::in_chain(&std::io::Error::other("plain")), + None + ); + } +} diff --git a/tests/backend-e2e/spv_reconnect.rs b/tests/backend-e2e/spv_reconnect.rs index f8c76b492..a134c5028 100644 --- a/tests/backend-e2e/spv_reconnect.rs +++ b/tests/backend-e2e/spv_reconnect.rs @@ -49,6 +49,10 @@ use std::time::Duration; /// It only needs live testnet peers (outbound TCP on port 19999). #[tokio_shared_rt::test(shared, flavor = "multi_thread", worker_threads = 12)] #[ignore = "network-dependent; requires testnet egress — no funded wallet needed"] +#[expect( + clippy::disallowed_methods, + reason = "test fixture setup/teardown outside any production deletion path" +)] async fn spv_reconnect_succeeds_without_already_open() { // ── Isolated context setup ────────────────────────────────────────────── let workdir = From 0c9e4f91600089ef330b2f7b65ce5368597f5b0e Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Wed, 30 Sep 2026 09:51:48 +0000 Subject: [PATCH 22/30] fix(storage): retry the upgrade lifecycle lock through fork-inherited holds `flock` belongs to the open file description, and a child forked by any thread of this process shares that description until it execs. A lifecycle lock that was just released could therefore still look held, so an open or backup failed with a retryable "storage unavailable" error whenever another thread spawned a process at that moment (seen as a CI flake in platform_compatibility_failed_publication_keeps_previous_snapshot). Contended locks are now retried for a 500 ms grace period before reporting WouldBlock. A regression test re-acquires the lock while processes spawn alongside; it fails without the grace period. Co-Authored-By: Claude Opus 5.5 --- .../platform_compatibility/engine.rs | 23 ++++++++++++- .../platform_compatibility/engine/tests.rs | 33 +++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/src/wallet_backend/platform_compatibility/engine.rs b/src/wallet_backend/platform_compatibility/engine.rs index 2c8382785..9dee94e04 100644 --- a/src/wallet_backend/platform_compatibility/engine.rs +++ b/src/wallet_backend/platform_compatibility/engine.rs @@ -418,6 +418,27 @@ pub(super) struct BackupGuard { _file: Option, } +/// How long a contended lifecycle lock is retried before reporting `WouldBlock`. +/// +/// `flock` belongs to the open file description, and a child that any thread of this +/// process forks shares it until the child execs. A lock just released by its guard +/// can therefore still look held for that brief window. +const LOCK_GRACE: std::time::Duration = std::time::Duration::from_millis(500); +const LOCK_RETRY_INTERVAL: std::time::Duration = std::time::Duration::from_millis(5); + +fn lock_with_grace(file: &std::fs::File) -> std::io::Result<()> { + let deadline = std::time::Instant::now() + LOCK_GRACE; + loop { + match file.try_lock() { + Ok(()) => return Ok(()), + Err(std::fs::TryLockError::WouldBlock) if std::time::Instant::now() < deadline => { + std::thread::sleep(LOCK_RETRY_INTERVAL); + } + Err(error) => return Err(error.into()), + } + } +} + pub(super) fn backup_lock(path: &Path) -> std::io::Result { let Some(name) = path.file_name() else { return Err(std::io::ErrorKind::InvalidInput.into()); @@ -451,7 +472,7 @@ pub(super) fn backup_lock(path: &Path) -> std::io::Result { Err(error) => return Err(error), }; validate_backup_file(&lock_path, path)?; - file.try_lock().map_err(std::io::Error::from)?; + lock_with_grace(&file)?; // Keep the pathname stable: unlinking it could let contenders lock different files. Ok(BackupGuard { path: path.to_owned(), diff --git a/src/wallet_backend/platform_compatibility/engine/tests.rs b/src/wallet_backend/platform_compatibility/engine/tests.rs index cc9433636..3354733b5 100644 --- a/src/wallet_backend/platform_compatibility/engine/tests.rs +++ b/src/wallet_backend/platform_compatibility/engine/tests.rs @@ -1135,3 +1135,36 @@ fn platform_compatibility_prune_rejects_malformed_upstream_timestamps() { assert!(auto.join(name).exists(), "{name}"); } } + +/// A child forked by another thread briefly shares the lock file's open description, +/// so a lock this thread just released can still look held; the grace period must +/// absorb that instead of reporting the database as busy. +#[test] +fn platform_compatibility_lock_survives_concurrent_process_spawns() { + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("wallet.sqlite"); + let stop = AtomicBool::new(false); + let spawned = AtomicUsize::new(0); + std::thread::scope(|scope| { + for _ in 0..2 { + scope.spawn(|| { + while !stop.load(Ordering::Relaxed) { + let _ = std::process::Command::new(std::env::current_exe().unwrap()) + .arg("--help") + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status(); + spawned.fetch_add(1, Ordering::Relaxed); + } + }); + } + // Keep re-acquiring while at least 40 processes are spawned alongside. + let mut outcome = Ok(()); + while outcome.is_ok() && spawned.load(Ordering::Relaxed) < 40 { + outcome = backup_lock(&path).map(drop); + } + stop.store(true, Ordering::Relaxed); + outcome.expect("a released lock must be re-acquirable while processes spawn"); + }); +} From 3fc061d3a2f388f975b1940c874f554d3181bc17 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Wed, 30 Sep 2026 10:35:34 +0000 Subject: [PATCH 23/30] fix(storage): keep the newest usable upgrade backup and bound failed-open copies The retention floor could keep an empty or truncated newest file while deleting the only complete snapshot. The floor is now the newest backup that passes a cheap structural SQLite check (magic, page size, file size covering every page the header claims, no hot rollback journal), for bridge, upstream and legacy backups alike. If nothing passes, the newest file is still kept. The floor no longer trusts a skewable clock alone: backups dated more than a day in the future do not displace one dated in the past, and the upstream snapshot of the latest migration (a clock-independent order) is kept too. The failed-open bound test depended on the umask: with a group-writable temp dir upstream refused the open before snapshotting, so it passed locally but failed in CI, where upstream writes a real pre-migration copy per failed open. It now uses an owner-only data dir and asserts that the real copies collapse to one. Also: one SPV directory helper, typed scan-classification errors, a shared lifecycle-lock path helper, truthful and shared cleanup-retry wording, and stale comments fixed. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 6 +- .../2026-09-10-platform-pin/upgrade-notes.md | 5 +- docs/user-stories.md | 2 +- src/backend_task/error.rs | 28 +- src/backend_task/mod.rs | 3 +- src/backend_task/system_task/mod.rs | 6 +- src/context/mod.rs | 4 +- src/context/wallet_lifecycle/mod.rs | 11 +- src/context/wallet_lifecycle/retention.rs | 19 +- .../tests/deletion_sentinel.rs | 16 +- src/database/legacy_backups.rs | 32 +- src/logging.rs | 2 - src/ui/network_chooser_screen.rs | 10 +- src/utils/backup_prune.rs | 309 +++++++++++++++++- src/utils/file_deletion.rs | 13 +- src/utils/file_deletion/tests.rs | 2 +- src/wallet_backend/mod.rs | 24 +- .../platform_compatibility/engine.rs | 67 +++- .../platform_compatibility/engine/tests.rs | 115 ++++++- .../platform_compatibility/mod.rs | 55 +++- 20 files changed, 615 insertions(+), 114 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 047074f88..10681570a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,9 +51,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). - **Upgrade backup retention setting**: Settings → Upgrade Backups controls how long the backup copies taken before a storage upgrade are kept. By default, backups older than 90 days are now deleted automatically; choose another - period (1 to 3650 days) or keep them forever. The newest backup of each - database is always kept, and a shorter period asks for confirmation because - it deletes backups at once. + period (1 to 3650 days) or keep them forever. The newest complete backup of + each database is always kept, and a shorter period asks for confirmation + because it deletes backups at once. - **Add wallet-created identity keys**: for identities loaded from a wallet on this device, the Add Key screen defaults to "Create from wallet", which diff --git a/docs/ai-design/2026-09-10-platform-pin/upgrade-notes.md b/docs/ai-design/2026-09-10-platform-pin/upgrade-notes.md index 514691fa8..b9219ab48 100644 --- a/docs/ai-design/2026-09-10-platform-pin/upgrade-notes.md +++ b/docs/ai-design/2026-09-10-platform-pin/upgrade-notes.md @@ -72,7 +72,10 @@ name format and be regular files, not symbolic links (or hard links on Unix). Backups older than a user-configurable retention period (default 90 days, or never) are deleted by one global policy covering every database in the data directory: bridge, upstream pre-migration, and legacy `data.db` backups. The -newest backup of each database is always kept, whatever its age or the clock. +newest usable backup of each database is always kept, whatever its age or the +clock: an empty or truncated file never counts, a backup dated in the future does +not displace one dated in the past, and the upstream snapshot of the latest +migration is kept as well. Clearing a network's data deletes that network's wallet database backups, because they copy its wallet and identity history; backups of the shared `det-app.sqlite` also hold other networks' data and stay under retention. Removing a single wallet diff --git a/docs/user-stories.md b/docs/user-stories.md index 56dc3b9a4..6a375121d 100644 --- a/docs/user-stories.md +++ b/docs/user-stories.md @@ -345,7 +345,7 @@ As a user whose wallet data was copied before a storage upgrade, I want old upgr - Upgrade backups older than 90 days are deleted automatically by default, covering app, wallet, upstream pre-migration and legacy `data.db` backups. - Settings → Upgrade Backups lets the user change the period (1 to 3650 days) or keep backups forever. Turning automatic deletion on lets the user choose the period before applying it, and any stricter setting asks for confirmation first, because it deletes backups immediately. A period outside the range is shown with an error and never saved. - The setting is one policy for every backup in the data directory, whichever network is open; a saved change applies to every network's backups as soon as the wallet data has opened. -- The newest backup of each database is always kept, whatever its age or the system clock, so the last recovery copy is never deleted automatically. +- The newest complete backup of each database is always kept, whatever its age or the system clock, so the last recovery copy is never deleted automatically. An empty or cut-off copy never counts as that backup. - With "keep forever", no published backup is deleted automatically. Opening the data removes only unfinished copies left by a crash and exact duplicates of a newer backup. - Removing a wallet or an identity does not delete upgrade backups. Clearing a network's data deletes that network's wallet backups; backups of the shared app data stay under the retention setting. diff --git a/src/backend_task/error.rs b/src/backend_task/error.rs index 87a83413f..0bcfbdd17 100644 --- a/src/backend_task/error.rs +++ b/src/backend_task/error.rs @@ -110,6 +110,13 @@ impl std::error::Error for BackendTaskJoinError {} const RPC_WALLET_NOT_SPECIFIED: i32 = -19; /// Shown whenever another window or session holds the wallet database. +/// Some expired upgrade backups remain. Retention runs again whenever wallet data +/// is opened (start, network switch) and whenever the setting is saved. +pub(crate) const UPGRADE_BACKUP_CLEANUP_INCOMPLETE: &str = "Some old upgrade backups could not be deleted. The app tries again the next time it opens your wallet data."; + +/// [`UPGRADE_BACKUP_CLEANUP_INCOMPLETE`] after a retention setting save. +pub(crate) const BACKUP_RETENTION_SAVED_CLEANUP_INCOMPLETE: &str = "Your backup retention setting was saved, but some old upgrade backups could not be deleted. The app tries again the next time it opens your wallet data."; + pub(crate) const WALLET_DATA_IN_USE: &str = "Your wallet data is open in another Dash Evo Tool window or command-line session. Close it and try again."; /// App-level error envelope for backend tasks. @@ -1198,9 +1205,7 @@ pub enum TaskError { }, /// Expired upgrade backups could not all be deleted. - #[error( - "Some old upgrade backups could not be deleted. The app will try again the next time it starts." - )] + #[error("{}", UPGRADE_BACKUP_CLEANUP_INCOMPLETE)] UpgradeBackupCleanup { #[source] source: std::io::Error, @@ -3767,6 +3772,23 @@ fn wallet_already_imported_message(alias: Option<&str>) -> String { mod tests { use super::*; use dash_sdk::dapi_client::DapiClientError; + + /// Both cleanup messages promise the same, truthful retry. + #[test] + fn upgrade_backup_cleanup_messages_share_the_retry_promise() { + let error = TaskError::UpgradeBackupCleanup { + source: std::io::Error::other("permission denied"), + }; + assert_eq!(error.to_string(), UPGRADE_BACKUP_CLEANUP_INCOMPLETE); + let (_, retry) = UPGRADE_BACKUP_CLEANUP_INCOMPLETE + .split_once(". ") + .expect("two sentences"); + assert_eq!( + retry, + "The app tries again the next time it opens your wallet data." + ); + assert!(BACKUP_RETENTION_SAVED_CLEANUP_INCOMPLETE.ends_with(retry)); + } use dash_sdk::dapi_client::transport::TransportError; use dash_sdk::dpp::consensus::basic::data_contract::{ DecimalsOverLimitError, InvalidTokenBaseSupplyError, InvalidTokenLanguageCodeError, diff --git a/src/backend_task/mod.rs b/src/backend_task/mod.rs index 2a536f277..d4857ed07 100644 --- a/src/backend_task/mod.rs +++ b/src/backend_task/mod.rs @@ -686,7 +686,8 @@ pub enum BackendTaskSuccessResult { retention: crate::model::backup_retention::BackupRetention, /// Expired backups deleted under the new policy. deleted: usize, - /// Why some expired backups could not be deleted; they are retried at the next start. + /// Why some expired backups could not be deleted; they are retried the next + /// time wallet data opens. cleanup_failure: Option>, }, PlatformInfo(PlatformInfoTaskResult), diff --git a/src/backend_task/system_task/mod.rs b/src/backend_task/system_task/mod.rs index e6b754e5f..208c6031f 100644 --- a/src/backend_task/system_task/mod.rs +++ b/src/backend_task/system_task/mod.rs @@ -67,7 +67,7 @@ impl AppContext { /// Backend-task handler for `SystemTask::UpdateBackupRetention`. /// /// Only the save can fail the task. Pruning under the new policy is - /// best-effort and is retried on every start. + /// best-effort and is retried the next time wallet data opens. pub fn handle_update_backup_retention( self: &Arc, retention: BackupRetention, @@ -75,8 +75,8 @@ impl AppContext { let retention = retention.sanitized(); self.set_backup_retention(retention) .map_err(|source| TaskError::AppSettingsWrite { source })?; - // Only the databases this process has already opened successfully are safe - // to prune; before the wallet backend is up, the next start applies it. + // Before wallet storage is up, its start-up applies the saved policy; the + // newest usable backup of every database is kept either way. let report = if self.wallet_backend().is_ok() { self.prune_expired_upgrade_backups_best_effort() } else { diff --git a/src/context/mod.rs b/src/context/mod.rs index 5cf75bec5..6e339e258 100644 --- a/src/context/mod.rs +++ b/src/context/mod.rs @@ -1160,8 +1160,8 @@ impl AppContext { .await?; self.wallet_backend.store(Some(Arc::new(backend))); drop(_build_guard); - // Both databases opened successfully, so an expired snapshot is no longer - // needed for recovery. + // Wallet storage is up: apply the retention policy to every database's + // backups (the newest usable backup of each is always kept). self.prune_expired_upgrade_backups_best_effort(); if let Err(error) = self.refresh_pending_dpns_usernames() { tracing::warn!( diff --git a/src/context/wallet_lifecycle/mod.rs b/src/context/wallet_lifecycle/mod.rs index e617fd339..420613934 100644 --- a/src/context/wallet_lifecycle/mod.rs +++ b/src/context/wallet_lifecycle/mod.rs @@ -24,10 +24,10 @@ use crate::model::wallet::{Wallet, WalletSeedHash}; use crate::utils::file_deletion::{DeletionIntent, delete_file, delete_tree}; use crate::wallet_backend::poison::RwLockRecover; use crate::wallet_backend::{ - ClearAllOutcome, DetScope, WalletBackend, WalletMetaView, WalletSeedView, network_prefix, + ClearAllOutcome, DetScope, WalletBackend, WalletMetaView, WalletSeedView, spv_storage_dir, }; use dash_sdk::dpp::dashcore::Network; -use std::path::{Path, PathBuf}; +use std::path::Path; use std::sync::atomic::Ordering; use std::sync::{Arc, RwLock}; @@ -67,13 +67,6 @@ pub enum WalletUnlockRetention { UntilAppClose, } -/// Per-network SPV storage directory: `/spv//`. Mirrors -/// `WalletBackend::resolve_spv_storage_dir` so the path resolves identically -/// whether or not the wallet backend is wired yet. -fn spv_storage_dir(data_dir: &Path, network: Network) -> PathBuf { - data_dir.join("spv").join(network_prefix(network)) -} - /// Remove the upstream chain-sync cache files under `spv_dir`, leaving the /// legacy shielded sidecars in that directory untouched (the durable wallet /// databases are not in it at all — see [`SPV_CHAIN_STORAGE_ENTRIES`]). The diff --git a/src/context/wallet_lifecycle/retention.rs b/src/context/wallet_lifecycle/retention.rs index 21bd0f51a..395dd164a 100644 --- a/src/context/wallet_lifecycle/retention.rs +++ b/src/context/wallet_lifecycle/retention.rs @@ -8,7 +8,8 @@ use std::time::SystemTime; pub struct BackupPruneReport { /// Expired backups deleted. Undercounts when a location fails part-way. pub deleted: usize, - /// The first failure; the remaining expired backups are retried at the next start. + /// The first failure; the remaining expired backups are retried the next time + /// wallet data opens. pub failure: Option, } @@ -230,11 +231,13 @@ mod tests { "the newest backup of each database is kept" ); assert!( - !crate::wallet_backend::wallet_database_path( - ctx.data_dir(), - dash_sdk::dpp::dashcore::Network::Devnet + !crate::wallet_backend::platform_compatibility::backup_lock_path( + &crate::wallet_backend::wallet_database_path( + ctx.data_dir(), + dash_sdk::dpp::dashcore::Network::Devnet + ) ) - .with_extension("sqlite.platform-upgrade.lock") + .unwrap() .exists(), "a database without backups is not locked or touched" ); @@ -263,10 +266,8 @@ mod tests { let ctx = test_app_context(tmp.path()); let fixture = backups(&ctx); let wallet = crate::wallet_backend::wallet_database_path(ctx.data_dir(), ctx.network); - let lock = wallet.with_file_name(format!( - "{}.platform-upgrade.lock", - wallet.file_name().unwrap().to_string_lossy() - )); + let lock = + crate::wallet_backend::platform_compatibility::backup_lock_path(&wallet).unwrap(); let held = std::fs::File::create(&lock).unwrap(); held.try_lock().unwrap(); diff --git a/src/context/wallet_lifecycle/tests/deletion_sentinel.rs b/src/context/wallet_lifecycle/tests/deletion_sentinel.rs index 228781bfa..32d2a39ab 100644 --- a/src/context/wallet_lifecycle/tests/deletion_sentinel.rs +++ b/src/context/wallet_lifecycle/tests/deletion_sentinel.rs @@ -5,6 +5,7 @@ use super::*; use dash_sdk::dpp::identity::accessors::IdentityGettersV0; use std::collections::{BTreeMap, BTreeSet}; +use std::path::PathBuf; use std::time::{Duration, SystemTime}; const DAY: Duration = Duration::from_secs(24 * 60 * 60); @@ -54,9 +55,14 @@ fn written_by_context(data_dir: &Path, path: &Path) -> bool { path.parent() == Some(data_dir) && live.iter().any(|base| { name == *base - || ["-wal", "-shm", "-journal", ".platform-upgrade.lock"] - .iter() - .any(|suffix| name == format!("{base}{suffix}")) + || [ + "-wal", + "-shm", + "-journal", + crate::wallet_backend::platform_compatibility::LOCK_SUFFIX, + ] + .iter() + .any(|suffix| name == format!("{base}{suffix}")) }) } @@ -130,8 +136,10 @@ async fn deletion_paths_leave_every_untargeted_file_intact() { "det-testnet.sqlite.platform-67d4ef3-backup-old.sqlite", "backups/auto/pre-migration-det-testnet-1-to-2-20000101T000000Z.db", "backups/data_backup_20000101_000000.db", - // Network clear: every remaining testnet upgrade backup. + // Network clear: every remaining testnet upgrade backup, including the + // expired one retention keeps as the latest migration's snapshot. "det-testnet.sqlite.platform-67d4ef3-backup-new.sqlite", + "backups/auto/pre-migration-det-testnet-2-to-3-20000102T000000Z.db", // Network clear (SPV cache + retired shielded files): testnet only. "spv/testnet/blocks/segment.dat", "spv/testnet/filters/nested/segment.dat", diff --git a/src/database/legacy_backups.rs b/src/database/legacy_backups.rs index c11a3d291..1c4bafc11 100644 --- a/src/database/legacy_backups.rs +++ b/src/database/legacy_backups.rs @@ -29,7 +29,7 @@ pub(crate) fn backup_timestamp(name: &str) -> Option { } /// Delete legacy backups in `/backups` older than `max_age`, always -/// keeping the newest one. +/// keeping the newest usable one. /// /// Deletion goes through the [`delete_file`](crate::utils::file_deletion::delete_file) /// chokepoint, which refuses live databases, aliases and hard links. @@ -66,7 +66,11 @@ pub(crate) fn prune_expired( other => other, }; let created = metadata.and_then(|metadata| Ok(named.max(metadata.modified()?))); - candidates.push((entry.path(), created)); + candidates.push(crate::utils::backup_prune::Candidate { + path: entry.path(), + created, + sequence: None, + }); } crate::utils::backup_prune::prune_expired( candidates, @@ -234,6 +238,30 @@ mod tests { assert!(newest.exists()); } + /// An interrupted legacy copy is never the floor: the newest complete backup + /// survives, and the expired interrupted copy is deleted. + #[test] + fn newest_legacy_backup_floor_must_be_a_complete_database() { + let dir = tempfile::tempdir().unwrap(); + let backups = dir.path().join(BACKUP_DIR); + std::fs::create_dir_all(&backups).unwrap(); + let complete = backups.join("data_backup_20200101_000000.db"); + rusqlite::Connection::open(&complete) + .unwrap() + .execute_batch("CREATE TABLE settings (v INTEGER); INSERT INTO settings VALUES (1);") + .unwrap(); + let bytes = std::fs::read(&complete).unwrap(); + let truncated = backups.join("data_backup_20200601_000000.db"); + std::fs::write(&truncated, &bytes[..bytes.len() / 2]).unwrap(); + let now = SystemTime::now(); + set_mtime(&complete, now - DAY * 200); + set_mtime(&truncated, now - DAY * 100); + + assert_eq!(prune_expired(dir.path(), DAY * 90, now).unwrap(), 1); + assert!(complete.exists(), "the only complete backup is the floor"); + assert!(!truncated.exists()); + } + #[cfg(unix)] #[test] fn legacy_backup_symlinks_are_never_followed() { diff --git a/src/logging.rs b/src/logging.rs index 83ed14365..68ec0255b 100644 --- a/src/logging.rs +++ b/src/logging.rs @@ -546,8 +546,6 @@ fn rotated_name(stem: &str, ts: i64) -> String { format!("{stem}.{ts:010}.log") } -/// Parses the timestamp out of a rotated log file name produced by -/// [`rotated_name`], returning `None` for names that don't match the stem. /// The timestamp in a rotated log name `..log`, or `None` for any /// other name. Also the deletion chokepoint's definition of a rotated log. pub(crate) fn parse_rotated_ts(name: &str, stem: &str) -> Option { diff --git a/src/ui/network_chooser_screen.rs b/src/ui/network_chooser_screen.rs index 2a4b053ff..6438db0f7 100644 --- a/src/ui/network_chooser_screen.rs +++ b/src/ui/network_chooser_screen.rs @@ -1,7 +1,7 @@ use crate::app::AppAction; use crate::backend_task::core::CoreTask; use crate::backend_task::dapi_discovery::persist_dapi_addresses; -use crate::backend_task::error::TaskError; +use crate::backend_task::error::{BACKUP_RETENTION_SAVED_CLEANUP_INCOMPLETE, TaskError}; use crate::backend_task::system_task::SystemTask; use crate::backend_task::{BackendTask, BackendTaskContext, BackendTaskSuccessResult}; use crate::context::AppContext; @@ -1382,18 +1382,14 @@ impl NetworkChooserScreen { /// Shown when the retention setting cannot be read; the backend then deletes nothing. const BACKUP_RETENTION_UNREADABLE: &str = "Your backup retention setting could not be read, so no old upgrade backups are deleted. Choose a backup retention setting and save it."; -/// Shown when the new policy was saved but some expired backups could not be deleted. -const BACKUP_RETENTION_CLEANUP_INCOMPLETE: &str = "Your backup retention setting was saved, but some old upgrade backups could not be deleted. The app will try again the next time it starts."; - /// Result banner after saving a retention policy. fn backup_retention_saved_banner( deleted: usize, cleanup_incomplete: bool, ) -> (String, MessageType) { - // The wording matches `TaskError::UpgradeBackupCleanup`, prefixed by the save outcome. if cleanup_incomplete { return ( - BACKUP_RETENTION_CLEANUP_INCOMPLETE.to_owned(), + BACKUP_RETENTION_SAVED_CLEANUP_INCOMPLETE.to_owned(), MessageType::Warning, ); } @@ -2056,7 +2052,7 @@ mod tests { assert_eq!( backup_retention_saved_banner(3, true), ( - BACKUP_RETENTION_CLEANUP_INCOMPLETE.to_owned(), + BACKUP_RETENTION_SAVED_CLEANUP_INCOMPLETE.to_owned(), MessageType::Warning ) ); diff --git a/src/utils/backup_prune.rs b/src/utils/backup_prune.rs index d99d396f9..ae66735c5 100644 --- a/src/utils/backup_prune.rs +++ b/src/utils/backup_prune.rs @@ -1,20 +1,53 @@ //! Time-based expiry shared by every upgrade-backup location. //! //! One algorithm for the bridge, upstream `pre-migration-*` and legacy `data.db` -//! backups: delete those older than the retention period, except the newest backup -//! of the database, which is kept whatever its age. A wrong-forward clock can -//! therefore never delete the last recovery copy. +//! backups: delete those older than the retention period, except the newest usable +//! backup of the database, which is kept whatever its age. Neither a wrong clock nor +//! an interrupted copy can therefore delete the last recovery copy. +//! +//! "Newest" is judged by every ordering signal a backup carries, and the newest by +//! each signal is kept: +//! - its creation time, preferring backups not dated in the future, so a copy taken +//! while the clock ran ahead cannot displace a genuinely newer one; +//! - its position in the database's upgrade history (upstream snapshots name the +//! migration they precede), which no clock can skew. +//! +//! "Usable" is a cheap structural check ([`usable_snapshot`]). Empty, truncated, +//! header-only and torn files fail it. When no backup of a database passes, the +//! newest files are kept anyway, so a false negative of the check can never delete +//! every backup. use super::file_deletion::{DeletionIntent, delete_file}; use std::collections::BTreeSet; use std::path::{Path, PathBuf}; use std::time::{Duration, SystemTime}; -/// A published backup of one database, with its creation time or the error that -/// prevented reading it. -pub(crate) type Candidate = (PathBuf, std::io::Result); +/// How far ahead of `now` a creation time may be and still count as not in the +/// future when choosing the newest backup. Absorbs ordinary clock drift. +const FUTURE_TOLERANCE: Duration = Duration::from_secs(24 * 60 * 60); + +/// A published backup of one database. +#[derive(Debug)] +pub(crate) struct Candidate { + /// The backup file. + pub(crate) path: PathBuf, + /// When the backup was created, or the error that prevented reading it. + pub(crate) created: std::io::Result, + /// Clock-independent position in the database's upgrade history; a larger value + /// is a later snapshot. `None` when the name carries no such signal. + pub(crate) sequence: Option<(u32, u32)>, +} + +/// A candidate whose creation time is known. +struct Dated { + created: SystemTime, + path: PathBuf, + sequence: Option<(u32, u32)>, + usable: bool, +} -/// Delete every candidate older than `max_age` at `now`, except the newest one. +/// Delete every candidate older than `max_age` at `now`, except the newest usable one +/// by each ordering signal (see the module docs). /// /// All candidates must be backups of the database named by `intent`. A candidate /// whose creation time cannot be read is kept and reported. A candidate that is @@ -29,28 +62,33 @@ pub(crate) fn prune_expired( ) -> std::io::Result { let mut first_error = None; let mut dated = Vec::with_capacity(candidates.len()); - for (path, created) in candidates { - match created { - Ok(created) => dated.push((created, path)), + for candidate in candidates { + match candidate.created { + Ok(created) => dated.push(Dated { + created, + usable: usable_snapshot(&candidate.path), + path: candidate.path, + sequence: candidate.sequence, + }), Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} Err(error) => { first_error.get_or_insert(error); } } } - dated.sort(); - // The newest backup is the floor: it survives any age and any clock. - dated.pop(); + let floors = floors(&dated, now); let mut removed = 0; let mut touched = BTreeSet::new(); - for (created, path) in dated { - if !crate::model::backup_retention::backup_expired(created, now, max_age) { + for (index, backup) in dated.iter().enumerate() { + if floors.contains(&index) + || !crate::model::backup_retention::backup_expired(backup.created, now, max_age) + { continue; } - match delete_file(&path, intent) { + match delete_file(&backup.path, intent) { Ok(()) => { removed += 1; - if let Some(directory) = path.parent() { + if let Some(directory) = backup.path.parent() { touched.insert(directory.to_owned()); } } @@ -68,6 +106,106 @@ pub(crate) fn prune_expired( first_error.map_or(Ok(removed), Err) } +/// Indices of the backups retention keeps whatever their age: the newest usable one +/// by creation time (preferring those not dated in the future) and the newest usable +/// one by upgrade-history position. Ties break on the path, so every pass and every +/// concurrent context picks the same files. With no usable backup at all, the same +/// choice is made among every backup instead. +fn floors(dated: &[Dated], now: SystemTime) -> BTreeSet { + let any_usable = dated.iter().any(|backup| backup.usable); + let eligible = || { + dated + .iter() + .enumerate() + .filter(move |(_, backup)| backup.usable || !any_usable) + }; + let latest_trusted = now.checked_add(FUTURE_TOLERANCE); + let by_time = eligible() + .max_by_key(|(_, backup)| { + let trusted = latest_trusted.is_none_or(|latest| backup.created <= latest); + (trusted, backup.created, &backup.path) + }) + .map(|(index, _)| index); + let by_sequence = eligible() + .filter_map(|(index, backup)| Some((index, backup.sequence?, backup))) + .max_by_key(|(_, sequence, backup)| (*sequence, backup.created, &backup.path)) + .map(|(index, _, _)| index); + by_time.into_iter().chain(by_sequence).collect() +} + +/// Length of the SQLite database header. +const SQLITE_HEADER_LEN: usize = 100; +/// Magic string every SQLite 3 database file starts with. +const SQLITE_MAGIC: &[u8] = b"SQLite format 3\0"; + +/// Whether `backup` looks like a complete SQLite database that can be restored. +/// +/// A structural check that reads only the 100-byte header and never opens the file +/// through SQLite, which could create `-wal`/`-shm` sidecars or roll back a journal. +/// It requires the magic string, a valid page size and file-format version, a file +/// size covering every page the header claims, and no non-empty `-journal` beside +/// the file: a hot rollback journal means the file holds a torn, uncommitted write. +/// Any read error counts as unusable. +pub(crate) fn usable_snapshot(backup: &Path) -> bool { + match check_snapshot(backup) { + Ok(usable) => usable, + Err(error) => { + tracing::debug!(backup = %backup.display(), ?error, "Upgrade backup could not be checked"); + false + } + } +} + +fn check_snapshot(backup: &Path) -> std::io::Result { + use std::io::Read; + let mut file = std::fs::File::open(backup)?; + let length = file.metadata()?.len(); + if length < SQLITE_HEADER_LEN as u64 { + return Ok(false); + } + let mut header = [0u8; SQLITE_HEADER_LEN]; + file.read_exact(&mut header)?; + if !header.starts_with(SQLITE_MAGIC) { + return Ok(false); + } + let be_u32 = |offset: usize| { + u32::from_be_bytes([ + header[offset], + header[offset + 1], + header[offset + 2], + header[offset + 3], + ]) + }; + let page_size = match u16::from_be_bytes([header[16], header[17]]) { + 1 => 65_536, + size if (512..=32_768).contains(&size) && size.is_power_of_two() => u64::from(size), + _ => return Ok(false), + }; + // File-format write/read versions: 1 = rollback journal, 2 = WAL. + if !matches!(header[18], 1 | 2) || !matches!(header[19], 1 | 2) { + return Ok(false); + } + // The in-header page count is valid only when "version-valid-for" matches the + // change counter, as every SQLite since 3.7.0 writes it; otherwise require + // whole pages. + let (change_counter, pages, valid_for) = (be_u32(24), be_u32(28), be_u32(92)); + let complete = if pages > 0 && change_counter == valid_for { + length >= u64::from(pages) * page_size + } else { + length >= page_size && length % page_size == 0 + }; + if !complete { + return Ok(false); + } + let mut journal = backup.as_os_str().to_owned(); + journal.push("-journal"); + match std::fs::symlink_metadata(journal) { + Ok(metadata) => Ok(metadata.len() == 0), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(true), + Err(error) => Err(error), + } +} + /// Persist directory-entry changes (unlinks) in `directory`. /// /// Windows cannot open a directory as a `File` without extra flags and NTFS @@ -79,3 +217,140 @@ pub(crate) fn sync_directory(directory: &Path) -> std::io::Result<()> { let _ = directory; Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + const DAY: Duration = Duration::from_secs(24 * 60 * 60); + + /// A small, complete SQLite database at `path`. + fn write_database(path: &Path) { + rusqlite::Connection::open(path) + .unwrap() + .execute_batch("CREATE TABLE t (v INTEGER); INSERT INTO t VALUES (1);") + .unwrap(); + } + + fn dated(path: &Path, created: SystemTime, sequence: Option<(u32, u32)>) -> Dated { + Dated { + created, + path: path.to_owned(), + sequence, + usable: usable_snapshot(path), + } + } + + #[test] + fn complete_databases_are_usable() { + let dir = tempfile::tempdir().unwrap(); + let rollback = dir.path().join("rollback.db"); + write_database(&rollback); + assert!(usable_snapshot(&rollback), "rollback-journal database"); + + let wal = dir.path().join("wal.db"); + let conn = rusqlite::Connection::open(&wal).unwrap(); + conn.pragma_update(None, "journal_mode", "WAL").unwrap(); + conn.execute_batch("CREATE TABLE t (v INTEGER); INSERT INTO t VALUES (1);") + .unwrap(); + drop(conn); + assert!(usable_snapshot(&wal), "WAL database"); + } + + #[test] + fn interrupted_copies_are_not_usable() { + let dir = tempfile::tempdir().unwrap(); + let complete = dir.path().join("complete.db"); + write_database(&complete); + let bytes = std::fs::read(&complete).unwrap(); + let zeroed = vec![0u8; bytes.len()]; + let cases: [(&str, &[u8]); 6] = [ + ("empty", b""), + ("header only", &bytes[..SQLITE_HEADER_LEN]), + ("one byte short", &bytes[..bytes.len() - 1]), + ("half", &bytes[..bytes.len() / 2]), + ("zero-filled", &zeroed), + ("not sqlite", b"candy wrappers, not a database"), + ]; + for (case, content) in cases { + let path = dir.path().join(format!("{case}.db")); + std::fs::write(&path, content).unwrap(); + assert!(!usable_snapshot(&path), "{case}"); + } + assert!(!usable_snapshot(&dir.path().join("missing.db")), "missing"); + } + + #[test] + fn hot_journal_makes_a_copy_unusable() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("torn.db"); + write_database(&path); + let journal = dir.path().join("torn.db-journal"); + std::fs::write(&journal, b"").unwrap(); + assert!(usable_snapshot(&path), "an empty journal is not hot"); + std::fs::write(&journal, b"uncommitted pages").unwrap(); + assert!(!usable_snapshot(&path)); + } + + /// A newer unusable file never becomes the floor; the newest usable one does. + #[test] + fn floor_skips_unusable_newer_files() { + let dir = tempfile::tempdir().unwrap(); + let now = SystemTime::now(); + let valid = dir.path().join("valid.db"); + write_database(&valid); + let killed = dir.path().join("killed.db"); + std::fs::write(&killed, b"").unwrap(); + let backups = [ + dated(&valid, now - DAY * 200, None), + dated(&killed, now, Some((9, 8))), + ]; + assert_eq!(floors(&backups, now), BTreeSet::from([0])); + } + + /// Without any usable backup the newest file is still kept. + #[test] + fn floor_falls_back_to_newest_file_when_none_is_usable() { + let dir = tempfile::tempdir().unwrap(); + let now = SystemTime::now(); + let [older, newer] = ["older.db", "newer.db"].map(|name| dir.path().join(name)); + for path in [&older, &newer] { + std::fs::write(path, b"junk").unwrap(); + } + let backups = [ + dated(&older, now - DAY * 20, None), + dated(&newer, now - DAY * 10, None), + ]; + assert_eq!(floors(&backups, now), BTreeSet::from([1])); + } + + /// A backup dated in the future does not displace the newest backup dated in the + /// past, and the latest migration's snapshot is kept whatever its date. + #[test] + fn floor_resists_clock_skew() { + let dir = tempfile::tempdir().unwrap(); + let now = SystemTime::now(); + let paths = ["a.db", "b.db", "c.db"].map(|name| dir.path().join(name)); + for path in &paths { + write_database(path); + } + let backups = [ + dated(&paths[0], now + DAY * 30, None), + dated(&paths[1], now - DAY * 100, Some((2, 1))), + dated(&paths[2], now - DAY * 400, Some((3, 2))), + ]; + assert_eq!(floors(&backups, now), BTreeSet::from([1, 2])); + // Ordinary drift is not the future. + let backups = [ + dated(&paths[0], now + Duration::from_secs(60), None), + dated(&paths[1], now - DAY, None), + ]; + assert_eq!(floors(&backups, now), BTreeSet::from([0])); + // Only future-dated backups: the newest of them. + let backups = [ + dated(&paths[0], now + DAY * 30, None), + dated(&paths[1], now + DAY * 20, None), + ]; + assert_eq!(floors(&backups, now), BTreeSet::from([0])); + } +} diff --git a/src/utils/file_deletion.rs b/src/utils/file_deletion.rs index d46340b16..87bd22595 100644 --- a/src/utils/file_deletion.rs +++ b/src/utils/file_deletion.rs @@ -163,7 +163,7 @@ fn check_dir_scope(dir: &Path, intent: DeletionIntent<'_>) -> std::io::Result<() let located = canonical_location(dir)?; let allowed = match intent { DeletionIntent::NetworkClear { data_dir, network } => { - let root = canonical_dir(&network_spv_dir(data_dir, network)); + let root = canonical_dir(&crate::wallet_backend::spv_storage_dir(data_dir, network)); located.starts_with(&root) && located != root } DeletionIntent::Backup { .. } | DeletionIntent::Log { .. } => false, @@ -207,12 +207,6 @@ fn unlink(path: &Path, metadata: &std::fs::Metadata) -> std::io::Result<()> { std::fs::remove_file(path) } -fn network_spv_dir(data_dir: &Path, network: Network) -> PathBuf { - data_dir - .join("spv") - .join(crate::wallet_backend::network_prefix(network)) -} - /// Every check before an unlink; returns the target's own (unfollowed) metadata. fn check(path: &Path, intent: DeletionIntent<'_>) -> std::io::Result { let refused = @@ -390,8 +384,9 @@ fn in_scope(located: &Path, name: &str, intent: DeletionIntent<'_>) -> bool { DeletionIntent::NetworkClear { data_dir, network } => { let spv = data_dir.join("spv"); let lock = format!("{}.lock", crate::wallet_backend::network_prefix(network)); - parent.starts_with(canonical_dir(&network_spv_dir(data_dir, network))) - || (is(&spv) && name == lock) + parent.starts_with(canonical_dir(&crate::wallet_backend::spv_storage_dir( + data_dir, network, + ))) || (is(&spv) && name == lock) } DeletionIntent::Log { dir, stem } => { is(dir) && crate::logging::parse_rotated_ts(name, stem).is_some() diff --git a/src/utils/file_deletion/tests.rs b/src/utils/file_deletion/tests.rs index 820b05a05..3f64de561 100644 --- a/src/utils/file_deletion/tests.rs +++ b/src/utils/file_deletion/tests.rs @@ -263,7 +263,7 @@ fn delete_file_refuses_targets_outside_the_intent_scope() { }, ), ( - dir.path().join("det-app.sqlite.platform-upgrade.lock"), + crate::wallet_backend::platform_compatibility::backup_lock_path(&database).unwrap(), DeletionIntent::Backup { database: &database, }, diff --git a/src/wallet_backend/mod.rs b/src/wallet_backend/mod.rs index 6dde5805e..e87487b07 100644 --- a/src/wallet_backend/mod.rs +++ b/src/wallet_backend/mod.rs @@ -561,6 +561,18 @@ pub(crate) fn wallet_database_path(data_dir: &Path, network: Network) -> std::pa data_dir.join(format!("det-{}.sqlite", network_prefix(network))) } +/// Parent of every network's [`spv_storage_dir`]. +const SPV_ROOT_DIR: &str = "spv"; + +/// The per-network chain-sync (SPV) cache directory, `/spv//`. +/// +/// The single definition shared by the backend (which creates it), the network +/// clear (which empties it) and the deletion chokepoint (which confines that clear +/// to it). +pub(crate) fn spv_storage_dir(data_dir: &Path, network: Network) -> std::path::PathBuf { + data_dir.join(SPV_ROOT_DIR).join(network_prefix(network)) +} + /// The upstream shielded coordinator's store, /// `/det--shielded.sqlite` — the sibling of /// [`wallet_database_path`] holding all Orchard state. @@ -3067,12 +3079,12 @@ impl WalletBackend { app_data_dir: &Path, network: Network, ) -> Result { - let mut dir = app_data_dir.join("spv"); - crate::app_dir::ensure_data_dir_exists(&dir) - .map_err(|source| TaskError::FileSystem { source })?; - dir.push(kv::network_prefix(network)); - crate::app_dir::ensure_data_dir_exists(&dir) - .map_err(|source| TaskError::FileSystem { source })?; + let dir = spv_storage_dir(app_data_dir, network); + // Create (and lock down) the shared `spv` root before the network directory. + for directory in [app_data_dir.join(SPV_ROOT_DIR), dir.clone()] { + crate::app_dir::ensure_data_dir_exists(&directory) + .map_err(|source| TaskError::FileSystem { source })?; + } Ok(dir) } } diff --git a/src/wallet_backend/platform_compatibility/engine.rs b/src/wallet_backend/platform_compatibility/engine.rs index 9dee94e04..476139270 100644 --- a/src/wallet_backend/platform_compatibility/engine.rs +++ b/src/wallet_backend/platform_compatibility/engine.rs @@ -378,29 +378,40 @@ fn scan_backups_in(path: &Path, auto_dir: Option<&Path>) -> std::io::Result std::io::Result<()> { + let unexpected = |kind| Err(std::io::Error::other(kind)); let metadata = std::fs::symlink_metadata(backup)?; - if !metadata.is_file() || backup == database { - return Err(std::io::Error::other( - "Refusing to remove a non-regular backup file", - )); + if !metadata.is_file() { + return unexpected(UnexpectedBackupFile::NotRegularFile); + } + if backup == database { + return unexpected(UnexpectedBackupFile::LiveDatabase); } if let Ok(live_path) = database.canonicalize() && backup.canonicalize()? == live_path { - return Err(std::io::Error::other( - "Refusing to remove the live database", - )); + return unexpected(UnexpectedBackupFile::LiveDatabase); } #[cfg(unix)] { use std::os::unix::fs::MetadataExt; if metadata.nlink() != 1 { - return Err(std::io::Error::other( - "Refusing to remove a hard-linked backup file", - )); + return unexpected(UnexpectedBackupFile::HardLinked); } } Ok(()) @@ -439,13 +450,21 @@ fn lock_with_grace(file: &std::fs::File) -> std::io::Result<()> { } } +/// Suffix appended to a database's file name to form its lifecycle lock file. +pub(crate) const LOCK_SUFFIX: &str = ".platform-upgrade.lock"; + +/// The lifecycle lock file of the database at `database`; `None` when the path has +/// no file name. +pub(crate) fn backup_lock_path(database: &Path) -> Option { + let mut name = database.file_name()?.to_os_string(); + name.push(LOCK_SUFFIX); + Some(database.with_file_name(name)) +} + pub(super) fn backup_lock(path: &Path) -> std::io::Result { - let Some(name) = path.file_name() else { + let Some(lock_path) = backup_lock_path(path) else { return Err(std::io::ErrorKind::InvalidInput.into()); }; - let mut lock_name = name.to_os_string(); - lock_name.push(".platform-upgrade.lock"); - let lock_path = path.with_file_name(lock_name); let mut options = std::fs::File::options(); options.read(true).write(true).create_new(true); #[cfg(unix)] @@ -577,8 +596,10 @@ pub(crate) fn remove_backups(path: &Path) -> std::io::Result<()> { /// Delete the upgrade backups of the database at `path` that are older than `max_age`. /// /// Covers both the bridge snapshots next to the database and upstream `pre-migration-*` -/// snapshots, and always keeps the newest one, so it is safe whether or not the database -/// currently opens. A database with no backups is skipped without taking its lock, and +/// snapshots, and always keeps the newest usable one (see +/// [`prune_expired`](crate::utils::backup_prune::prune_expired)), so it is safe +/// whether or not the database currently opens. A database with no backups is +/// skipped without taking its lock, and /// one whose lock is held (another context is opening or upgrading it) is skipped too; /// the next pass covers it. /// @@ -615,7 +636,17 @@ pub(crate) fn prune_expired_backups( }) .map(|backup| { let created = backup_created(path, &backup); - (backup, created) + // A later migration is a later snapshot, whatever the clock said. + let sequence = backup + .file_name() + .and_then(|name| name.to_str()) + .and_then(|name| parse_upstream_name(path, name)) + .map(|snapshot| (snapshot.to, snapshot.from)); + crate::utils::backup_prune::Candidate { + path: backup, + created, + sequence, + } }) .collect(); let pruned = crate::utils::backup_prune::prune_expired( diff --git a/src/wallet_backend/platform_compatibility/engine/tests.rs b/src/wallet_backend/platform_compatibility/engine/tests.rs index 3354733b5..a3c80ba21 100644 --- a/src/wallet_backend/platform_compatibility/engine/tests.rs +++ b/src/wallet_backend/platform_compatibility/engine/tests.rs @@ -287,7 +287,7 @@ fn platform_compatibility_backup_removal_only_touches_the_named_database() { "det-mainnet.sqlite.platform-67d4ef3-backup-c3.sqlite", "det-testnet-shielded.sqlite", "det-testnet.sqlite", - "det-testnet.sqlite.platform-upgrade.lock", + format!("det-testnet.sqlite{LOCK_SUFFIX}").as_str(), ] ); remove_backups(&dir.path().join("absent.sqlite")).unwrap(); @@ -671,6 +671,10 @@ fn platform_compatibility_wal_snapshot_includes_committed_uncheckpointed_data() live.execute_batch("PRAGMA journal_mode=WAL; PRAGMA wal_autocheckpoint=0; INSERT INTO meta_global VALUES ('wal-fixture', X'00ff', 42)").unwrap(); let before = snapshot(&path); let backup = upgrade(&path, &target, |_| Ok(())).unwrap().unwrap(); + assert!( + crate::utils::backup_prune::usable_snapshot(&backup), + "a snapshot of a WAL database must count as usable" + ); assert_eq!(snapshot(&backup), before); let conn = Connection::open(&path).unwrap(); assert_eq!( @@ -1017,6 +1021,115 @@ fn platform_compatibility_prune_keeps_newest_snapshot_for_any_clock() { drop(dir); } +/// An interrupted copy left under a final snapshot name (empty, truncated or +/// header-only) never becomes the floor: the newest complete snapshot survives any +/// age, and the expired interrupted copy is deleted. +#[test] +fn platform_compatibility_prune_floor_skips_interrupted_newest_copy() { + const DAY_SECONDS: i64 = 24 * 60 * 60; + for (incomplete_age, expected_removed) in [(0, 0), (-100 * DAY_SECONDS, 1)] { + for case in [ + "empty bridge", + "truncated bridge", + "header-only bridge", + "truncated upstream", + "empty upstream", + ] { + let (dir, path, _target) = fixture(); + let valid = backup(&path).unwrap(); + let full = std::fs::read(&valid).unwrap(); + set_mtime(&valid, -200 * DAY_SECONDS); + let auto = dir.path().join("backups/auto"); + std::fs::create_dir_all(&auto).unwrap(); + let (incomplete, bytes): (PathBuf, &[u8]) = match case { + "empty bridge" => ( + dir.path() + .join("wallet.sqlite.platform-67d4ef3-backup-killed.sqlite"), + b"", + ), + "truncated bridge" => ( + dir.path() + .join("wallet.sqlite.platform-67d4ef3-backup-killed.sqlite"), + &full[..full.len() / 2], + ), + "header-only bridge" => ( + dir.path() + .join("wallet.sqlite.platform-67d4ef3-backup-killed.sqlite"), + &full[..100], + ), + "truncated upstream" => ( + auto.join("pre-migration-wallet-1-to-2-20200101T000000Z.db"), + &full[..full.len() - 1], + ), + _ => ( + auto.join("pre-migration-wallet-1-to-2-20200101T000000Z.db"), + b"", + ), + }; + std::fs::write(&incomplete, bytes).unwrap(); + set_mtime(&incomplete, incomplete_age); + + let removed = + prune_expired_backups(&path, RETENTION_DAY * 90, std::time::SystemTime::now()) + .unwrap(); + + assert!( + valid.exists(), + "{case}: the only complete snapshot must be kept" + ); + assert_eq!(removed, expected_removed, "{case}"); + assert_eq!(incomplete.exists(), expected_removed == 0, "{case}"); + } + } +} + +/// A snapshot written while the clock ran ahead is dated in the future; it must not +/// displace the genuinely newer snapshot taken after the clock was corrected. +#[test] +fn platform_compatibility_prune_floor_resists_forward_clock_skew() { + let (_dir, path, _target) = fixture(); + let skewed = backup(&path).unwrap(); + let current = backup(&path).unwrap(); + set_mtime(&skewed, 30 * 24 * 60 * 60); + set_mtime(¤t, -100 * 24 * 60 * 60); + + assert_eq!( + prune_expired_backups(&path, RETENTION_DAY * 90, std::time::SystemTime::now()).unwrap(), + 0 + ); + assert!( + current.exists(), + "the newest snapshot dated in the past is kept" + ); + assert!(skewed.exists(), "a future-dated snapshot is never expired"); +} + +/// Upstream snapshots name the migration they precede, a clock-independent order: +/// the snapshot of the latest migration survives even when an older migration's +/// snapshot carries later dates. +#[test] +fn platform_compatibility_prune_floor_keeps_latest_migration_snapshot() { + let (dir, path, _target) = fixture(); + let valid = backup(&path).unwrap(); + let auto = dir.path().join("backups/auto"); + std::fs::create_dir_all(&auto).unwrap(); + let latest_migration = auto.join("pre-migration-wallet-2-to-3-20200101T000000Z.db"); + let earlier_migration = auto.join("pre-migration-wallet-1-to-2-20200601T000000Z.db"); + std::fs::copy(&valid, &latest_migration).unwrap(); + std::fs::copy(&valid, &earlier_migration).unwrap(); + set_mtime(&latest_migration, -400 * 24 * 60 * 60); + set_mtime(&earlier_migration, -300 * 24 * 60 * 60); + set_mtime(&valid, -500 * 24 * 60 * 60); + + assert_eq!( + prune_expired_backups(&path, RETENTION_DAY * 90, std::time::SystemTime::now()).unwrap(), + 1 + ); + assert!(!valid.exists()); + assert!(latest_migration.exists(), "newest by migration order"); + assert!(earlier_migration.exists(), "newest by creation time"); +} + /// A rejected candidate is reported, but expired valid snapshots are still removed and /// the rejected entry is left alone. #[test] diff --git a/src/wallet_backend/platform_compatibility/mod.rs b/src/wallet_backend/platform_compatibility/mod.rs index 8147cbc6b..ad6135a47 100644 --- a/src/wallet_backend/platform_compatibility/mod.rs +++ b/src/wallet_backend/platform_compatibility/mod.rs @@ -3,6 +3,8 @@ mod engine; mod storage_failure; pub use engine::UpgradeError; +#[cfg(test)] +pub(crate) use engine::{LOCK_SUFFIX, backup_lock_path}; pub(crate) use engine::{ is_bridge_backup_name, prune_expired_backups, remove_backups, upstream_backup_timestamp, }; @@ -153,11 +155,17 @@ mod tests { use super::*; use dash_sdk::dpp::identity::accessors::IdentityGettersV0; - /// Repeated failed opens can leave identical copies of the unchanged database; - /// the open's own tidying collapses them to the newest, even when the open fails. + /// Every failed open of an unsupported database makes upstream write another + /// `pre-migration-*` copy of the unchanged database; the open's own tidying + /// collapses them to the newest, even when the open fails. + /// + /// Upstream refuses a group- or world-writable data or backup directory before + /// taking any snapshot, so the data directory is made owner-only and the backup + /// directory is left for upstream to create; neither may depend on the umask. #[test] fn platform_compatibility_bounds_upstream_backups_after_failed_opens() { let dir = tempfile::tempdir().unwrap(); + crate::app_dir::ensure_data_dir_exists(dir.path()).unwrap(); let path = dir.path().join("wallet.sqlite"); rusqlite::Connection::open(&path) .unwrap() @@ -168,19 +176,36 @@ mod tests { .execute_batch("ALTER TABLE wallets ADD COLUMN unsupported INTEGER;") .unwrap(); let auto = platform_wallet_storage::default_auto_backup_dir(&path); - std::fs::create_dir_all(&auto).unwrap(); - let copy = std::fs::read(&path).unwrap(); + let mut previous: Option = None; for attempt in 1..=3 { - let snapshot = auto.join(format!( - "pre-migration-wallet-1-to-2-20260915T12000{attempt}Z.db" - )); - std::fs::write(&snapshot, ©).unwrap(); assert!(open(SqlitePersisterConfig::new(&path)).is_err()); - let retained: Vec<_> = std::fs::read_dir(&auto) - .unwrap() - .map(|entry| entry.unwrap().path()) - .collect(); - assert_eq!(retained, vec![snapshot], "attempt {attempt}"); + let retained = dir_entries(&auto); + assert_eq!( + retained.len(), + 1, + "attempt {attempt}: one snapshot must remain, got {retained:?}" + ); + let snapshot = retained[0].clone(); + assert!( + snapshot.starts_with("pre-migration-wallet-") && snapshot.ends_with(".db"), + "attempt {attempt}: unexpected entry {snapshot}" + ); + assert!( + crate::utils::backup_prune::usable_snapshot(&auto.join(&snapshot)), + "attempt {attempt}: a real upstream snapshot must count as usable" + ); + if let Some(previous) = &previous { + assert_ne!( + &snapshot, previous, + "attempt {attempt}: the newest copy must be the one kept" + ); + } + // Age the kept copy's name so the next open's copy gets a distinct, + // newer name even within the same second. + let (head, _) = snapshot.rsplit_once('-').unwrap(); + let aged = format!("{head}-2020010{attempt}T000000Z.db"); + std::fs::rename(auto.join(&snapshot), auto.join(&aged)).unwrap(); + previous = Some(aged); } } @@ -297,7 +322,7 @@ mod tests { dir_entries(dir.path()), vec![ "wallet.sqlite".to_owned(), - "wallet.sqlite.platform-upgrade.lock".to_owned() + format!("wallet.sqlite{}", engine::LOCK_SUFFIX) ] ); } @@ -350,7 +375,7 @@ mod tests { "the original database must stay untouched" ); let mut entries_after = dir_entries(dir.path()); - entries_after.retain(|name| name != "wallet.sqlite.platform-upgrade.lock"); + entries_after.retain(|name| *name != format!("wallet.sqlite{}", engine::LOCK_SUFFIX)); assert_eq!( entries_after, entries_before, "a refused probe must leave no files or directories behind" From ea999c463bac54401f0de90dc3dfb029b9360616 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Wed, 30 Sep 2026 10:56:05 +0000 Subject: [PATCH 24/30] fix(storage): never prune a backup whose usability is unproven The retention check treated an unreadable backup as unusable, so a read error on the newest snapshot handed the floor to an older copy and let the newest one expire. Usability is now three-way: usable, unusable, or undetermined. Undetermined covers a file that cannot be read (also reported, so the pass retries) and a complete-looking file with a non-empty rollback journal beside it. Undetermined backups are never deleted automatically and never displace the newest usable one. Legacy data.db backups are now written to a pending file, synced and renamed without overwriting, so a crash cannot leave a torn copy under a backup name. The clock-skew limit is documented and pinned by a test: a future-dated backup stops being de-preferred once real time reaches its date. WAL-034, the CHANGELOG and the upgrade notes now mention the no-usable-backup fallback. Also: misplaced doc comment and test in error.rs, and the SPV root directory now comes from one helper. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 +- .../2026-09-10-platform-pin/upgrade-notes.md | 12 +- docs/user-stories.md | 2 +- src/backend_task/error.rs | 26 +-- src/database/initialization.rs | 19 +- src/database/legacy_backups.rs | 66 ++++++- src/utils/backup_prune.rs | 175 ++++++++++++------ src/utils/file_deletion.rs | 2 +- src/wallet_backend/mod.rs | 10 +- .../platform_compatibility/engine/tests.rs | 52 ++++++ 10 files changed, 272 insertions(+), 96 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 10681570a..6009343b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,8 +52,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). long the backup copies taken before a storage upgrade are kept. By default, backups older than 90 days are now deleted automatically; choose another period (1 to 3650 days) or keep them forever. The newest complete backup of - each database is always kept, and a shorter period asks for confirmation - because it deletes backups at once. + each database is always kept (or the newest backup, if none is complete), and + a shorter period asks for confirmation because it deletes backups at once. - **Add wallet-created identity keys**: for identities loaded from a wallet on this device, the Add Key screen defaults to "Create from wallet", which diff --git a/docs/ai-design/2026-09-10-platform-pin/upgrade-notes.md b/docs/ai-design/2026-09-10-platform-pin/upgrade-notes.md index b9219ab48..18266731a 100644 --- a/docs/ai-design/2026-09-10-platform-pin/upgrade-notes.md +++ b/docs/ai-design/2026-09-10-platform-pin/upgrade-notes.md @@ -72,10 +72,14 @@ name format and be regular files, not symbolic links (or hard links on Unix). Backups older than a user-configurable retention period (default 90 days, or never) are deleted by one global policy covering every database in the data directory: bridge, upstream pre-migration, and legacy `data.db` backups. The -newest usable backup of each database is always kept, whatever its age or the -clock: an empty or truncated file never counts, a backup dated in the future does -not displace one dated in the past, and the upstream snapshot of the latest -migration is kept as well. +newest usable backup of each database is always kept, whatever its age: an empty or +truncated file never counts, and the upstream snapshot of the latest migration is +kept as well. If no backup is usable, the newest file is kept anyway. A backup +dated in the future does not displace one dated in the past, but only until real +time reaches that date; the migration order is the only clock-proof signal. A copy +that cannot be read, or has a non-empty rollback journal beside it, is never +deleted automatically. Legacy `data.db` copies are written to a temporary file, +synced and renamed, so a crash cannot leave a torn copy under a backup name. Clearing a network's data deletes that network's wallet database backups, because they copy its wallet and identity history; backups of the shared `det-app.sqlite` also hold other networks' data and stay under retention. Removing a single wallet diff --git a/docs/user-stories.md b/docs/user-stories.md index 6a375121d..a51fb8498 100644 --- a/docs/user-stories.md +++ b/docs/user-stories.md @@ -345,7 +345,7 @@ As a user whose wallet data was copied before a storage upgrade, I want old upgr - Upgrade backups older than 90 days are deleted automatically by default, covering app, wallet, upstream pre-migration and legacy `data.db` backups. - Settings → Upgrade Backups lets the user change the period (1 to 3650 days) or keep backups forever. Turning automatic deletion on lets the user choose the period before applying it, and any stricter setting asks for confirmation first, because it deletes backups immediately. A period outside the range is shown with an error and never saved. - The setting is one policy for every backup in the data directory, whichever network is open; a saved change applies to every network's backups as soon as the wallet data has opened. -- The newest complete backup of each database is always kept, whatever its age or the system clock, so the last recovery copy is never deleted automatically. An empty or cut-off copy never counts as that backup. +- The newest complete backup of each database is always kept, whatever its age or the system clock, so the last recovery copy is never deleted automatically. An empty or cut-off copy never counts as that backup; if no complete copy exists, the newest copy is kept anyway. A copy that cannot be read is never deleted automatically. - With "keep forever", no published backup is deleted automatically. Opening the data removes only unfinished copies left by a crash and exact duplicates of a newer backup. - Removing a wallet or an identity does not delete upgrade backups. Clearing a network's data deletes that network's wallet backups; backups of the shared app data stay under the retention setting. diff --git a/src/backend_task/error.rs b/src/backend_task/error.rs index 0bcfbdd17..e9b751692 100644 --- a/src/backend_task/error.rs +++ b/src/backend_task/error.rs @@ -109,7 +109,6 @@ impl std::error::Error for BackendTaskJoinError {} /// Dash Core RPC error code: wallet file not specified (multi-wallet node). const RPC_WALLET_NOT_SPECIFIED: i32 = -19; -/// Shown whenever another window or session holds the wallet database. /// Some expired upgrade backups remain. Retention runs again whenever wallet data /// is opened (start, network switch) and whenever the setting is saved. pub(crate) const UPGRADE_BACKUP_CLEANUP_INCOMPLETE: &str = "Some old upgrade backups could not be deleted. The app tries again the next time it opens your wallet data."; @@ -117,6 +116,7 @@ pub(crate) const UPGRADE_BACKUP_CLEANUP_INCOMPLETE: &str = "Some old upgrade bac /// [`UPGRADE_BACKUP_CLEANUP_INCOMPLETE`] after a retention setting save. pub(crate) const BACKUP_RETENTION_SAVED_CLEANUP_INCOMPLETE: &str = "Your backup retention setting was saved, but some old upgrade backups could not be deleted. The app tries again the next time it opens your wallet data."; +/// Shown whenever another window or session holds the wallet database. pub(crate) const WALLET_DATA_IN_USE: &str = "Your wallet data is open in another Dash Evo Tool window or command-line session. Close it and try again."; /// App-level error envelope for backend tasks. @@ -3772,6 +3772,18 @@ fn wallet_already_imported_message(alias: Option<&str>) -> String { mod tests { use super::*; use dash_sdk::dapi_client::DapiClientError; + use dash_sdk::dapi_client::transport::TransportError; + use dash_sdk::dpp::consensus::basic::data_contract::{ + DecimalsOverLimitError, InvalidTokenBaseSupplyError, InvalidTokenLanguageCodeError, + InvalidTokenNameCharacterError, InvalidTokenNameLengthError, + }; + use dash_sdk::dpp::consensus::basic::identity::InvalidInstantAssetLockProofSignatureError; + use dash_sdk::dpp::consensus::state::identity::duplicated_identity_public_key_id_state_error::DuplicatedIdentityPublicKeyIdStateError; + use dash_sdk::dpp::consensus::state::identity::duplicated_identity_public_key_state_error::DuplicatedIdentityPublicKeyStateError; + use dash_sdk::dpp::consensus::state::identity::IdentityInsufficientBalanceError; + use dash_sdk::dpp::consensus::state::identity::identity_public_key_already_exists_for_unique_contract_bounds_error::IdentityPublicKeyAlreadyExistsForUniqueContractBoundsError; + use dash_sdk::dpp::identity::Purpose; + use dash_sdk::platform::Identifier; /// Both cleanup messages promise the same, truthful retry. #[test] @@ -3789,18 +3801,6 @@ mod tests { ); assert!(BACKUP_RETENTION_SAVED_CLEANUP_INCOMPLETE.ends_with(retry)); } - use dash_sdk::dapi_client::transport::TransportError; - use dash_sdk::dpp::consensus::basic::data_contract::{ - DecimalsOverLimitError, InvalidTokenBaseSupplyError, InvalidTokenLanguageCodeError, - InvalidTokenNameCharacterError, InvalidTokenNameLengthError, - }; - use dash_sdk::dpp::consensus::basic::identity::InvalidInstantAssetLockProofSignatureError; - use dash_sdk::dpp::consensus::state::identity::duplicated_identity_public_key_id_state_error::DuplicatedIdentityPublicKeyIdStateError; - use dash_sdk::dpp::consensus::state::identity::duplicated_identity_public_key_state_error::DuplicatedIdentityPublicKeyStateError; - use dash_sdk::dpp::consensus::state::identity::IdentityInsufficientBalanceError; - use dash_sdk::dpp::consensus::state::identity::identity_public_key_already_exists_for_unique_contract_bounds_error::IdentityPublicKeyAlreadyExistsForUniqueContractBoundsError; - use dash_sdk::dpp::identity::Purpose; - use dash_sdk::platform::Identifier; const DAPI_EXHAUSTED_MESSAGE: &str = "All Dash network servers are temporarily unreachable. Please wait a minute and retry."; diff --git a/src/database/initialization.rs b/src/database/initialization.rs index 42e461216..bf47bff83 100644 --- a/src/database/initialization.rs +++ b/src/database/initialization.rs @@ -1,7 +1,6 @@ use crate::database::Database; use chrono::Utc; use rusqlite::{Connection, params}; -use std::fs; use std::path::Path; /// Error during database migration with structured context. @@ -709,23 +708,7 @@ impl Database { /// Backs up the existing database with a unique timestamped filename in backups directory. fn backup_db(&self, db_file_path: &Path) -> rusqlite::Result<()> { if db_file_path.exists() { - // Create a "backups" folder in the same directory as `data.db` if not exists - let backups_dir = db_file_path - .parent() - .expect("Expected parent directory in creating db backup folder") - .join(super::legacy_backups::BACKUP_DIR); - fs::create_dir_all(&backups_dir).map_err(|e| { - rusqlite::Error::ToSqlConversionFailure( - format!("Failed to create db backups directory: {}", e).into(), - ) - })?; - - // Generate a unique filename with a timestamp for the backup - let backup_filename = super::legacy_backups::backup_file_name(Utc::now()); - let backup_path = backups_dir.join(backup_filename); - - // Copy `data.db` to the unique backup file - fs::copy(db_file_path, &backup_path) + let backup_path = super::legacy_backups::write_backup(db_file_path, Utc::now()) .map_err(|e| rusqlite::Error::ToSqlConversionFailure(e.into()))?; tracing::info!("Old database backed up to {:?}", backup_path); } diff --git a/src/database/legacy_backups.rs b/src/database/legacy_backups.rs index 1c4bafc11..b18a84dd5 100644 --- a/src/database/legacy_backups.rs +++ b/src/database/legacy_backups.rs @@ -3,7 +3,7 @@ //! The legacy migration ladder copies `data.db` before upgrading it. These copies are //! subject to the same time-based retention as the storage-upgrade backups. -use std::path::Path; +use std::path::{Path, PathBuf}; use std::time::{Duration, SystemTime}; /// Directory, next to the database, that holds the legacy backups. @@ -20,6 +20,38 @@ pub(crate) fn backup_file_name(timestamp: chrono::DateTime) -> Stri format!("{PREFIX}{}{SUFFIX}", timestamp.format(TIMESTAMP_FORMAT)) } +/// Copy `database` to `/backups/` under the legacy name for `taken`. +/// +/// The copy goes to a unique `.pending` file first, is synced, and only then is +/// renamed to its final name (never replacing an existing backup), after which the +/// directory is synced. A crash or power loss therefore never leaves a torn copy +/// under a backup name, where it could pass as a complete backup. +pub(crate) fn write_backup( + database: &Path, + taken: chrono::DateTime, +) -> std::io::Result { + let directory = database + .parent() + .ok_or(std::io::ErrorKind::InvalidInput)? + .join(BACKUP_DIR); + std::fs::create_dir_all(&directory)?; + let name = backup_file_name(taken); + // Not routed through `delete_file`: on failure the temp file removes only the + // uniquely named `.pending` file it created itself. + let pending = tempfile::Builder::new() + .prefix(&format!("{name}.")) + .suffix(".pending") + .tempfile_in(&directory)?; + std::fs::copy(database, pending.path())?; + pending.as_file().sync_all()?; + let published = directory.join(name); + pending + .persist_noclobber(&published) + .map_err(|error| error.error)?; + crate::utils::backup_prune::sync_directory(&directory)?; + Ok(published) +} + /// The UTC creation time embedded in a legacy backup name, or `None` for any other /// file. Also the deletion chokepoint's definition of a legacy backup name. pub(crate) fn backup_timestamp(name: &str) -> Option { @@ -97,6 +129,38 @@ mod tests { .unwrap(); } + /// The published copy is complete, no temporary file is left, and an existing + /// backup of the same second is never overwritten. + #[test] + fn legacy_backup_is_published_complete_and_never_overwrites() { + let dir = tempfile::tempdir().unwrap(); + let database = dir.path().join(LEGACY_DATABASE); + rusqlite::Connection::open(&database) + .unwrap() + .execute_batch("CREATE TABLE settings (v INTEGER); INSERT INTO settings VALUES (1);") + .unwrap(); + let taken = chrono::Utc::now(); + + let published = write_backup(&database, taken).unwrap(); + + assert_eq!( + published, + dir.path().join(BACKUP_DIR).join(backup_file_name(taken)) + ); + assert_eq!( + std::fs::read(&published).unwrap(), + std::fs::read(&database).unwrap() + ); + assert!(crate::utils::backup_prune::usable_snapshot(&published)); + let error = write_backup(&database, taken).unwrap_err(); + assert_eq!(error.kind(), std::io::ErrorKind::AlreadyExists); + let entries: Vec<_> = std::fs::read_dir(dir.path().join(BACKUP_DIR)) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .collect(); + assert_eq!(entries, [published.file_name().unwrap().to_owned()]); + } + #[test] fn legacy_backup_names_round_trip() { let taken = chrono::DateTime::parse_from_rfc3339("2024-03-05T06:07:08Z") diff --git a/src/utils/backup_prune.rs b/src/utils/backup_prune.rs index ae66735c5..3054d2bde 100644 --- a/src/utils/backup_prune.rs +++ b/src/utils/backup_prune.rs @@ -2,20 +2,30 @@ //! //! One algorithm for the bridge, upstream `pre-migration-*` and legacy `data.db` //! backups: delete those older than the retention period, except the newest usable -//! backup of the database, which is kept whatever its age. Neither a wrong clock nor -//! an interrupted copy can therefore delete the last recovery copy. +//! backup of the database, which is kept whatever its age. An interrupted copy can +//! therefore never delete the last recovery copy. //! //! "Newest" is judged by every ordering signal a backup carries, and the newest by //! each signal is kept: -//! - its creation time, preferring backups not dated in the future, so a copy taken -//! while the clock ran ahead cannot displace a genuinely newer one; +//! - its creation time, preferring backups not dated in the future. A copy taken +//! while the clock ran ahead does not displace a genuinely newer one, but only +//! while its date is still in the future: once real time catches up, it counts as +//! the newest again and the newer copy expires normally. No clock-based order can +//! do better without remembering past passes; //! - its position in the database's upgrade history (upstream snapshots name the //! migration they precede), which no clock can skew. //! -//! "Usable" is a cheap structural check ([`usable_snapshot`]). Empty, truncated, -//! header-only and torn files fail it. When no backup of a database passes, the -//! newest files are kept anyway, so a false negative of the check can never delete -//! every backup. +//! Usability is a cheap structural check ([`check_snapshot`]) with three outcomes: +//! - usable: only these can be the newest backup that is kept; +//! - unusable (empty, truncated, header-only or not SQLite): expires like any other; +//! - undetermined (the file cannot be read, or a non-empty rollback journal lies +//! beside it, so it may hold a torn write): never deleted automatically, and never +//! the reason an older usable copy stops being kept. A read error is also reported, +//! so the pass is retried; a leftover journal keeps the copy until the user removes +//! the journal. +//! +//! When no backup of a database is usable, the newest files are kept anyway, so a +//! false negative of the check can never delete every backup. use super::file_deletion::{DeletionIntent, delete_file}; use std::collections::BTreeSet; @@ -43,16 +53,29 @@ struct Dated { created: SystemTime, path: PathBuf, sequence: Option<(u32, u32)>, - usable: bool, + usability: Usability, +} + +/// Whether a backup can be restored, as far as a cheap structural check can tell. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Usability { + /// A complete SQLite database. + Usable, + /// Empty, truncated, header-only or not SQLite at all. + Unusable, + /// Not judged: the file could not be read, or it looks complete but has a + /// non-empty rollback journal beside it (a torn write or a harmless leftover). + Undetermined, } /// Delete every candidate older than `max_age` at `now`, except the newest usable one -/// by each ordering signal (see the module docs). +/// by each ordering signal and any whose usability is undetermined (see the module +/// docs). /// /// All candidates must be backups of the database named by `intent`. A candidate -/// whose creation time cannot be read is kept and reported. A candidate that is -/// already gone (e.g. pruned concurrently from another network's context) counts -/// as handled. Attempts every candidate, syncs the directories it deleted from, +/// whose creation time or contents cannot be read is kept and reported. A candidate +/// that is already gone (e.g. pruned concurrently from another network's context) +/// counts as handled. Attempts every candidate, syncs the directories it deleted from, /// and returns the first failure; `Ok` carries the number of backups deleted. pub(crate) fn prune_expired( candidates: Vec, @@ -63,24 +86,37 @@ pub(crate) fn prune_expired( let mut first_error = None; let mut dated = Vec::with_capacity(candidates.len()); for candidate in candidates { - match candidate.created { - Ok(created) => dated.push(Dated { - created, - usable: usable_snapshot(&candidate.path), - path: candidate.path, - sequence: candidate.sequence, - }), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + let checked = candidate + .created + .map(|created| (created, check_snapshot(&candidate.path))); + let (created, usability) = match checked { + Ok((created, Ok(usability))) => (created, usability), + Ok((created, Err(error))) if error.kind() != std::io::ErrorKind::NotFound => { + // Unreadable is not proof of unusable: keep the file and retry later. + first_error.get_or_insert(error); + (created, Usability::Undetermined) + } + // Already gone, e.g. pruned concurrently from another network's context. + Ok((_, Err(_))) => continue, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, Err(error) => { first_error.get_or_insert(error); + continue; } - } + }; + dated.push(Dated { + created, + path: candidate.path, + sequence: candidate.sequence, + usability, + }); } let floors = floors(&dated, now); let mut removed = 0; let mut touched = BTreeSet::new(); for (index, backup) in dated.iter().enumerate() { if floors.contains(&index) + || backup.usability == Usability::Undetermined || !crate::model::backup_retention::backup_expired(backup.created, now, max_age) { continue; @@ -112,12 +148,13 @@ pub(crate) fn prune_expired( /// concurrent context picks the same files. With no usable backup at all, the same /// choice is made among every backup instead. fn floors(dated: &[Dated], now: SystemTime) -> BTreeSet { - let any_usable = dated.iter().any(|backup| backup.usable); + let usable = |backup: &Dated| backup.usability == Usability::Usable; + let any_usable = dated.iter().any(usable); let eligible = || { dated .iter() .enumerate() - .filter(move |(_, backup)| backup.usable || !any_usable) + .filter(move |(_, backup)| usable(backup) || !any_usable) }; let latest_trusted = now.checked_add(FUTURE_TOLERANCE); let by_time = eligible() @@ -138,35 +175,31 @@ const SQLITE_HEADER_LEN: usize = 100; /// Magic string every SQLite 3 database file starts with. const SQLITE_MAGIC: &[u8] = b"SQLite format 3\0"; -/// Whether `backup` looks like a complete SQLite database that can be restored. -/// -/// A structural check that reads only the 100-byte header and never opens the file -/// through SQLite, which could create `-wal`/`-shm` sidecars or roll back a journal. -/// It requires the magic string, a valid page size and file-format version, a file -/// size covering every page the header claims, and no non-empty `-journal` beside -/// the file: a hot rollback journal means the file holds a torn, uncommitted write. -/// Any read error counts as unusable. +/// Whether `backup` is proven to be a complete SQLite database. +#[cfg(test)] pub(crate) fn usable_snapshot(backup: &Path) -> bool { - match check_snapshot(backup) { - Ok(usable) => usable, - Err(error) => { - tracing::debug!(backup = %backup.display(), ?error, "Upgrade backup could not be checked"); - false - } - } + matches!(check_snapshot(backup), Ok(Usability::Usable)) } -fn check_snapshot(backup: &Path) -> std::io::Result { +/// How restorable `backup` looks. +/// +/// A structural check that reads only the 100-byte header and never opens the file +/// through SQLite, which could create `-wal`/`-shm` sidecars or roll back a journal. +/// Usable requires the magic string, a valid page size and file-format version, and +/// a file size covering every page the header claims. A complete-looking file with a +/// non-empty `-journal` beside it is [`Usability::Undetermined`]. A file that cannot +/// be read is an error, never a verdict. +fn check_snapshot(backup: &Path) -> std::io::Result { use std::io::Read; let mut file = std::fs::File::open(backup)?; let length = file.metadata()?.len(); if length < SQLITE_HEADER_LEN as u64 { - return Ok(false); + return Ok(Usability::Unusable); } let mut header = [0u8; SQLITE_HEADER_LEN]; file.read_exact(&mut header)?; if !header.starts_with(SQLITE_MAGIC) { - return Ok(false); + return Ok(Usability::Unusable); } let be_u32 = |offset: usize| { u32::from_be_bytes([ @@ -179,11 +212,11 @@ fn check_snapshot(backup: &Path) -> std::io::Result { let page_size = match u16::from_be_bytes([header[16], header[17]]) { 1 => 65_536, size if (512..=32_768).contains(&size) && size.is_power_of_two() => u64::from(size), - _ => return Ok(false), + _ => return Ok(Usability::Unusable), }; // File-format write/read versions: 1 = rollback journal, 2 = WAL. if !matches!(header[18], 1 | 2) || !matches!(header[19], 1 | 2) { - return Ok(false); + return Ok(Usability::Unusable); } // The in-header page count is valid only when "version-valid-for" matches the // change counter, as every SQLite since 3.7.0 writes it; otherwise require @@ -195,13 +228,17 @@ fn check_snapshot(backup: &Path) -> std::io::Result { length >= page_size && length % page_size == 0 }; if !complete { - return Ok(false); + return Ok(Usability::Unusable); } let mut journal = backup.as_os_str().to_owned(); journal.push("-journal"); match std::fs::symlink_metadata(journal) { - Ok(metadata) => Ok(metadata.len() == 0), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(true), + Ok(metadata) if metadata.len() > 0 => { + tracing::debug!(backup = %backup.display(), "Upgrade backup has a rollback journal beside it; kept"); + Ok(Usability::Undetermined) + } + Ok(_) => Ok(Usability::Usable), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(Usability::Usable), Err(error) => Err(error), } } @@ -237,7 +274,7 @@ mod tests { created, path: path.to_owned(), sequence, - usable: usable_snapshot(path), + usability: check_snapshot(path).unwrap_or(Usability::Undetermined), } } @@ -275,21 +312,29 @@ mod tests { for (case, content) in cases { let path = dir.path().join(format!("{case}.db")); std::fs::write(&path, content).unwrap(); - assert!(!usable_snapshot(&path), "{case}"); + assert_eq!( + check_snapshot(&path).unwrap(), + Usability::Unusable, + "{case}" + ); } assert!(!usable_snapshot(&dir.path().join("missing.db")), "missing"); } #[test] - fn hot_journal_makes_a_copy_unusable() { + fn hot_journal_leaves_a_copy_undetermined() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("torn.db"); write_database(&path); let journal = dir.path().join("torn.db-journal"); std::fs::write(&journal, b"").unwrap(); - assert!(usable_snapshot(&path), "an empty journal is not hot"); + assert_eq!( + check_snapshot(&path).unwrap(), + Usability::Usable, + "an empty journal is not hot" + ); std::fs::write(&journal, b"uncommitted pages").unwrap(); - assert!(!usable_snapshot(&path)); + assert_eq!(check_snapshot(&path).unwrap(), Usability::Undetermined); } /// A newer unusable file never becomes the floor; the newest usable one does. @@ -308,6 +353,23 @@ mod tests { assert_eq!(floors(&backups, now), BTreeSet::from([0])); } + /// A newer copy of undetermined usability does not displace the newest usable one. + #[test] + fn floor_ignores_undetermined_newer_files() { + let dir = tempfile::tempdir().unwrap(); + let now = SystemTime::now(); + let [valid, journaled] = ["valid.db", "journaled.db"].map(|name| dir.path().join(name)); + write_database(&valid); + write_database(&journaled); + std::fs::write(dir.path().join("journaled.db-journal"), b"pages").unwrap(); + let backups = [ + dated(&valid, now - DAY * 200, None), + dated(&journaled, now - DAY * 100, Some((2, 1))), + ]; + assert_eq!(backups[1].usability, Usability::Undetermined); + assert_eq!(floors(&backups, now), BTreeSet::from([0])); + } + /// Without any usable backup the newest file is still kept. #[test] fn floor_falls_back_to_newest_file_when_none_is_usable() { @@ -346,6 +408,15 @@ mod tests { dated(&paths[1], now - DAY, None), ]; assert_eq!(floors(&backups, now), BTreeSet::from([0])); + // Once real time catches up with a skewed date, that backup is trusted + // again and, being dated later, is the time floor once more. A limit of + // any clock-based order; only the migration order is immune. + let skewed = now + DAY * 30; + let backups = [ + dated(&paths[0], skewed, None), + dated(&paths[1], now - DAY * 100, None), + ]; + assert_eq!(floors(&backups, skewed), BTreeSet::from([0])); // Only future-dated backups: the newest of them. let backups = [ dated(&paths[0], now + DAY * 30, None), diff --git a/src/utils/file_deletion.rs b/src/utils/file_deletion.rs index 87bd22595..0cc0d8808 100644 --- a/src/utils/file_deletion.rs +++ b/src/utils/file_deletion.rs @@ -382,7 +382,7 @@ fn in_scope(located: &Path, name: &str, intent: DeletionIntent<'_>) -> bool { && crate::database::legacy_backups::backup_timestamp(name).is_some()) } DeletionIntent::NetworkClear { data_dir, network } => { - let spv = data_dir.join("spv"); + let spv = crate::wallet_backend::spv_root_dir(data_dir); let lock = format!("{}.lock", crate::wallet_backend::network_prefix(network)); parent.starts_with(canonical_dir(&crate::wallet_backend::spv_storage_dir( data_dir, network, diff --git a/src/wallet_backend/mod.rs b/src/wallet_backend/mod.rs index e87487b07..cf6978c68 100644 --- a/src/wallet_backend/mod.rs +++ b/src/wallet_backend/mod.rs @@ -561,8 +561,10 @@ pub(crate) fn wallet_database_path(data_dir: &Path, network: Network) -> std::pa data_dir.join(format!("det-{}.sqlite", network_prefix(network))) } -/// Parent of every network's [`spv_storage_dir`]. -const SPV_ROOT_DIR: &str = "spv"; +/// Parent of every network's [`spv_storage_dir`], `/spv/`. +pub(crate) fn spv_root_dir(data_dir: &Path) -> std::path::PathBuf { + data_dir.join("spv") +} /// The per-network chain-sync (SPV) cache directory, `/spv//`. /// @@ -570,7 +572,7 @@ const SPV_ROOT_DIR: &str = "spv"; /// clear (which empties it) and the deletion chokepoint (which confines that clear /// to it). pub(crate) fn spv_storage_dir(data_dir: &Path, network: Network) -> std::path::PathBuf { - data_dir.join(SPV_ROOT_DIR).join(network_prefix(network)) + spv_root_dir(data_dir).join(network_prefix(network)) } /// The upstream shielded coordinator's store, @@ -3081,7 +3083,7 @@ impl WalletBackend { ) -> Result { let dir = spv_storage_dir(app_data_dir, network); // Create (and lock down) the shared `spv` root before the network directory. - for directory in [app_data_dir.join(SPV_ROOT_DIR), dir.clone()] { + for directory in [spv_root_dir(app_data_dir), dir.clone()] { crate::app_dir::ensure_data_dir_exists(&directory) .map_err(|source| TaskError::FileSystem { source })?; } diff --git a/src/wallet_backend/platform_compatibility/engine/tests.rs b/src/wallet_backend/platform_compatibility/engine/tests.rs index a3c80ba21..16fdce3ad 100644 --- a/src/wallet_backend/platform_compatibility/engine/tests.rs +++ b/src/wallet_backend/platform_compatibility/engine/tests.rs @@ -1083,6 +1083,58 @@ fn platform_compatibility_prune_floor_skips_interrupted_newest_copy() { } } +/// A newest snapshot whose usability cannot be proven (unreadable, or with a +/// leftover rollback journal beside it) is never deleted, and never hands the floor +/// to an older copy that would then be the only one left. An unreadable one is +/// reported so the pass is retried. +#[test] +fn platform_compatibility_prune_keeps_snapshots_it_cannot_judge() { + const DAY_SECONDS: i64 = 24 * 60 * 60; + for case in ["unreadable", "leftover journal"] { + let (_dir, path, _target) = fixture(); + let older = backup(&path).unwrap(); + let newest = backup(&path).unwrap(); + set_mtime(&older, -200 * DAY_SECONDS); + set_mtime(&newest, -100 * DAY_SECONDS); + let journal = PathBuf::from(format!("{}-journal", newest.display())); + match case { + "unreadable" => { + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&newest, std::fs::Permissions::from_mode(0o000)) + .unwrap(); + if std::fs::File::open(&newest).is_ok() { + // Running as root: permissions cannot make it unreadable. + std::fs::set_permissions(&newest, std::fs::Permissions::from_mode(0o600)) + .unwrap(); + continue; + } + } + #[cfg(not(unix))] + continue; + } + _ => std::fs::write(&journal, b"pages of an unfinished write").unwrap(), + } + + let result = prune_expired_backups(&path, RETENTION_DAY * 90, std::time::SystemTime::now()); + + assert!(newest.exists(), "{case}: an unjudged snapshot is kept"); + assert!(older.exists(), "{case}: the newest proven snapshot is kept"); + match case { + "unreadable" => { + result.unwrap_err(); + } + _ => assert_eq!(result.unwrap(), 0, "{case}"), + } + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&newest, std::fs::Permissions::from_mode(0o600)).unwrap(); + } + } +} + /// A snapshot written while the clock ran ahead is dated in the future; it must not /// displace the genuinely newer snapshot taken after the clock was corrected. #[test] From 40462873c6e71f37b52de1f1759b270e2dcd4258 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Wed, 30 Sep 2026 11:08:52 +0000 Subject: [PATCH 25/30] fix(storage): sweep crash-left legacy backup copies A legacy data.db backup interrupted before its rename left a `..pending` file that nothing ever removed. Every retention pass now sweeps such copies, whatever the policy, once they are more than a day old (so a copy still being written is left alone). Only the exact temporary-name grammar qualifies, and the deletion chokepoint's legacy allow-list accepts that grammar and nothing looser. WAL-034 now states the accepted clock limit instead of promising the floor holds whatever the system clock does. Co-Authored-By: Claude Opus 5.5 --- docs/user-stories.md | 2 +- src/context/wallet_lifecycle/retention.rs | 47 ++++++- src/database/legacy_backups.rs | 161 ++++++++++++++++++++-- src/utils/file_deletion.rs | 3 +- src/utils/file_deletion/tests.rs | 12 ++ 5 files changed, 208 insertions(+), 17 deletions(-) diff --git a/docs/user-stories.md b/docs/user-stories.md index a51fb8498..5c4c65896 100644 --- a/docs/user-stories.md +++ b/docs/user-stories.md @@ -345,7 +345,7 @@ As a user whose wallet data was copied before a storage upgrade, I want old upgr - Upgrade backups older than 90 days are deleted automatically by default, covering app, wallet, upstream pre-migration and legacy `data.db` backups. - Settings → Upgrade Backups lets the user change the period (1 to 3650 days) or keep backups forever. Turning automatic deletion on lets the user choose the period before applying it, and any stricter setting asks for confirmation first, because it deletes backups immediately. A period outside the range is shown with an error and never saved. - The setting is one policy for every backup in the data directory, whichever network is open; a saved change applies to every network's backups as soon as the wallet data has opened. -- The newest complete backup of each database is always kept, whatever its age or the system clock, so the last recovery copy is never deleted automatically. An empty or cut-off copy never counts as that backup; if no complete copy exists, the newest copy is kept anyway. A copy that cannot be read is never deleted automatically. +- The newest complete backup of each database is always kept, whatever its age, so the last recovery copy is never deleted automatically. A backup dated in the future because the system clock ran ahead is not treated as the newest until that date arrives. An empty or cut-off copy never counts as that backup; if no complete copy exists, the newest copy is kept anyway. A copy that cannot be read is never deleted automatically. - With "keep forever", no published backup is deleted automatically. Opening the data removes only unfinished copies left by a crash and exact duplicates of a newer backup. - Removing a wallet or an identity does not delete upgrade backups. Clearing a network's data deletes that network's wallet backups; backups of the shared app data stay under the retention setting. diff --git a/src/context/wallet_lifecycle/retention.rs b/src/context/wallet_lifecycle/retention.rs index 395dd164a..ceb2e61a9 100644 --- a/src/context/wallet_lifecycle/retention.rs +++ b/src/context/wallet_lifecycle/retention.rs @@ -31,9 +31,14 @@ impl AppContext { /// The policy is global: it covers every database in the data directory (the app /// database and every network's wallet database, bridge and upstream /// `pre-migration-*` snapshots), plus legacy `data.db` backups, whichever network - /// this context serves. The newest backup of each database is always kept, so the - /// pass is safe for databases this process has not opened. A database another - /// context is opening or upgrading is skipped until the next pass. + /// this context serves. The newest usable backup of each database is always kept, + /// so the pass is safe for databases this process has not opened. A database + /// another context is opening or upgrading is skipped until the next pass. + /// + /// Every policy, including keeping backups forever, also removes legacy copies a + /// crash left unpublished (see + /// [`sweep_pending`](crate::database::legacy_backups::sweep_pending)); those are not + /// counted as deleted backups. /// /// Attempts every location and reports the first failure. Nothing is deleted when the /// policy cannot be read. @@ -51,6 +56,17 @@ impl AppContext { return report; } }; + // Crash-left legacy copies were never published backups, so every policy + // removes them. + match crate::database::legacy_backups::sweep_pending(self.data_dir(), now) { + Ok(0) => {} + Ok(swept) => tracing::info!(swept, "Removed unfinished legacy backup copies"), + Err(source) => { + report + .failure + .get_or_insert(TaskError::UpgradeBackupCleanup { source }); + } + } let Some(max_age) = retention.max_age() else { return report; }; @@ -180,6 +196,31 @@ mod tests { assert!(fixture.fresh.iter().all(|path| path.exists())); } + /// A legacy copy left unpublished by a crash is removed by every retention pass, + /// whatever the policy, once it is clearly not a copy still being written. It + /// is not a backup, so it is not counted as a deleted one. + #[test] + fn crash_left_legacy_copies_are_swept_under_any_policy() { + for policy in [BackupRetention::KeepForever, BackupRetention::default()] { + let tmp = tempfile::tempdir().unwrap(); + let ctx = test_app_context(tmp.path()); + ctx.set_backup_retention(policy).unwrap(); + let backups = ctx.data_dir().join("backups"); + let crashed = backups.join("data_backup_20200101_000000.db.a1B2c3.pending"); + let in_progress = backups.join("data_backup_20200102_000000.db.Z9y8X7.pending"); + write_aged(&crashed, DAY * 2); + write_aged(&in_progress, Duration::from_secs(5)); + + assert_eq!(pruned(&ctx), 0, "{policy:?}"); + + assert!(!crashed.exists(), "{policy:?}: a crash-left copy is swept"); + assert!( + in_progress.exists(), + "{policy:?}: a recent copy may still be written" + ); + } + } + #[test] fn keep_forever_deletes_nothing() { let tmp = tempfile::tempdir().unwrap(); diff --git a/src/database/legacy_backups.rs b/src/database/legacy_backups.rs index b18a84dd5..1dc0e2a97 100644 --- a/src/database/legacy_backups.rs +++ b/src/database/legacy_backups.rs @@ -14,6 +14,11 @@ const PREFIX: &str = "data_backup_"; const SUFFIX: &str = ".db"; /// UTC timestamp format embedded in the backup name. const TIMESTAMP_FORMAT: &str = "%Y%m%d_%H%M%S"; +/// Suffix of the unpublished copy [`write_backup`] writes before renaming it. +const PENDING_SUFFIX: &str = ".pending"; +/// How old an unpublished copy must be before it counts as left behind by a crash +/// rather than still being written. Copying `data.db` takes seconds. +const PENDING_GRACE: Duration = Duration::from_secs(24 * 60 * 60); /// File name of a legacy backup taken at `timestamp` (UTC). pub(crate) fn backup_file_name(timestamp: chrono::DateTime) -> String { @@ -40,7 +45,7 @@ pub(crate) fn write_backup( // uniquely named `.pending` file it created itself. let pending = tempfile::Builder::new() .prefix(&format!("{name}.")) - .suffix(".pending") + .suffix(PENDING_SUFFIX) .tempfile_in(&directory)?; std::fs::copy(database, pending.path())?; pending.as_file().sync_all()?; @@ -60,6 +65,91 @@ pub(crate) fn backup_timestamp(name: &str) -> Option { Some(parsed.and_utc().into()) } +/// Whether `name` is an unpublished copy written by [`write_backup`]: +/// `..pending`. Also the deletion +/// chokepoint's definition of such a copy. +pub(crate) fn is_pending_backup_name(name: &str) -> bool { + name.strip_suffix(PENDING_SUFFIX) + .and_then(|rest| rest.rsplit_once('.')) + .is_some_and(|(published, random)| { + !random.is_empty() + && random.bytes().all(|b| b.is_ascii_alphanumeric()) + && backup_timestamp(published).is_some() + }) +} + +/// `/backups` if it exists; an error if it is a symlink or not a directory. +fn backup_directory(data_dir: &Path) -> std::io::Result> { + let directory = data_dir.join(BACKUP_DIR); + match std::fs::symlink_metadata(&directory) { + Ok(metadata) if metadata.is_dir() => Ok(Some(directory)), + Ok(_) => Err(std::io::Error::other( + "Legacy backup directory is not a regular directory", + )), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(error) => Err(error), + } +} + +/// Delete unpublished copies [`write_backup`] left behind when the app stopped +/// mid-copy. +/// +/// Only regular files matching [`is_pending_backup_name`] whose modification time is +/// more than a day before `now` are removed, so a copy another process is still +/// writing is left alone; one dated in the future is kept. They never held a +/// published backup, so every retention policy removes them. Deletion goes through +/// the chokepoint. Attempts every file and returns the first failure; `Ok` carries +/// the number removed. +pub(crate) fn sweep_pending(data_dir: &Path, now: SystemTime) -> std::io::Result { + let Some(directory) = backup_directory(data_dir)? else { + return Ok(0); + }; + let database = data_dir.join(LEGACY_DATABASE); + let mut first_error = None; + let mut removed = 0; + for entry in std::fs::read_dir(&directory)? { + let entry = entry?; + if !entry + .file_name() + .to_str() + .is_some_and(is_pending_backup_name) + { + continue; + } + let outcome = match abandoned(&entry.path(), now) { + Ok(true) => crate::utils::file_deletion::delete_file( + &entry.path(), + crate::utils::file_deletion::DeletionIntent::Backup { + database: &database, + }, + ), + Ok(false) => continue, + Err(error) => Err(error), + }; + match outcome { + Ok(()) => removed += 1, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + first_error.get_or_insert(error); + } + } + } + if removed > 0 + && let Err(error) = crate::utils::backup_prune::sync_directory(&directory) + { + first_error.get_or_insert(error); + } + first_error.map_or(Ok(removed), Err) +} + +/// Whether the unpublished copy at `path` is a regular file last written more than +/// [`PENDING_GRACE`] before `now`. +fn abandoned(path: &Path, now: SystemTime) -> std::io::Result { + let metadata = std::fs::symlink_metadata(path)?; + Ok(metadata.is_file() + && crate::model::backup_retention::backup_expired(metadata.modified()?, now, PENDING_GRACE)) +} + /// Delete legacy backups in `/backups` older than `max_age`, always /// keeping the newest usable one. /// @@ -76,17 +166,9 @@ pub(crate) fn prune_expired( max_age: Duration, now: SystemTime, ) -> std::io::Result { - let directory = data_dir.join(BACKUP_DIR); - match std::fs::symlink_metadata(&directory) { - Ok(metadata) if metadata.is_dir() => {} - Ok(_) => { - return Err(std::io::Error::other( - "Legacy backup directory is not a regular directory", - )); - } - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(0), - Err(error) => return Err(error), - } + let Some(directory) = backup_directory(data_dir)? else { + return Ok(0); + }; let mut candidates = Vec::new(); for entry in std::fs::read_dir(&directory)? { let entry = entry?; @@ -161,6 +243,61 @@ mod tests { assert_eq!(entries, [published.file_name().unwrap().to_owned()]); } + #[test] + fn pending_copy_names_follow_write_backup_exactly() { + assert!(is_pending_backup_name( + "data_backup_20200101_000000.db.a1B2c3.pending" + )); + for other in [ + "data_backup_20200101_000000.db.pending", + "data_backup_20200101_000000.db..pending", + "data_backup_20200101_000000.db.a-1.pending", + "data_backup_20200101_000000.db.abc.pending-journal", + "data_backup_latest.db.abc123.pending", + "notes.db.abc123.pending", + "data.db.abc123.pending", + "data_backup_20200101_000000.db", + ] { + assert!(!is_pending_backup_name(other), "{other}"); + } + } + + /// Only old regular files with the exact temporary name are swept, through the + /// chokepoint; lookalikes, directories, symlinks and published backups stay. + #[test] + fn sweep_removes_only_abandoned_pending_copies() { + let dir = tempfile::tempdir().unwrap(); + let backups = dir.path().join(BACKUP_DIR); + std::fs::create_dir_all(&backups).unwrap(); + let now = SystemTime::now(); + let write = |name: &str, age: Duration| { + let path = backups.join(name); + std::fs::write(&path, b"partial").unwrap(); + set_mtime(&path, now - age); + path + }; + let abandoned = write("data_backup_20200101_000000.db.a1B2c3.pending", DAY * 2); + let fresh = write("data_backup_20200102_000000.db.Z9y8X7.pending", DAY / 24); + let published = write("data_backup_20200101_000000.db", DAY * 400); + let lookalike = write("data_backup_20200101_000000.db.pending", DAY * 400); + let unrelated = write("notes.db.abc123.pending", DAY * 400); + let directory = backups.join("data_backup_20200103_000000.db.dir123.pending"); + std::fs::create_dir(&directory).unwrap(); + + assert_eq!(sweep_pending(dir.path(), now).unwrap(), 1); + + assert!(!abandoned.exists()); + for kept in [&fresh, &published, &lookalike, &unrelated, &directory] { + assert!(kept.exists(), "{}", kept.display()); + } + assert_eq!(sweep_pending(dir.path(), now).unwrap(), 0); + assert_eq!( + sweep_pending(tempfile::tempdir().unwrap().path(), now).unwrap(), + 0, + "no backups directory" + ); + } + #[test] fn legacy_backup_names_round_trip() { let taken = chrono::DateTime::parse_from_rfc3339("2024-03-05T06:07:08Z") diff --git a/src/utils/file_deletion.rs b/src/utils/file_deletion.rs index 0cc0d8808..013aad3f8 100644 --- a/src/utils/file_deletion.rs +++ b/src/utils/file_deletion.rs @@ -379,7 +379,8 @@ fn in_scope(located: &Path, name: &str, intent: DeletionIntent<'_>) -> bool { .is_some()) || (legacy && is(&directory.join(crate::database::legacy_backups::BACKUP_DIR)) - && crate::database::legacy_backups::backup_timestamp(name).is_some()) + && (crate::database::legacy_backups::backup_timestamp(name).is_some() + || crate::database::legacy_backups::is_pending_backup_name(name))) } DeletionIntent::NetworkClear { data_dir, network } => { let spv = crate::wallet_backend::spv_root_dir(data_dir); diff --git a/src/utils/file_deletion/tests.rs b/src/utils/file_deletion/tests.rs index 3f64de561..426206208 100644 --- a/src/utils/file_deletion/tests.rs +++ b/src/utils/file_deletion/tests.rs @@ -281,6 +281,13 @@ fn delete_file_refuses_targets_outside_the_intent_scope() { database: &database, }, ), + ( + dir.path() + .join("backups/data_backup_20000101_000000.db.a1B2c3.pending"), + DeletionIntent::Backup { + database: &database, + }, + ), ( dir.path() .join("backups/auto/pre-migration-det-mainnet-1-to-2-20000101T000000Z.db"), @@ -346,6 +353,11 @@ fn delete_file_deletes_targets_in_scope() { dir.path().join("backups/data_backup_20000101_000000.db"), DeletionIntent::Backup { database: &legacy }, ), + ( + dir.path() + .join("backups/data_backup_20000101_000000.db.a1B2c3.pending"), + DeletionIntent::Backup { database: &legacy }, + ), ( dir.path() .join("backups/auto/pre-migration-det-testnet-1-to-2-20000101T000000Z.db"), From 15ef632019c664162fe8868b78f571a03860e5ca Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Wed, 30 Sep 2026 11:12:41 +0000 Subject: [PATCH 26/30] fix(storage): never let a legacy backup name collision block startup Legacy backup names have one-second resolution, and the non-clobbering publish returned AlreadyExists when a backup of the same second existed, aborting startup through backup_db. The copy is now published under the next free second (up to 60 tries), which keeps the data_backup_ grammar and the chokepoint allow-list, never overwrites an existing backup, and leaves no temporary copy behind. Co-Authored-By: Claude Opus 5.5 --- src/database/legacy_backups.rs | 110 +++++++++++++++++++++++++++------ 1 file changed, 90 insertions(+), 20 deletions(-) diff --git a/src/database/legacy_backups.rs b/src/database/legacy_backups.rs index 1dc0e2a97..bc97eba6f 100644 --- a/src/database/legacy_backups.rs +++ b/src/database/legacy_backups.rs @@ -14,6 +14,8 @@ const PREFIX: &str = "data_backup_"; const SUFFIX: &str = ".db"; /// UTC timestamp format embedded in the backup name. const TIMESTAMP_FORMAT: &str = "%Y%m%d_%H%M%S"; +/// How many consecutive one-second names [`write_backup`] tries before giving up. +const MAX_NAME_ATTEMPTS: usize = 60; /// Suffix of the unpublished copy [`write_backup`] writes before renaming it. const PENDING_SUFFIX: &str = ".pending"; /// How old an unpublished copy must be before it counts as left behind by a crash @@ -28,9 +30,11 @@ pub(crate) fn backup_file_name(timestamp: chrono::DateTime) -> Stri /// Copy `database` to `/backups/` under the legacy name for `taken`. /// /// The copy goes to a unique `.pending` file first, is synced, and only then is -/// renamed to its final name (never replacing an existing backup), after which the -/// directory is synced. A crash or power loss therefore never leaves a torn copy -/// under a backup name, where it could pass as a complete backup. +/// renamed to its final name, after which the directory is synced. A crash or power +/// loss therefore never leaves a torn copy under a backup name, where it could pass +/// as a complete backup. An existing backup is never replaced: when the name for +/// `taken` is already used (names have one-second resolution), the next free second +/// is used instead, so a collision never fails the copy. pub(crate) fn write_backup( database: &Path, taken: chrono::DateTime, @@ -43,18 +47,28 @@ pub(crate) fn write_backup( let name = backup_file_name(taken); // Not routed through `delete_file`: on failure the temp file removes only the // uniquely named `.pending` file it created itself. - let pending = tempfile::Builder::new() + let mut pending = tempfile::Builder::new() .prefix(&format!("{name}.")) .suffix(PENDING_SUFFIX) .tempfile_in(&directory)?; std::fs::copy(database, pending.path())?; pending.as_file().sync_all()?; - let published = directory.join(name); - pending - .persist_noclobber(&published) - .map_err(|error| error.error)?; - crate::utils::backup_prune::sync_directory(&directory)?; - Ok(published) + let mut stamp = taken; + for _ in 0..MAX_NAME_ATTEMPTS { + let published = directory.join(backup_file_name(stamp)); + match pending.persist_noclobber(&published) { + Ok(_) => { + crate::utils::backup_prune::sync_directory(&directory)?; + return Ok(published); + } + Err(error) if error.error.kind() == std::io::ErrorKind::AlreadyExists => { + pending = error.file; + stamp += chrono::TimeDelta::seconds(1); + } + Err(error) => return Err(error.error), + } + } + Err(std::io::ErrorKind::AlreadyExists.into()) } /// The UTC creation time embedded in a legacy backup name, or `None` for any other @@ -211,8 +225,9 @@ mod tests { .unwrap(); } - /// The published copy is complete, no temporary file is left, and an existing - /// backup of the same second is never overwritten. + /// The published copy is complete and no temporary file is left. A backup of the + /// same second is never overwritten, and never blocks the new copy: it is + /// published under the next free second, still a legacy backup name. #[test] fn legacy_backup_is_published_complete_and_never_overwrites() { let dir = tempfile::tempdir().unwrap(); @@ -222,25 +237,80 @@ mod tests { .execute_batch("CREATE TABLE settings (v INTEGER); INSERT INTO settings VALUES (1);") .unwrap(); let taken = chrono::Utc::now(); + let backups = dir.path().join(BACKUP_DIR); - let published = write_backup(&database, taken).unwrap(); + let first = write_backup(&database, taken).unwrap(); + assert_eq!(first, backups.join(backup_file_name(taken))); assert_eq!( - published, - dir.path().join(BACKUP_DIR).join(backup_file_name(taken)) + std::fs::read(&first).unwrap(), + std::fs::read(&database).unwrap() ); + assert!(crate::utils::backup_prune::usable_snapshot(&first)); + + // Same second twice more: an unrelated file already holds the next name. + std::fs::write(&first, b"earlier backup").unwrap(); + let taken_next = backups.join(backup_file_name(taken + chrono::TimeDelta::seconds(1))); + std::fs::write(&taken_next, b"another earlier backup").unwrap(); + let second = write_backup(&database, taken).unwrap(); + assert_eq!( - std::fs::read(&published).unwrap(), + second, + backups.join(backup_file_name(taken + chrono::TimeDelta::seconds(2))) + ); + assert!(backup_timestamp(&second.file_name().unwrap().to_string_lossy()).is_some()); + assert_eq!( + std::fs::read(&second).unwrap(), std::fs::read(&database).unwrap() ); - assert!(crate::utils::backup_prune::usable_snapshot(&published)); + assert_eq!(std::fs::read(&first).unwrap(), b"earlier backup"); + assert_eq!( + std::fs::read(&taken_next).unwrap(), + b"another earlier backup" + ); + let mut entries: Vec<_> = std::fs::read_dir(&backups) + .unwrap() + .map(|entry| entry.unwrap().path()) + .collect(); + entries.sort(); + let mut expected = vec![first, taken_next, second]; + expected.sort(); + assert_eq!(entries, expected, "no temporary copy is left"); + } + + /// If every name in the retry window is taken, the copy fails cleanly: nothing is + /// overwritten and no temporary copy is left. + #[test] + fn legacy_backup_gives_up_after_the_retry_window() { + let dir = tempfile::tempdir().unwrap(); + let database = dir.path().join(LEGACY_DATABASE); + std::fs::write(&database, b"database").unwrap(); + let backups = dir.path().join(BACKUP_DIR); + std::fs::create_dir_all(&backups).unwrap(); + let taken = chrono::Utc::now(); + let seconds = i64::try_from(MAX_NAME_ATTEMPTS).unwrap(); + for offset in 0..seconds { + let name = backup_file_name(taken + chrono::TimeDelta::seconds(offset)); + std::fs::write(backups.join(name), b"existing").unwrap(); + } + let error = write_backup(&database, taken).unwrap_err(); + assert_eq!(error.kind(), std::io::ErrorKind::AlreadyExists); - let entries: Vec<_> = std::fs::read_dir(dir.path().join(BACKUP_DIR)) + let entries: Vec<_> = std::fs::read_dir(&backups) .unwrap() - .map(|entry| entry.unwrap().file_name()) + .map(|entry| entry.unwrap().path()) .collect(); - assert_eq!(entries, [published.file_name().unwrap().to_owned()]); + assert_eq!( + entries.len(), + MAX_NAME_ATTEMPTS, + "no temporary copy is left" + ); + assert!( + entries + .iter() + .all(|path| std::fs::read(path).unwrap() == b"existing") + ); } #[test] From 5ff47a749d48f0eea971419db951618dc66e713a Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Wed, 30 Sep 2026 13:32:14 +0000 Subject: [PATCH 27/30] refactor(storage): drop the dead legacy data.db backup write path Every production boot path (GUI `AppState::boot_inputs`, det-cli/headless `mcp::server`) opens an existing `data.db` read-only and calls `Database::initialize` only for a fresh file, so `backup_db` and `legacy_backups::write_backup` never ran outside tests. Remove `backup_db`, `write_backup` with its collision retry, the `.pending` unpublished-copy scheme (`is_pending_backup_name`, `sweep_pending`, its retention call and the chokepoint allow-list branch) and their tests. The `.pending` scheme never shipped, so no user has such files to sweep. Time-based retention of existing `backups/data_backup_*.db` files left by older versions stays, including the name grammar, the usability floor and the chokepoint allow-list for those names. The legacy migration ladder is kept for the migration-fixture tests. Co-Authored-By: Claude Opus 5.5 --- .../2026-09-10-platform-pin/upgrade-notes.md | 5 +- src/context/wallet_lifecycle/retention.rs | 41 --- src/database/initialization.rs | 16 +- src/database/legacy_backups.rs | 275 +----------------- src/utils/file_deletion.rs | 3 +- src/utils/file_deletion/tests.rs | 10 +- 6 files changed, 15 insertions(+), 335 deletions(-) diff --git a/docs/ai-design/2026-09-10-platform-pin/upgrade-notes.md b/docs/ai-design/2026-09-10-platform-pin/upgrade-notes.md index 18266731a..728642cc7 100644 --- a/docs/ai-design/2026-09-10-platform-pin/upgrade-notes.md +++ b/docs/ai-design/2026-09-10-platform-pin/upgrade-notes.md @@ -78,8 +78,9 @@ kept as well. If no backup is usable, the newest file is kept anyway. A backup dated in the future does not displace one dated in the past, but only until real time reaches that date; the migration order is the only clock-proof signal. A copy that cannot be read, or has a non-empty rollback journal beside it, is never -deleted automatically. Legacy `data.db` copies are written to a temporary file, -synced and renamed, so a crash cannot leave a torn copy under a backup name. +deleted automatically. This version never writes new legacy `data.db` copies: it +opens an existing `data.db` read-only, so only copies earlier versions left behind +are expired. Clearing a network's data deletes that network's wallet database backups, because they copy its wallet and identity history; backups of the shared `det-app.sqlite` also hold other networks' data and stay under retention. Removing a single wallet diff --git a/src/context/wallet_lifecycle/retention.rs b/src/context/wallet_lifecycle/retention.rs index ceb2e61a9..085134c0b 100644 --- a/src/context/wallet_lifecycle/retention.rs +++ b/src/context/wallet_lifecycle/retention.rs @@ -35,11 +35,6 @@ impl AppContext { /// so the pass is safe for databases this process has not opened. A database /// another context is opening or upgrading is skipped until the next pass. /// - /// Every policy, including keeping backups forever, also removes legacy copies a - /// crash left unpublished (see - /// [`sweep_pending`](crate::database::legacy_backups::sweep_pending)); those are not - /// counted as deleted backups. - /// /// Attempts every location and reports the first failure. Nothing is deleted when the /// policy cannot be read. pub(crate) fn prune_expired_upgrade_backups(&self) -> BackupPruneReport { @@ -56,17 +51,6 @@ impl AppContext { return report; } }; - // Crash-left legacy copies were never published backups, so every policy - // removes them. - match crate::database::legacy_backups::sweep_pending(self.data_dir(), now) { - Ok(0) => {} - Ok(swept) => tracing::info!(swept, "Removed unfinished legacy backup copies"), - Err(source) => { - report - .failure - .get_or_insert(TaskError::UpgradeBackupCleanup { source }); - } - } let Some(max_age) = retention.max_age() else { return report; }; @@ -196,31 +180,6 @@ mod tests { assert!(fixture.fresh.iter().all(|path| path.exists())); } - /// A legacy copy left unpublished by a crash is removed by every retention pass, - /// whatever the policy, once it is clearly not a copy still being written. It - /// is not a backup, so it is not counted as a deleted one. - #[test] - fn crash_left_legacy_copies_are_swept_under_any_policy() { - for policy in [BackupRetention::KeepForever, BackupRetention::default()] { - let tmp = tempfile::tempdir().unwrap(); - let ctx = test_app_context(tmp.path()); - ctx.set_backup_retention(policy).unwrap(); - let backups = ctx.data_dir().join("backups"); - let crashed = backups.join("data_backup_20200101_000000.db.a1B2c3.pending"); - let in_progress = backups.join("data_backup_20200102_000000.db.Z9y8X7.pending"); - write_aged(&crashed, DAY * 2); - write_aged(&in_progress, Duration::from_secs(5)); - - assert_eq!(pruned(&ctx), 0, "{policy:?}"); - - assert!(!crashed.exists(), "{policy:?}: a crash-left copy is swept"); - assert!( - in_progress.exists(), - "{policy:?}: a recent copy may still be written" - ); - } - } - #[test] fn keep_forever_deletes_nothing() { let tmp = tempfile::tempdir().unwrap(); diff --git a/src/database/initialization.rs b/src/database/initialization.rs index bf47bff83..5d6f1a410 100644 --- a/src/database/initialization.rs +++ b/src/database/initialization.rs @@ -1,5 +1,4 @@ use crate::database::Database; -use chrono::Utc; use rusqlite::{Connection, params}; use std::path::Path; @@ -211,7 +210,9 @@ impl Database { let current_version = self.db_schema_version()?; if current_version != DEFAULT_DB_VERSION { - self.backup_db(db_file_path)?; + // No backup is taken: production boot paths open an existing + // `data.db` read-only and call `initialize` only for a fresh file, + // so this ladder runs on test fixtures, never on user data. // Migrations may need to read `.env` from the data directory // (see v34 for one such arm). The DB file typically lives at // `/`, so its parent is the data dir. @@ -705,17 +706,6 @@ impl Database { u16::try_from(version).map_err(|_| rusqlite::Error::IntegralValueOutOfRange(0, version)) } - /// Backs up the existing database with a unique timestamped filename in backups directory. - fn backup_db(&self, db_file_path: &Path) -> rusqlite::Result<()> { - if db_file_path.exists() { - let backup_path = super::legacy_backups::write_backup(db_file_path, Utc::now()) - .map_err(|e| rusqlite::Error::ToSqlConversionFailure(e.into()))?; - tracing::info!("Old database backed up to {:?}", backup_path); - } - - Ok(()) - } - /// Creates all required tables with indexes if they don't already exist. /// /// `include_legacy` controls whether the wallet-family tables diff --git a/src/database/legacy_backups.rs b/src/database/legacy_backups.rs index bc97eba6f..9b569cbfd 100644 --- a/src/database/legacy_backups.rs +++ b/src/database/legacy_backups.rs @@ -1,7 +1,8 @@ //! Legacy `data.db` migration backups (`backups/data_backup_.db`). //! -//! The legacy migration ladder copies `data.db` before upgrading it. These copies are -//! subject to the same time-based retention as the storage-upgrade backups. +//! Earlier app versions copied `data.db` here before upgrading it; this version opens an +//! existing `data.db` read-only and never writes new copies. The copies already on disk +//! are subject to the same time-based retention as the storage-upgrade backups. use std::path::{Path, PathBuf}; use std::time::{Duration, SystemTime}; @@ -14,63 +15,14 @@ const PREFIX: &str = "data_backup_"; const SUFFIX: &str = ".db"; /// UTC timestamp format embedded in the backup name. const TIMESTAMP_FORMAT: &str = "%Y%m%d_%H%M%S"; -/// How many consecutive one-second names [`write_backup`] tries before giving up. -const MAX_NAME_ATTEMPTS: usize = 60; -/// Suffix of the unpublished copy [`write_backup`] writes before renaming it. -const PENDING_SUFFIX: &str = ".pending"; -/// How old an unpublished copy must be before it counts as left behind by a crash -/// rather than still being written. Copying `data.db` takes seconds. -const PENDING_GRACE: Duration = Duration::from_secs(24 * 60 * 60); -/// File name of a legacy backup taken at `timestamp` (UTC). +/// File name of a legacy backup taken at `timestamp` (UTC). Tests only: this version +/// never writes legacy backups, it only recognises and expires existing ones. +#[cfg(test)] pub(crate) fn backup_file_name(timestamp: chrono::DateTime) -> String { format!("{PREFIX}{}{SUFFIX}", timestamp.format(TIMESTAMP_FORMAT)) } -/// Copy `database` to `/backups/` under the legacy name for `taken`. -/// -/// The copy goes to a unique `.pending` file first, is synced, and only then is -/// renamed to its final name, after which the directory is synced. A crash or power -/// loss therefore never leaves a torn copy under a backup name, where it could pass -/// as a complete backup. An existing backup is never replaced: when the name for -/// `taken` is already used (names have one-second resolution), the next free second -/// is used instead, so a collision never fails the copy. -pub(crate) fn write_backup( - database: &Path, - taken: chrono::DateTime, -) -> std::io::Result { - let directory = database - .parent() - .ok_or(std::io::ErrorKind::InvalidInput)? - .join(BACKUP_DIR); - std::fs::create_dir_all(&directory)?; - let name = backup_file_name(taken); - // Not routed through `delete_file`: on failure the temp file removes only the - // uniquely named `.pending` file it created itself. - let mut pending = tempfile::Builder::new() - .prefix(&format!("{name}.")) - .suffix(PENDING_SUFFIX) - .tempfile_in(&directory)?; - std::fs::copy(database, pending.path())?; - pending.as_file().sync_all()?; - let mut stamp = taken; - for _ in 0..MAX_NAME_ATTEMPTS { - let published = directory.join(backup_file_name(stamp)); - match pending.persist_noclobber(&published) { - Ok(_) => { - crate::utils::backup_prune::sync_directory(&directory)?; - return Ok(published); - } - Err(error) if error.error.kind() == std::io::ErrorKind::AlreadyExists => { - pending = error.file; - stamp += chrono::TimeDelta::seconds(1); - } - Err(error) => return Err(error.error), - } - } - Err(std::io::ErrorKind::AlreadyExists.into()) -} - /// The UTC creation time embedded in a legacy backup name, or `None` for any other /// file. Also the deletion chokepoint's definition of a legacy backup name. pub(crate) fn backup_timestamp(name: &str) -> Option { @@ -79,19 +31,6 @@ pub(crate) fn backup_timestamp(name: &str) -> Option { Some(parsed.and_utc().into()) } -/// Whether `name` is an unpublished copy written by [`write_backup`]: -/// `..pending`. Also the deletion -/// chokepoint's definition of such a copy. -pub(crate) fn is_pending_backup_name(name: &str) -> bool { - name.strip_suffix(PENDING_SUFFIX) - .and_then(|rest| rest.rsplit_once('.')) - .is_some_and(|(published, random)| { - !random.is_empty() - && random.bytes().all(|b| b.is_ascii_alphanumeric()) - && backup_timestamp(published).is_some() - }) -} - /// `/backups` if it exists; an error if it is a symlink or not a directory. fn backup_directory(data_dir: &Path) -> std::io::Result> { let directory = data_dir.join(BACKUP_DIR); @@ -105,65 +44,6 @@ fn backup_directory(data_dir: &Path) -> std::io::Result> { } } -/// Delete unpublished copies [`write_backup`] left behind when the app stopped -/// mid-copy. -/// -/// Only regular files matching [`is_pending_backup_name`] whose modification time is -/// more than a day before `now` are removed, so a copy another process is still -/// writing is left alone; one dated in the future is kept. They never held a -/// published backup, so every retention policy removes them. Deletion goes through -/// the chokepoint. Attempts every file and returns the first failure; `Ok` carries -/// the number removed. -pub(crate) fn sweep_pending(data_dir: &Path, now: SystemTime) -> std::io::Result { - let Some(directory) = backup_directory(data_dir)? else { - return Ok(0); - }; - let database = data_dir.join(LEGACY_DATABASE); - let mut first_error = None; - let mut removed = 0; - for entry in std::fs::read_dir(&directory)? { - let entry = entry?; - if !entry - .file_name() - .to_str() - .is_some_and(is_pending_backup_name) - { - continue; - } - let outcome = match abandoned(&entry.path(), now) { - Ok(true) => crate::utils::file_deletion::delete_file( - &entry.path(), - crate::utils::file_deletion::DeletionIntent::Backup { - database: &database, - }, - ), - Ok(false) => continue, - Err(error) => Err(error), - }; - match outcome { - Ok(()) => removed += 1, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(error) => { - first_error.get_or_insert(error); - } - } - } - if removed > 0 - && let Err(error) = crate::utils::backup_prune::sync_directory(&directory) - { - first_error.get_or_insert(error); - } - first_error.map_or(Ok(removed), Err) -} - -/// Whether the unpublished copy at `path` is a regular file last written more than -/// [`PENDING_GRACE`] before `now`. -fn abandoned(path: &Path, now: SystemTime) -> std::io::Result { - let metadata = std::fs::symlink_metadata(path)?; - Ok(metadata.is_file() - && crate::model::backup_retention::backup_expired(metadata.modified()?, now, PENDING_GRACE)) -} - /// Delete legacy backups in `/backups` older than `max_age`, always /// keeping the newest usable one. /// @@ -225,149 +105,6 @@ mod tests { .unwrap(); } - /// The published copy is complete and no temporary file is left. A backup of the - /// same second is never overwritten, and never blocks the new copy: it is - /// published under the next free second, still a legacy backup name. - #[test] - fn legacy_backup_is_published_complete_and_never_overwrites() { - let dir = tempfile::tempdir().unwrap(); - let database = dir.path().join(LEGACY_DATABASE); - rusqlite::Connection::open(&database) - .unwrap() - .execute_batch("CREATE TABLE settings (v INTEGER); INSERT INTO settings VALUES (1);") - .unwrap(); - let taken = chrono::Utc::now(); - let backups = dir.path().join(BACKUP_DIR); - - let first = write_backup(&database, taken).unwrap(); - - assert_eq!(first, backups.join(backup_file_name(taken))); - assert_eq!( - std::fs::read(&first).unwrap(), - std::fs::read(&database).unwrap() - ); - assert!(crate::utils::backup_prune::usable_snapshot(&first)); - - // Same second twice more: an unrelated file already holds the next name. - std::fs::write(&first, b"earlier backup").unwrap(); - let taken_next = backups.join(backup_file_name(taken + chrono::TimeDelta::seconds(1))); - std::fs::write(&taken_next, b"another earlier backup").unwrap(); - let second = write_backup(&database, taken).unwrap(); - - assert_eq!( - second, - backups.join(backup_file_name(taken + chrono::TimeDelta::seconds(2))) - ); - assert!(backup_timestamp(&second.file_name().unwrap().to_string_lossy()).is_some()); - assert_eq!( - std::fs::read(&second).unwrap(), - std::fs::read(&database).unwrap() - ); - assert_eq!(std::fs::read(&first).unwrap(), b"earlier backup"); - assert_eq!( - std::fs::read(&taken_next).unwrap(), - b"another earlier backup" - ); - let mut entries: Vec<_> = std::fs::read_dir(&backups) - .unwrap() - .map(|entry| entry.unwrap().path()) - .collect(); - entries.sort(); - let mut expected = vec![first, taken_next, second]; - expected.sort(); - assert_eq!(entries, expected, "no temporary copy is left"); - } - - /// If every name in the retry window is taken, the copy fails cleanly: nothing is - /// overwritten and no temporary copy is left. - #[test] - fn legacy_backup_gives_up_after_the_retry_window() { - let dir = tempfile::tempdir().unwrap(); - let database = dir.path().join(LEGACY_DATABASE); - std::fs::write(&database, b"database").unwrap(); - let backups = dir.path().join(BACKUP_DIR); - std::fs::create_dir_all(&backups).unwrap(); - let taken = chrono::Utc::now(); - let seconds = i64::try_from(MAX_NAME_ATTEMPTS).unwrap(); - for offset in 0..seconds { - let name = backup_file_name(taken + chrono::TimeDelta::seconds(offset)); - std::fs::write(backups.join(name), b"existing").unwrap(); - } - - let error = write_backup(&database, taken).unwrap_err(); - - assert_eq!(error.kind(), std::io::ErrorKind::AlreadyExists); - let entries: Vec<_> = std::fs::read_dir(&backups) - .unwrap() - .map(|entry| entry.unwrap().path()) - .collect(); - assert_eq!( - entries.len(), - MAX_NAME_ATTEMPTS, - "no temporary copy is left" - ); - assert!( - entries - .iter() - .all(|path| std::fs::read(path).unwrap() == b"existing") - ); - } - - #[test] - fn pending_copy_names_follow_write_backup_exactly() { - assert!(is_pending_backup_name( - "data_backup_20200101_000000.db.a1B2c3.pending" - )); - for other in [ - "data_backup_20200101_000000.db.pending", - "data_backup_20200101_000000.db..pending", - "data_backup_20200101_000000.db.a-1.pending", - "data_backup_20200101_000000.db.abc.pending-journal", - "data_backup_latest.db.abc123.pending", - "notes.db.abc123.pending", - "data.db.abc123.pending", - "data_backup_20200101_000000.db", - ] { - assert!(!is_pending_backup_name(other), "{other}"); - } - } - - /// Only old regular files with the exact temporary name are swept, through the - /// chokepoint; lookalikes, directories, symlinks and published backups stay. - #[test] - fn sweep_removes_only_abandoned_pending_copies() { - let dir = tempfile::tempdir().unwrap(); - let backups = dir.path().join(BACKUP_DIR); - std::fs::create_dir_all(&backups).unwrap(); - let now = SystemTime::now(); - let write = |name: &str, age: Duration| { - let path = backups.join(name); - std::fs::write(&path, b"partial").unwrap(); - set_mtime(&path, now - age); - path - }; - let abandoned = write("data_backup_20200101_000000.db.a1B2c3.pending", DAY * 2); - let fresh = write("data_backup_20200102_000000.db.Z9y8X7.pending", DAY / 24); - let published = write("data_backup_20200101_000000.db", DAY * 400); - let lookalike = write("data_backup_20200101_000000.db.pending", DAY * 400); - let unrelated = write("notes.db.abc123.pending", DAY * 400); - let directory = backups.join("data_backup_20200103_000000.db.dir123.pending"); - std::fs::create_dir(&directory).unwrap(); - - assert_eq!(sweep_pending(dir.path(), now).unwrap(), 1); - - assert!(!abandoned.exists()); - for kept in [&fresh, &published, &lookalike, &unrelated, &directory] { - assert!(kept.exists(), "{}", kept.display()); - } - assert_eq!(sweep_pending(dir.path(), now).unwrap(), 0); - assert_eq!( - sweep_pending(tempfile::tempdir().unwrap().path(), now).unwrap(), - 0, - "no backups directory" - ); - } - #[test] fn legacy_backup_names_round_trip() { let taken = chrono::DateTime::parse_from_rfc3339("2024-03-05T06:07:08Z") diff --git a/src/utils/file_deletion.rs b/src/utils/file_deletion.rs index 013aad3f8..0cc0d8808 100644 --- a/src/utils/file_deletion.rs +++ b/src/utils/file_deletion.rs @@ -379,8 +379,7 @@ fn in_scope(located: &Path, name: &str, intent: DeletionIntent<'_>) -> bool { .is_some()) || (legacy && is(&directory.join(crate::database::legacy_backups::BACKUP_DIR)) - && (crate::database::legacy_backups::backup_timestamp(name).is_some() - || crate::database::legacy_backups::is_pending_backup_name(name))) + && crate::database::legacy_backups::backup_timestamp(name).is_some()) } DeletionIntent::NetworkClear { data_dir, network } => { let spv = crate::wallet_backend::spv_root_dir(data_dir); diff --git a/src/utils/file_deletion/tests.rs b/src/utils/file_deletion/tests.rs index 426206208..304f4c185 100644 --- a/src/utils/file_deletion/tests.rs +++ b/src/utils/file_deletion/tests.rs @@ -227,6 +227,7 @@ fn delete_file_refuses_any_hard_linked_file() { fn delete_file_refuses_targets_outside_the_intent_scope() { let dir = tempfile::tempdir().unwrap(); let database = dir.path().join("det-app.sqlite"); + let legacy = dir.path().join("data.db"); let cases = [ ( dir.path() @@ -284,9 +285,7 @@ fn delete_file_refuses_targets_outside_the_intent_scope() { ( dir.path() .join("backups/data_backup_20000101_000000.db.a1B2c3.pending"), - DeletionIntent::Backup { - database: &database, - }, + DeletionIntent::Backup { database: &legacy }, ), ( dir.path() @@ -353,11 +352,6 @@ fn delete_file_deletes_targets_in_scope() { dir.path().join("backups/data_backup_20000101_000000.db"), DeletionIntent::Backup { database: &legacy }, ), - ( - dir.path() - .join("backups/data_backup_20000101_000000.db.a1B2c3.pending"), - DeletionIntent::Backup { database: &legacy }, - ), ( dir.path() .join("backups/auto/pre-migration-det-testnet-1-to-2-20000101T000000Z.db"), From 660e4b3c5fe5a55321dd25cd5cc4e0132335a8fb Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Wed, 30 Sep 2026 13:49:13 +0000 Subject: [PATCH 28/30] test(storage): trim duplicated backup-retention tests and docs Keep each retention/floor/classification invariant proven once at the right level instead of re-asserting it at three or four layers: - drop engine/legacy/retention floor and clock tests already covered by backup_prune unit tests; keep the unreadable-copy case - one table-driven StorageFailure test plus one mapping test per consumer - merge overlapping backup-retention form tests - add a bridge_backup_path test helper for hand-built fixture names - compute a backup's creation time in one place (backup_prune::created_at) - drop the always-true selection filter from remove_backups - trim internal docs to present state Co-Authored-By: Claude Opus 5.5 --- src/app.rs | 4 - src/backend_task/error.rs | 72 ++---- src/backend_task/identity/remove_identity.rs | 6 +- src/backend_task/mod.rs | 17 +- src/context/wallet_lifecycle/retention.rs | 56 +---- src/context/wallet_lifecycle/tests.rs | 53 +---- src/database/legacy_backups.rs | 84 +------ src/ui/network_chooser_screen.rs | 45 ++-- src/utils/backup_prune.rs | 52 ++-- src/utils/file_deletion.rs | 40 +--- src/utils/file_deletion/tests.rs | 18 -- .../platform_compatibility/engine.rs | 63 ++--- .../platform_compatibility/engine/tests.rs | 222 +++--------------- .../platform_compatibility/mod.rs | 93 ++------ .../platform_compatibility/storage_failure.rs | 64 +++-- 15 files changed, 200 insertions(+), 689 deletions(-) diff --git a/src/app.rs b/src/app.rs index 7b233af5c..491d16e74 100644 --- a/src/app.rs +++ b/src/app.rs @@ -4550,9 +4550,5 @@ mod network_chooser_routing_tests { let context = BackendTaskContext::from(&task); assert_eq!(context, BackendTaskContext::UpdateBackupRetention); assert!(network_chooser_owns_task(&context)); - assert!(network_chooser_owns_task( - &BackendTaskContext::ClearNetworkDatabase - )); - assert!(!network_chooser_owns_task(&BackendTaskContext::Other)); } } diff --git a/src/backend_task/error.rs b/src/backend_task/error.rs index e9b751692..4b3fada71 100644 --- a/src/backend_task/error.rs +++ b/src/backend_task/error.rs @@ -109,11 +109,7 @@ impl std::error::Error for BackendTaskJoinError {} /// Dash Core RPC error code: wallet file not specified (multi-wallet node). const RPC_WALLET_NOT_SPECIFIED: i32 = -19; -/// Some expired upgrade backups remain. Retention runs again whenever wallet data -/// is opened (start, network switch) and whenever the setting is saved. -pub(crate) const UPGRADE_BACKUP_CLEANUP_INCOMPLETE: &str = "Some old upgrade backups could not be deleted. The app tries again the next time it opens your wallet data."; - -/// [`UPGRADE_BACKUP_CLEANUP_INCOMPLETE`] after a retention setting save. +/// Some expired upgrade backups remain after a retention setting save. pub(crate) const BACKUP_RETENTION_SAVED_CLEANUP_INCOMPLETE: &str = "Your backup retention setting was saved, but some old upgrade backups could not be deleted. The app tries again the next time it opens your wallet data."; /// Shown whenever another window or session holds the wallet database. @@ -1205,7 +1201,9 @@ pub enum TaskError { }, /// Expired upgrade backups could not all be deleted. - #[error("{}", UPGRADE_BACKUP_CLEANUP_INCOMPLETE)] + #[error( + "Some old upgrade backups could not be deleted. The app tries again the next time it opens your wallet data." + )] UpgradeBackupCleanup { #[source] source: std::io::Error, @@ -3785,23 +3783,6 @@ mod tests { use dash_sdk::dpp::identity::Purpose; use dash_sdk::platform::Identifier; - /// Both cleanup messages promise the same, truthful retry. - #[test] - fn upgrade_backup_cleanup_messages_share_the_retry_promise() { - let error = TaskError::UpgradeBackupCleanup { - source: std::io::Error::other("permission denied"), - }; - assert_eq!(error.to_string(), UPGRADE_BACKUP_CLEANUP_INCOMPLETE); - let (_, retry) = UPGRADE_BACKUP_CLEANUP_INCOMPLETE - .split_once(". ") - .expect("two sentences"); - assert_eq!( - retry, - "The app tries again the next time it opens your wallet data." - ); - assert!(BACKUP_RETENTION_SAVED_CLEANUP_INCOMPLETE.ends_with(retry)); - } - const DAPI_EXHAUSTED_MESSAGE: &str = "All Dash network servers are temporarily unreachable. Please wait a minute and retry."; @@ -6169,44 +6150,31 @@ mod tests { .expect_err("wrapped SQLite failure") } - /// Running out of disk space, an OS I/O failure or memory exhaustion - /// during a migration is recoverable by freeing the resource and - /// retrying. It must keep the generic, non-terminal storage copy instead - /// of the terminal move-the-databases-aside guidance. + /// A migration that ran out of a resource keeps the generic, retryable storage + /// copy; access failures never fix themselves and stay out of that class. #[test] fn resource_migration_errors_map_to_retryable_wallet_storage() { - for code in [ - rusqlite::ffi::SQLITE_FULL, - rusqlite::ffi::SQLITE_IOERR, - rusqlite::ffi::SQLITE_NOMEM, + use rusqlite::ffi; + for (code, retryable) in [ + (ffi::SQLITE_FULL, true), + (ffi::SQLITE_IOERR, true), + (ffi::SQLITE_NOMEM, true), + (ffi::SQLITE_PERM, false), + (ffi::SQLITE_READONLY, false), ] { let upstream = platform_wallet_storage::WalletStorageError::Migration( resource_migration_error(code), ); let err = TaskError::from_wallet_storage_open_error(upstream); - assert!( + assert_eq!( matches!(err, TaskError::WalletStorage { .. }), - "Expected WalletStorage for SQLite code {code}, got: {err:?}" + retryable, + "SQLite code {code}: {err:?}" ); - assert!( - !crate::backend_task::is_terminal_storage_open_error(&err), - "SQLite code {code} must stay retryable" - ); - } - } - - /// Access failures are not fixed by freeing space or retrying, so they - /// stay out of the retryable resource class. - #[test] - fn access_migration_errors_do_not_map_to_retryable_wallet_storage() { - for code in [rusqlite::ffi::SQLITE_PERM, rusqlite::ffi::SQLITE_READONLY] { - let upstream = platform_wallet_storage::WalletStorageError::Migration( - resource_migration_error(code), - ); - let err = TaskError::from_wallet_storage_open_error(upstream); - assert!( - !matches!(err, TaskError::WalletStorage { .. }), - "SQLite code {code} must not be classified as a resource failure, got: {err:?}" + assert_eq!( + crate::backend_task::is_terminal_storage_open_error(&err), + !retryable, + "SQLite code {code}" ); } } diff --git a/src/backend_task/identity/remove_identity.rs b/src/backend_task/identity/remove_identity.rs index 5d5face4c..d4ff15d7f 100644 --- a/src/backend_task/identity/remove_identity.rs +++ b/src/backend_task/identity/remove_identity.rs @@ -158,10 +158,8 @@ mod tests { staged.ctx.data_dir(), staged.ctx.network(), ); - let backup = database.with_file_name(format!( - "{}.platform-67d4ef3-backup-fixture.sqlite", - database.file_name().unwrap().to_string_lossy() - )); + let backup = + crate::wallet_backend::platform_compatibility::bridge_backup_path(&database, "fixture"); std::fs::write(&backup, b"old identity rows").unwrap(); staged diff --git a/src/backend_task/mod.rs b/src/backend_task/mod.rs index d4857ed07..d2670596a 100644 --- a/src/backend_task/mod.rs +++ b/src/backend_task/mod.rs @@ -2435,17 +2435,12 @@ mod tests { } )); use crate::wallet_backend::platform_compatibility::UpgradeError; - for source in [ - UpgradeError::Unrecognized, - UpgradeError::IdentityRoster { source: None }, - UpgradeError::Verification, - UpgradeError::TypedValidation(Box::new(std::io::Error::other("fixture"))), - ] { - assert!( - is_terminal_storage_open_error(&TaskError::PlatformDatabaseUpgrade { source }), - "a deterministic upgrade failure must surface instead of re-running" - ); - } + assert!( + is_terminal_storage_open_error(&TaskError::PlatformDatabaseUpgrade { + source: UpgradeError::Verification, + }), + "a deterministic upgrade failure must surface instead of re-running" + ); // A user-clearable upgrade failure keeps offering a retry. assert!(!is_terminal_storage_open_error( &TaskError::PlatformDatabaseUpgrade { diff --git a/src/context/wallet_lifecycle/retention.rs b/src/context/wallet_lifecycle/retention.rs index 085134c0b..87f9e1c5e 100644 --- a/src/context/wallet_lifecycle/retention.rs +++ b/src/context/wallet_lifecycle/retention.rs @@ -26,23 +26,10 @@ pub(crate) fn retained_backup_databases(data_dir: &std::path::Path) -> Vec BackupPruneReport { - self.prune_expired_upgrade_backups_at(SystemTime::now()) - } - - /// [`Self::prune_expired_upgrade_backups`] as if the clock read `now`. - pub(crate) fn prune_expired_upgrade_backups_at(&self, now: SystemTime) -> BackupPruneReport { + let now = SystemTime::now(); let mut report = BackupPruneReport::default(); let retention = match self.backup_retention() { Ok(retention) => retention, @@ -140,12 +127,7 @@ mod tests { crate::wallet_backend::app_database_path(dir), crate::wallet_backend::wallet_database_path(dir, ctx.network), ]; - let bridge = |database: &Path, suffix: &str| { - database.with_file_name(format!( - "{}.platform-67d4ef3-backup-{suffix}.sqlite", - database.file_name().unwrap().to_string_lossy() - )) - }; + let bridge = crate::wallet_backend::platform_compatibility::bridge_backup_path; let expired = vec![ bridge(&app, "old"), bridge(&wallet, "old"), @@ -180,19 +162,6 @@ mod tests { assert!(fixture.fresh.iter().all(|path| path.exists())); } - #[test] - fn keep_forever_deletes_nothing() { - let tmp = tempfile::tempdir().unwrap(); - let ctx = test_app_context(tmp.path()); - ctx.set_backup_retention(BackupRetention::KeepForever) - .unwrap(); - let fixture = backups(&ctx); - - assert_eq!(pruned(&ctx), 0); - - assert!(fixture.expired.iter().all(|path| path.exists())); - } - #[test] fn shorter_retention_deletes_more() { let tmp = tempfile::tempdir().unwrap(); @@ -243,23 +212,6 @@ mod tests { ); } - #[test] - fn newest_backup_of_each_database_survives_a_far_future_clock() { - let tmp = tempfile::tempdir().unwrap(); - let ctx = test_app_context(tmp.path()); - let fixture = backups(&ctx); - let far_future = SystemTime::now() + DAY * 365 * 100; - - let report = ctx.prune_expired_upgrade_backups_at(far_future); - assert!(report.failure.is_none(), "{:?}", report.failure); - assert_eq!(report.deleted, 3); - assert!(fixture.fresh.iter().all(|path| path.exists())); - assert!(fixture.expired.iter().all(|path| !path.exists())); - let again = ctx.prune_expired_upgrade_backups_at(far_future); - assert_eq!(again.deleted, 0, "the last backups are never deleted"); - assert!(fixture.fresh.iter().all(|path| path.exists())); - } - #[test] fn database_in_use_is_skipped_not_failed() { let tmp = tempfile::tempdir().unwrap(); diff --git a/src/context/wallet_lifecycle/tests.rs b/src/context/wallet_lifecycle/tests.rs index edcbbbc12..5c63b5e11 100644 --- a/src/context/wallet_lifecycle/tests.rs +++ b/src/context/wallet_lifecycle/tests.rs @@ -1714,53 +1714,6 @@ async fn remove_wallet_wipes_seed_envelope() { backend.shutdown().await; } -/// Upgrade backups copy whole databases, including other wallets' data, so removing -/// one wallet leaves them to time-based retention. -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn remove_wallet_keeps_upgrade_backups() { - let (ctx, sender, _tmp) = offline_testnet_context(); - ctx.ensure_wallet_backend(sender) - .await - .expect("ensure_wallet_backend should succeed offline"); - - let seed = [0xA7u8; 64]; - let wallet = crate::model::wallet::Wallet::new_from_seed(seed, Network::Testnet, None, None) - .expect("build wallet"); - let seed_hash = wallet.seed_hash(); - ctx.register_wallet(wallet, &seed, WalletOrigin::Fresh) - .expect("register wallet"); - let backend = ctx.wallet_backend().expect("backend wired"); - - let databases = [ - ctx.data_dir().join("det-app.sqlite"), - crate::wallet_backend::wallet_database_path(ctx.data_dir(), Network::Testnet), - ]; - let backups = databases.each_ref().map(|database| { - let name = database.file_name().expect("database file name"); - let backup = database.with_file_name(format!( - "{}.platform-67d4ef3-backup-fixture.sqlite", - name.to_string_lossy() - )); - std::fs::write(&backup, b"old wallet history").expect("write backup fixture"); - backup - }); - - ctx.remove_wallet(&seed_hash).expect("remove wallet"); - - for backup in &backups { - assert!( - backup.exists(), - "{} must survive wallet removal", - backup.display() - ); - } - for database in &databases { - assert!(database.exists(), "{} must be kept", database.display()); - } - - backend.shutdown().await; -} - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn remove_wallet_warns_when_local_secret_wipe_fails() { let (ctx, sender, _tmp) = offline_testnet_context(); @@ -7358,10 +7311,8 @@ fn named_aged_upgrade_backup( suffix: &str, days: u64, ) -> std::path::PathBuf { - let backup = database.with_file_name(format!( - "{}.platform-67d4ef3-backup-{suffix}.sqlite", - database.file_name().unwrap().to_string_lossy() - )); + let backup = + crate::wallet_backend::platform_compatibility::bridge_backup_path(database, suffix); std::fs::write(&backup, b"old wallet history").unwrap(); std::fs::File::options() .write(true) diff --git a/src/database/legacy_backups.rs b/src/database/legacy_backups.rs index 9b569cbfd..66700618b 100644 --- a/src/database/legacy_backups.rs +++ b/src/database/legacy_backups.rs @@ -44,17 +44,8 @@ fn backup_directory(data_dir: &Path) -> std::io::Result> { } } -/// Delete legacy backups in `/backups` older than `max_age`, always -/// keeping the newest usable one. -/// -/// Deletion goes through the [`delete_file`](crate::utils::file_deletion::delete_file) -/// chokepoint, which refuses live databases, aliases and hard links. -/// Only regular files with the exact legacy name are candidates; anything else, -/// including symlinks and subdirectories, is left alone. Age is measured from the -/// later of the name timestamp and the modification time, so a reset or skewed signal -/// only delays deletion. A `backups` path that is a symlink or not a directory is -/// refused. Expiry itself is [`prune_expired`](crate::utils::backup_prune::prune_expired); -/// `Ok` carries the number of backups this call deleted. +/// [`prune_expired`](crate::utils::backup_prune::prune_expired) over the regular files +/// with a legacy backup name in `/backups`; returns the number deleted. pub(crate) fn prune_expired( data_dir: &Path, max_age: Duration, @@ -69,14 +60,13 @@ pub(crate) fn prune_expired( let Some(named) = entry.file_name().to_str().and_then(backup_timestamp) else { continue; }; - let metadata = match std::fs::symlink_metadata(entry.path()) { - Ok(metadata) if !metadata.is_file() => continue, - other => other, - }; - let created = metadata.and_then(|metadata| Ok(named.max(metadata.modified()?))); + let path = entry.path(); + if std::fs::symlink_metadata(&path).is_ok_and(|metadata| !metadata.is_file()) { + continue; + } candidates.push(crate::utils::backup_prune::Candidate { - path: entry.path(), - created, + created: crate::utils::backup_prune::created_at(&path, Some(named)), + path, sequence: None, }); } @@ -193,12 +183,8 @@ mod tests { let backups = dir.path().join(BACKUP_DIR); std::fs::create_dir_all(&backups).unwrap(); let long_ago = SystemTime::now() - DAY * 400; - for second in 0..200 { - let path = backups.join(format!( - "data_backup_20000101_00{:02}{:02}.db", - second / 60, - second % 60 - )); + for second in 0..10 { + let path = backups.join(format!("data_backup_20000101_0000{second:02}.db")); std::fs::write(&path, b"old").unwrap(); set_mtime(&path, long_ago); } @@ -218,58 +204,10 @@ mod tests { .into_iter() .map(|result| result.expect("a concurrently deleted backup is not a failure")) .sum(); - assert_eq!( - deleted, 199, - "each backup is counted once; the newest stays" - ); + assert_eq!(deleted, 9, "each backup is counted once; the newest stays"); assert_eq!(std::fs::read_dir(&backups).unwrap().count(), 1); } - /// A clock far in the future must never delete the last legacy backup. - #[test] - fn newest_legacy_backup_survives_a_far_future_clock() { - let dir = tempfile::tempdir().unwrap(); - let backups = dir.path().join(BACKUP_DIR); - std::fs::create_dir_all(&backups).unwrap(); - let older = backups.join("data_backup_20200101_000000.db"); - let newest = backups.join("data_backup_20210101_000000.db"); - for (path, age) in [(&older, DAY * 30), (&newest, DAY * 2)] { - std::fs::write(path, b"backup").unwrap(); - set_mtime(path, SystemTime::now() - age); - } - let far_future = SystemTime::now() + DAY * 365 * 100; - - assert_eq!(prune_expired(dir.path(), DAY, far_future).unwrap(), 1); - assert!(!older.exists()); - assert!(newest.exists(), "the newest backup is the floor"); - assert_eq!(prune_expired(dir.path(), DAY, far_future).unwrap(), 0); - assert!(newest.exists()); - } - - /// An interrupted legacy copy is never the floor: the newest complete backup - /// survives, and the expired interrupted copy is deleted. - #[test] - fn newest_legacy_backup_floor_must_be_a_complete_database() { - let dir = tempfile::tempdir().unwrap(); - let backups = dir.path().join(BACKUP_DIR); - std::fs::create_dir_all(&backups).unwrap(); - let complete = backups.join("data_backup_20200101_000000.db"); - rusqlite::Connection::open(&complete) - .unwrap() - .execute_batch("CREATE TABLE settings (v INTEGER); INSERT INTO settings VALUES (1);") - .unwrap(); - let bytes = std::fs::read(&complete).unwrap(); - let truncated = backups.join("data_backup_20200601_000000.db"); - std::fs::write(&truncated, &bytes[..bytes.len() / 2]).unwrap(); - let now = SystemTime::now(); - set_mtime(&complete, now - DAY * 200); - set_mtime(&truncated, now - DAY * 100); - - assert_eq!(prune_expired(dir.path(), DAY * 90, now).unwrap(), 1); - assert!(complete.exists(), "the only complete backup is the floor"); - assert!(!truncated.exists()); - } - #[cfg(unix)] #[test] fn legacy_backup_symlinks_are_never_followed() { diff --git a/src/ui/network_chooser_screen.rs b/src/ui/network_chooser_screen.rs index 6438db0f7..eaa08811d 100644 --- a/src/ui/network_chooser_screen.rs +++ b/src/ui/network_chooser_screen.rs @@ -1852,12 +1852,6 @@ mod tests { } } - #[test] - fn stricter_retention_confirmation_reads_naturally() { - assert!(stricter_retention_confirmation(1).contains("older than one day will")); - assert!(stricter_retention_confirmation(30).contains("older than 30 days will")); - } - #[test] fn lost_backup_retention_result_does_not_wedge_the_form() { let mut form = BackupRetentionForm::new(Some(BackupRetention::default())); @@ -1873,16 +1867,16 @@ mod tests { form.save_succeeded(BackupRetention::DeleteAfterDays(200)); assert_eq!(form.saved, Some(BackupRetention::DeleteAfterDays(200))); assert_eq!(form.days, 200); - } - #[test] - fn reload_picks_up_a_setting_that_became_readable() { + // A setting that became readable is picked up. let mut form = BackupRetentionForm::new(None); form.reload(Some(BackupRetention::KeepForever)); assert_eq!(form.saved, Some(BackupRetention::KeepForever)); assert!(!form.enabled); } + /// Looser policies save without confirmation. The saved policy changes only when + /// the backend confirms it; a failed save restores the previous inputs. #[test] fn backup_retention_form_saves_looser_policies_without_confirmation() { let mut form = BackupRetentionForm::new(Some(BackupRetention::default())); @@ -1897,6 +1891,17 @@ mod tests { ); assert!(form.confirmation.is_none()); assert_eq!(step(&mut form, false), None, "a save is in flight"); + assert_eq!(form.saved, Some(BackupRetention::default())); + form.save_failed(); + assert_eq!((form.enabled, form.days), (true, 90)); + assert_eq!(step(&mut form, false), None, "no resend loop"); + + form.days = 120; + assert_eq!( + step(&mut form, false), + Some(BackupRetention::DeleteAfterDays(120)), + "the user can retry" + ); form.save_succeeded(BackupRetention::DeleteAfterDays(120)); form.enabled = false; @@ -1991,28 +1996,6 @@ mod tests { assert_eq!(saved_retention(form.take_task()), None, "emitted once"); } - /// The saved policy changes only when the backend confirms it; a failed save - /// restores the previous inputs instead of showing an unsaved policy. - #[test] - fn failed_backup_retention_save_keeps_the_saved_policy() { - let mut form = BackupRetentionForm::new(Some(BackupRetention::default())); - form.days = 200; - assert!(step(&mut form, false).is_some()); - assert_eq!(form.saved, Some(BackupRetention::default())); - - form.save_failed(); - assert_eq!(form.saved, Some(BackupRetention::default())); - assert_eq!((form.enabled, form.days), (true, 90)); - assert_eq!(step(&mut form, false), None, "no resend loop"); - - form.days = 200; - assert_eq!( - step(&mut form, false), - Some(BackupRetention::DeleteAfterDays(200)), - "the user can retry" - ); - } - /// An unreadable setting shows as unchecked (nothing is deleted) and can be /// repaired by saving explicitly. #[test] diff --git a/src/utils/backup_prune.rs b/src/utils/backup_prune.rs index 3054d2bde..d79f41959 100644 --- a/src/utils/backup_prune.rs +++ b/src/utils/backup_prune.rs @@ -1,31 +1,11 @@ -//! Time-based expiry shared by every upgrade-backup location. +//! Time-based expiry shared by every upgrade-backup location (bridge, upstream +//! `pre-migration-*`, legacy `data.db`). //! -//! One algorithm for the bridge, upstream `pre-migration-*` and legacy `data.db` -//! backups: delete those older than the retention period, except the newest usable -//! backup of the database, which is kept whatever its age. An interrupted copy can -//! therefore never delete the last recovery copy. -//! -//! "Newest" is judged by every ordering signal a backup carries, and the newest by -//! each signal is kept: -//! - its creation time, preferring backups not dated in the future. A copy taken -//! while the clock ran ahead does not displace a genuinely newer one, but only -//! while its date is still in the future: once real time catches up, it counts as -//! the newest again and the newer copy expires normally. No clock-based order can -//! do better without remembering past passes; -//! - its position in the database's upgrade history (upstream snapshots name the -//! migration they precede), which no clock can skew. -//! -//! Usability is a cheap structural check ([`check_snapshot`]) with three outcomes: -//! - usable: only these can be the newest backup that is kept; -//! - unusable (empty, truncated, header-only or not SQLite): expires like any other; -//! - undetermined (the file cannot be read, or a non-empty rollback journal lies -//! beside it, so it may hold a torn write): never deleted automatically, and never -//! the reason an older usable copy stops being kept. A read error is also reported, -//! so the pass is retried; a leftover journal keeps the copy until the user removes -//! the journal. -//! -//! When no backup of a database is usable, the newest files are kept anyway, so a -//! false negative of the check can never delete every backup. +//! Backups older than the retention period are deleted, except the newest usable one +//! of each database by creation time (preferring those not dated in the future) and by +//! upgrade-history position. Copies whose usability is undetermined (unreadable, or +//! with a non-empty rollback journal beside them) are never deleted. With no usable +//! backup, the newest files are kept anyway. use super::file_deletion::{DeletionIntent, delete_file}; use std::collections::BTreeSet; @@ -68,15 +48,8 @@ enum Usability { Undetermined, } -/// Delete every candidate older than `max_age` at `now`, except the newest usable one -/// by each ordering signal and any whose usability is undetermined (see the module -/// docs). -/// -/// All candidates must be backups of the database named by `intent`. A candidate -/// whose creation time or contents cannot be read is kept and reported. A candidate -/// that is already gone (e.g. pruned concurrently from another network's context) -/// counts as handled. Attempts every candidate, syncs the directories it deleted from, -/// and returns the first failure; `Ok` carries the number of backups deleted. +/// Delete expired `candidates` (all backups of `intent`'s database) as the module docs +/// describe. Attempts every one and returns the first failure, or the number deleted. pub(crate) fn prune_expired( candidates: Vec, now: SystemTime, @@ -170,6 +143,13 @@ fn floors(dated: &[Dated], now: SystemTime) -> BTreeSet { by_time.into_iter().chain(by_sequence).collect() } +/// When `backup` was created: the later of its modification time and `named` (the +/// timestamp in its name), so a reset or skewed signal only delays deletion. +pub(crate) fn created_at(backup: &Path, named: Option) -> std::io::Result { + let modified = std::fs::symlink_metadata(backup)?.modified()?; + Ok(named.map_or(modified, |named| named.max(modified))) +} + /// Length of the SQLite database header. const SQLITE_HEADER_LEN: usize = 100; /// Magic string every SQLite 3 database file starts with. diff --git a/src/utils/file_deletion.rs b/src/utils/file_deletion.rs index 0cc0d8808..b55a10856 100644 --- a/src/utils/file_deletion.rs +++ b/src/utils/file_deletion.rs @@ -1,34 +1,16 @@ -//! The single chokepoint for deleting files from disk. +//! The single chokepoint for deleting files from disk: every production deletion goes +//! through [`delete_file`] (or [`delete_tree`]) with a [`DeletionIntent`]. Before any +//! unlink it refuses, in order: //! -//! Every production file deletion goes through [`delete_file`] (or -//! [`delete_tree`], which unlinks each file through [`delete_file`]). A deletion -//! states its purpose as a [`DeletionIntent`], and the chokepoint enforces, in -//! this order, before anything is unlinked: +//! 1. anything but a regular file (a symlink is unlinked, never followed, only under +//! [`DeletionIntent::NetworkClear`]); +//! 2. live application files by name (app/legacy/wallet/shielded databases, vaults, +//! `.env`, `*.premigration`, with SQLite sidecars), independent of caller paths; +//! 3. the same files by canonical location and, on Unix, device/inode; +//! 4. on Unix, files with more than one hard link (on Windows, alias-prone names); +//! 5. anything outside the intent's directories and file-name grammar. //! -//! 1. the target is a regular file — never a directory, and a symlink only under -//! [`DeletionIntent::NetworkClear`], where the link itself is unlinked and its -//! target is never followed; -//! 2. a hard deny-list of live application files by **name** — `det-app.sqlite`, -//! `data.db`, every network's wallet and shielded database, `*.pwsvault`, -//! `.env`, `*.premigration`, each with its `-wal`/`-shm`/`-journal` sidecar. -//! It depends on no caller-supplied path, so a caller's matcher bug or a -//! wrong data directory can never reach these files; -//! 3. the same deny-list by **identity** — the canonical location, and on Unix -//! the device/inode, of every protected file in the intent's data directory, -//! which catches `..`, symlinked-directory and case aliases; -//! 4. on Unix, a single hard link, so no alias of another file is removed; -//! 5. the intent's scope — the directories that purpose may delete from and, -//! for backups and logs, the exact file-name grammar of that purpose. -//! -//! Per platform: Unix gets every layer. Windows has no device/inode or hard-link -//! check, so it instead refuses names that can alias another file there (8.3 -//! short names with `~`, `:` stream suffixes, a trailing `.` or space). -//! -//! The chokepoint needs no `AppContext`: each intent carries the paths it is -//! confined to, so it also serves pre-context callers such as log rotation. -//! -//! A check and the unlink that follows it are not atomic; a concurrent rename in -//! the window is out of scope. The data directory is owner-only. +//! Check and unlink are not atomic; a concurrent rename in the window is out of scope. use dash_sdk::dpp::dashcore::Network; use std::path::{Path, PathBuf}; diff --git a/src/utils/file_deletion/tests.rs b/src/utils/file_deletion/tests.rs index 304f4c185..b4c171616 100644 --- a/src/utils/file_deletion/tests.rs +++ b/src/utils/file_deletion/tests.rs @@ -590,21 +590,3 @@ fn delete_tree_never_follows_a_symlinked_directory() { assert!(!cache.exists()); assert!(outside.join("keep.dat").exists()); } - -#[test] -fn rotated_log_names_are_exact() { - let rotated = |name: &str, stem: &str| crate::logging::parse_rotated_ts(name, stem).is_some(); - assert!(rotated("det.0000000001.log", "det")); - for name in [ - "det.log", - "det..log", - "det.12a.log", - "det.-5.log", - "det.1.log.bak", - "detx.1.log", - "other.1.log", - ] { - assert!(!rotated(name, "det"), "{name}"); - } - assert!(!rotated(".1.log", "")); -} diff --git a/src/wallet_backend/platform_compatibility/engine.rs b/src/wallet_backend/platform_compatibility/engine.rs index 476139270..f78b9b876 100644 --- a/src/wallet_backend/platform_compatibility/engine.rs +++ b/src/wallet_backend/platform_compatibility/engine.rs @@ -246,6 +246,13 @@ fn backup_prefix(path: &Path) -> Option { )) } +/// The published bridge snapshot of `database` named with `suffix`; a test fixture. +#[cfg(test)] +pub(crate) fn bridge_backup_path(database: &Path, suffix: &str) -> PathBuf { + let prefix = backup_prefix(database).expect("database path has a file name"); + database.with_file_name(format!("{prefix}{suffix}.sqlite")) +} + /// Upstream snapshot names: `pre-migration---to--.db`. const UPSTREAM_PREFIX: &str = "pre-migration-"; const UPSTREAM_SUFFIX: &str = ".db"; @@ -312,12 +319,6 @@ pub(crate) fn upstream_backup_timestamp( parse_upstream_name(database, name).map(|snapshot| snapshot.taken) } -/// Retained upgrade backups of the database at `path`. -#[cfg(test)] -fn backups(path: &Path) -> std::io::Result> { - backups_in(path, Some(&default_auto_dir(path))) -} - fn default_auto_dir(path: &Path) -> PathBuf { platform_wallet_storage::default_auto_backup_dir(path) } @@ -506,13 +507,8 @@ pub(super) fn tidy_backups(path: &Path, auto_dir: Option<&Path>) -> std::io::Res tidy_backups_locked(&guard, auto_dir) } -/// Housekeeping that loses no recovery data: delete unpublished `.pending` copies left -/// by a crash, and published snapshots that are byte-identical to a newer retained -/// snapshot of the same migration (each failed open writes another copy of the -/// unchanged database). Every other snapshot is left to time-based retention. -/// -/// The scan is strict, so an unexpected candidate fails the call before anything is -/// deleted. +/// Delete crash-left `.pending` copies and published snapshots byte-identical to a newer +/// one of the same migration (each failed open writes another); the rest is left to retention. pub(super) fn tidy_backups_locked( guard: &BackupGuard, auto_dir: Option<&Path>, @@ -583,29 +579,15 @@ fn same_contents(a: &Path, b: &Path) -> std::io::Result { } } -/// Delete every retained upgrade backup of the database at `path`. -/// -/// Backups never contain vault secrets. Attempts every file and returns the first failure. -/// `Ok` means the deletions are durable: every existing backup directory is -/// synced on every call (including a retry that finds nothing left to delete), -/// so callers may retire their retry state afterwards. +/// Delete every upgrade backup of the database at `path`; attempts all and returns the +/// first failure. pub(crate) fn remove_backups(path: &Path) -> std::io::Result<()> { - remove_backups_with_sync(path, |_| true, crate::utils::backup_prune::sync_directory) + remove_backups_with_sync(path, crate::utils::backup_prune::sync_directory) } -/// Delete the upgrade backups of the database at `path` that are older than `max_age`. -/// -/// Covers both the bridge snapshots next to the database and upstream `pre-migration-*` -/// snapshots, and always keeps the newest usable one (see -/// [`prune_expired`](crate::utils::backup_prune::prune_expired)), so it is safe -/// whether or not the database currently opens. A database with no backups is -/// skipped without taking its lock, and -/// one whose lock is held (another context is opening or upgrading it) is skipped too; -/// the next pass covers it. -/// -/// Attempts every candidate and returns the first failure; a candidate that fails -/// validation is reported but never blocks removing the others. `Ok` carries the number -/// of backups deleted, and their directories are synced. +/// [`prune_expired`](crate::utils::backup_prune::prune_expired) over the bridge and +/// upstream backups of the database at `path`. Skips a database without backups or +/// whose lifecycle lock is held (the next pass covers it); returns the number deleted. pub(crate) fn prune_expired_backups( path: &Path, max_age: std::time::Duration, @@ -661,32 +643,23 @@ pub(crate) fn prune_expired_backups( } } -/// When `backup` of `database` was created: the later of its modification time and -/// the timestamp in its name, if any. Taking the later one means a reset or skewed -/// signal only delays deletion, never hastens it. fn backup_created(database: &Path, backup: &Path) -> std::io::Result { - let modified = std::fs::symlink_metadata(backup)?.modified()?; let named = backup .file_name() .and_then(|name| name.to_str()) .and_then(|name| upstream_backup_timestamp(database, name)); - Ok(named.map_or(modified, |named| named.max(modified))) + crate::utils::backup_prune::created_at(backup, named) } fn remove_backups_with_sync( path: &Path, - selected: impl Fn(&Path) -> bool, mut sync_dir: impl FnMut(&Path) -> std::io::Result<()>, ) -> std::io::Result<()> { let _guard = backup_lock(path)?; let auto_dir = default_auto_dir(path); let scan = scan_backups_in(path, Some(&auto_dir))?; - let mut first_error = scan - .rejected - .into_iter() - .find(|(backup, _)| selected(backup)) - .map(|(_, error)| error); - for backup in scan.found.into_iter().filter(|backup| selected(backup)) { + let mut first_error = scan.rejected.into_iter().next().map(|(_, error)| error); + for backup in scan.found { if let Err(error) = remove_backup(&backup, path) { first_error.get_or_insert(error); } diff --git a/src/wallet_backend/platform_compatibility/engine/tests.rs b/src/wallet_backend/platform_compatibility/engine/tests.rs index 16fdce3ad..8afa62300 100644 --- a/src/wallet_backend/platform_compatibility/engine/tests.rs +++ b/src/wallet_backend/platform_compatibility/engine/tests.rs @@ -327,14 +327,10 @@ fn platform_compatibility_backup_removal_fails_when_directory_sync_fails() { } let mut synced = Vec::new(); - let error = remove_backups_with_sync( - &path, - |_| true, - |directory| { - synced.push(directory.to_owned()); - Err(std::io::Error::other("injected directory sync failure")) - }, - ) + let error = remove_backups_with_sync(&path, |directory| { + synced.push(directory.to_owned()); + Err(std::io::Error::other("injected directory sync failure")) + }) .unwrap_err(); assert_eq!(error.kind(), std::io::ErrorKind::Other); @@ -358,23 +354,17 @@ fn platform_compatibility_backup_removal_retries_directory_sync_without_deletion .join("det-testnet.sqlite.platform-67d4ef3-backup-a1.sqlite"); std::fs::write(&sibling, b"backup").unwrap(); - remove_backups_with_sync( - &path, - |_| true, - |_| Err(std::io::Error::other("injected directory sync failure")), - ) + remove_backups_with_sync(&path, |_| { + Err(std::io::Error::other("injected directory sync failure")) + }) .unwrap_err(); assert!(!sibling.exists()); let mut synced = Vec::new(); - remove_backups_with_sync( - &path, - |_| true, - |directory| { - synced.push(directory.to_owned()); - Ok(()) - }, - ) + remove_backups_with_sync(&path, |directory| { + synced.push(directory.to_owned()); + Ok(()) + }) .unwrap(); assert_eq!( synced, @@ -771,49 +761,6 @@ fn platform_compatibility_deletion_removes_pending_snapshots() { assert!(path.exists()); } -#[test] -fn platform_compatibility_transient_sqlite_errors_can_retry() { - for code in [rusqlite::ffi::SQLITE_IOERR, rusqlite::ffi::SQLITE_NOMEM] { - let source = UpgradeError::from(sqlite_failure(code)); - assert!(source.is_retryable(), "{source:?}"); - assert!(!crate::backend_task::is_terminal_storage_open_error( - &crate::backend_task::error::TaskError::PlatformDatabaseUpgrade { source } - )); - } -} - -#[test] -fn platform_compatibility_deterministic_io_errors_do_not_retry() { - for kind in [ - std::io::ErrorKind::PermissionDenied, - std::io::ErrorKind::InvalidInput, - ] { - let source = UpgradeError::from(std::io::Error::from(kind)); - assert!(!source.is_retryable(), "{source:?}"); - } -} - -#[test] -fn platform_compatibility_transient_io_errors_preserve_causes() { - use std::io::ErrorKind; - for kind in [ - ErrorKind::Interrupted, - ErrorKind::WouldBlock, - ErrorKind::TimedOut, - ErrorKind::ResourceBusy, - ErrorKind::StorageFull, - ErrorKind::OutOfMemory, - ] { - let error = UpgradeError::from(std::io::Error::from(kind)); - assert!(error.is_retryable(), "{error:?}"); - let cause = std::error::Error::source(&error).unwrap(); - assert_eq!(cause.downcast_ref::().unwrap().kind(), kind); - if kind == ErrorKind::OutOfMemory { - assert!(error.to_string().contains("Close other applications")); - } - } -} - #[test] fn platform_compatibility_active_snapshot_survives_concurrent_cleanup() { use std::io::{BufRead, Read, Write}; @@ -997,142 +944,29 @@ fn platform_compatibility_prune_expired_backups_by_age() { assert!(path.exists(), "the live database is never a candidate"); } -/// The newest snapshot is the floor: it survives any age, even with the clock far in -/// the future, so the last recovery copy is never deleted. +/// An unreadable newest snapshot is kept and reported, and never hands the floor to an +/// older copy that would then be the only one left. +#[cfg(unix)] #[test] -fn platform_compatibility_prune_keeps_newest_snapshot_for_any_clock() { - let (dir, path, _target) = fixture(); +fn platform_compatibility_prune_keeps_unreadable_snapshots() { + use std::os::unix::fs::PermissionsExt; + let (_dir, path, _target) = fixture(); let older = backup(&path).unwrap(); let newest = backup(&path).unwrap(); - set_mtime(&older, -3 * 24 * 60 * 60); - set_mtime(&newest, -2 * 24 * 60 * 60); - let far_future = std::time::SystemTime::now() + RETENTION_DAY * 365 * 100; - assert_eq!( - prune_expired_backups(&path, RETENTION_DAY, far_future).unwrap(), - 1 - ); - assert!(!older.exists()); - assert!(newest.exists()); - assert_eq!( - prune_expired_backups(&path, RETENTION_DAY, far_future).unwrap(), - 0 - ); - assert_eq!(backups(&path).unwrap(), vec![newest]); - drop(dir); -} - -/// An interrupted copy left under a final snapshot name (empty, truncated or -/// header-only) never becomes the floor: the newest complete snapshot survives any -/// age, and the expired interrupted copy is deleted. -#[test] -fn platform_compatibility_prune_floor_skips_interrupted_newest_copy() { - const DAY_SECONDS: i64 = 24 * 60 * 60; - for (incomplete_age, expected_removed) in [(0, 0), (-100 * DAY_SECONDS, 1)] { - for case in [ - "empty bridge", - "truncated bridge", - "header-only bridge", - "truncated upstream", - "empty upstream", - ] { - let (dir, path, _target) = fixture(); - let valid = backup(&path).unwrap(); - let full = std::fs::read(&valid).unwrap(); - set_mtime(&valid, -200 * DAY_SECONDS); - let auto = dir.path().join("backups/auto"); - std::fs::create_dir_all(&auto).unwrap(); - let (incomplete, bytes): (PathBuf, &[u8]) = match case { - "empty bridge" => ( - dir.path() - .join("wallet.sqlite.platform-67d4ef3-backup-killed.sqlite"), - b"", - ), - "truncated bridge" => ( - dir.path() - .join("wallet.sqlite.platform-67d4ef3-backup-killed.sqlite"), - &full[..full.len() / 2], - ), - "header-only bridge" => ( - dir.path() - .join("wallet.sqlite.platform-67d4ef3-backup-killed.sqlite"), - &full[..100], - ), - "truncated upstream" => ( - auto.join("pre-migration-wallet-1-to-2-20200101T000000Z.db"), - &full[..full.len() - 1], - ), - _ => ( - auto.join("pre-migration-wallet-1-to-2-20200101T000000Z.db"), - b"", - ), - }; - std::fs::write(&incomplete, bytes).unwrap(); - set_mtime(&incomplete, incomplete_age); - - let removed = - prune_expired_backups(&path, RETENTION_DAY * 90, std::time::SystemTime::now()) - .unwrap(); - - assert!( - valid.exists(), - "{case}: the only complete snapshot must be kept" - ); - assert_eq!(removed, expected_removed, "{case}"); - assert_eq!(incomplete.exists(), expected_removed == 0, "{case}"); - } + set_mtime(&older, -200 * 24 * 60 * 60); + set_mtime(&newest, -100 * 24 * 60 * 60); + std::fs::set_permissions(&newest, std::fs::Permissions::from_mode(0o000)).unwrap(); + if std::fs::File::open(&newest).is_ok() { + // Running as root: permissions cannot make it unreadable. + return; } -} - -/// A newest snapshot whose usability cannot be proven (unreadable, or with a -/// leftover rollback journal beside it) is never deleted, and never hands the floor -/// to an older copy that would then be the only one left. An unreadable one is -/// reported so the pass is retried. -#[test] -fn platform_compatibility_prune_keeps_snapshots_it_cannot_judge() { - const DAY_SECONDS: i64 = 24 * 60 * 60; - for case in ["unreadable", "leftover journal"] { - let (_dir, path, _target) = fixture(); - let older = backup(&path).unwrap(); - let newest = backup(&path).unwrap(); - set_mtime(&older, -200 * DAY_SECONDS); - set_mtime(&newest, -100 * DAY_SECONDS); - let journal = PathBuf::from(format!("{}-journal", newest.display())); - match case { - "unreadable" => { - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - std::fs::set_permissions(&newest, std::fs::Permissions::from_mode(0o000)) - .unwrap(); - if std::fs::File::open(&newest).is_ok() { - // Running as root: permissions cannot make it unreadable. - std::fs::set_permissions(&newest, std::fs::Permissions::from_mode(0o600)) - .unwrap(); - continue; - } - } - #[cfg(not(unix))] - continue; - } - _ => std::fs::write(&journal, b"pages of an unfinished write").unwrap(), - } - let result = prune_expired_backups(&path, RETENTION_DAY * 90, std::time::SystemTime::now()); + let result = prune_expired_backups(&path, RETENTION_DAY * 90, std::time::SystemTime::now()); - assert!(newest.exists(), "{case}: an unjudged snapshot is kept"); - assert!(older.exists(), "{case}: the newest proven snapshot is kept"); - match case { - "unreadable" => { - result.unwrap_err(); - } - _ => assert_eq!(result.unwrap(), 0, "{case}"), - } - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - std::fs::set_permissions(&newest, std::fs::Permissions::from_mode(0o600)).unwrap(); - } - } + std::fs::set_permissions(&newest, std::fs::Permissions::from_mode(0o600)).unwrap(); + result.unwrap_err(); + assert!(newest.exists(), "an unjudged snapshot is kept"); + assert!(older.exists(), "the newest proven snapshot is kept"); } /// A snapshot written while the clock ran ahead is dated in the future; it must not diff --git a/src/wallet_backend/platform_compatibility/mod.rs b/src/wallet_backend/platform_compatibility/mod.rs index ad6135a47..d56916412 100644 --- a/src/wallet_backend/platform_compatibility/mod.rs +++ b/src/wallet_backend/platform_compatibility/mod.rs @@ -4,7 +4,7 @@ mod engine; mod storage_failure; pub use engine::UpgradeError; #[cfg(test)] -pub(crate) use engine::{LOCK_SUFFIX, backup_lock_path}; +pub(crate) use engine::{LOCK_SUFFIX, backup_lock_path, bridge_backup_path}; pub(crate) use engine::{ is_bridge_backup_name, prune_expired_backups, remove_backups, upstream_backup_timestamp, }; @@ -478,79 +478,32 @@ mod validation_tests { use super::*; use platform_wallet::changeset::PersistenceError; + /// Staged-validation failures keep their storage class through any wrapper: + /// resource failures stay retryable, invalid data stays terminal. #[test] - fn platform_compatibility_staged_resource_failures_remain_retryable() { - for code in [ - rusqlite::ffi::SQLITE_BUSY, - rusqlite::ffi::SQLITE_LOCKED, - rusqlite::ffi::SQLITE_FULL, - rusqlite::ffi::SQLITE_IOERR, - rusqlite::ffi::SQLITE_NOMEM, - ] { - for persistence_wrapper in [false, true] { - let storage = WalletStorageError::Sqlite(rusqlite::Error::SqliteFailure( - rusqlite::ffi::Error::new(code), - None, - )); - let source = if persistence_wrapper { - validation_error(PersistenceError::from(storage)) - } else { - validation_error(storage) - }; - assert!(source.is_retryable(), "{source:?}"); - let cause = std::error::Error::source(&source).unwrap(); - if persistence_wrapper { - assert!(cause.is::()); - } else { - assert!(cause.is::()); - } - assert!(!source.to_string().contains("previous application version")); - assert!(!crate::backend_task::is_terminal_storage_open_error( - &TaskError::PlatformDatabaseUpgrade { source } - )); - } - } - let source = validation_error(WalletStorageError::Io(std::io::Error::from( - std::io::ErrorKind::StorageFull, - ))); - assert!(matches!(source, UpgradeError::StorageFull(_))); - } - - #[test] - fn platform_compatibility_staged_migration_disk_full_remains_retryable() { - let mut connection = rusqlite::Connection::open_in_memory().unwrap(); - connection.pragma_update(None, "max_page_count", 1).unwrap(); - let migration = refinery::Migration::unapplied( - "V1__resource_fixture", - "CREATE TABLE fixture (id INTEGER);", - ) - .unwrap(); - let error = refinery::Runner::new(&[migration]) - .run(&mut connection) - .unwrap_err(); - let source = validation_error(WalletStorageError::Migration(error)); - assert!(matches!(source, UpgradeError::StorageFull(_)), "{source:?}"); - assert!(!crate::backend_task::is_terminal_storage_open_error( - &TaskError::PlatformDatabaseUpgrade { source } - )); - } - - #[test] - fn platform_compatibility_staged_invalid_data_remains_terminal() { - for storage in [ - WalletStorageError::SchemaHistoryMissing, - WalletStorageError::BlobDecode { - reason: "invalid fixture", - }, - WalletStorageError::IntegrityCheckFailed { - report: "invalid fixture".into(), - }, + fn platform_compatibility_staged_validation_errors_keep_their_class() { + let full = || { + WalletStorageError::Sqlite(rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error::new(rusqlite::ffi::SQLITE_FULL), + None, + )) + }; + for source in [ + validation_error(full()), + validation_error(PersistenceError::from(full())), ] { - let source = validation_error(PersistenceError::from(storage)); - assert!(matches!(source, UpgradeError::TypedValidation(_))); - assert!(crate::backend_task::is_terminal_storage_open_error( + assert!(matches!(source, UpgradeError::StorageFull(_)), "{source:?}"); + assert!(std::error::Error::source(&source).is_some()); + assert!(!crate::backend_task::is_terminal_storage_open_error( &TaskError::PlatformDatabaseUpgrade { source } )); } + let source = validation_error(PersistenceError::from( + WalletStorageError::SchemaHistoryMissing, + )); + assert!(matches!(source, UpgradeError::TypedValidation(_))); + assert!(crate::backend_task::is_terminal_storage_open_error( + &TaskError::PlatformDatabaseUpgrade { source } + )); } } diff --git a/src/wallet_backend/platform_compatibility/storage_failure.rs b/src/wallet_backend/platform_compatibility/storage_failure.rs index ec08aee83..f8cd5da64 100644 --- a/src/wallet_backend/platform_compatibility/storage_failure.rs +++ b/src/wallet_backend/platform_compatibility/storage_failure.rs @@ -77,25 +77,51 @@ mod tests { use super::*; #[test] - fn storage_failure_walks_the_source_chain() { - let sqlite = rusqlite::Error::SqliteFailure( - rusqlite::ffi::Error::new(rusqlite::ffi::SQLITE_FULL), - None, - ); - let wrapped = platform_wallet_storage::WalletStorageError::Sqlite(sqlite); - assert_eq!( - StorageFailure::in_chain(&wrapped), - Some(StorageFailure::Full) - ); - let io = std::io::Error::from(std::io::ErrorKind::ReadOnlyFilesystem); - assert_eq!( - StorageFailure::in_chain(&io), - Some(StorageFailure::AccessDenied) - ); + fn storage_failure_classifies_every_error_source() { + use rusqlite::ffi; + use std::io::ErrorKind; + let sqlite = |code| rusqlite::Error::SqliteFailure(ffi::Error::new(code), None); + let sqlite_cases = [ + (ffi::SQLITE_BUSY, Some(StorageFailure::InUse)), + (ffi::SQLITE_LOCKED, Some(StorageFailure::InUse)), + (ffi::SQLITE_FULL, Some(StorageFailure::Full)), + (ffi::SQLITE_IOERR, Some(StorageFailure::Unavailable)), + (ffi::SQLITE_NOMEM, Some(StorageFailure::OutOfMemory)), + (ffi::SQLITE_PERM, Some(StorageFailure::AccessDenied)), + (ffi::SQLITE_READONLY, Some(StorageFailure::AccessDenied)), + (ffi::SQLITE_CORRUPT, None), + ]; + for (code, expected) in sqlite_cases { + // Wrapped, so the source chain is walked too. + let wrapped = platform_wallet_storage::WalletStorageError::Sqlite(sqlite(code)); + assert_eq!(StorageFailure::in_chain(&wrapped), expected, "{code}"); + } + let io_cases = [ + (ErrorKind::StorageFull, Some(StorageFailure::Full)), + (ErrorKind::OutOfMemory, Some(StorageFailure::OutOfMemory)), + (ErrorKind::Interrupted, Some(StorageFailure::Unavailable)), + (ErrorKind::WouldBlock, Some(StorageFailure::Unavailable)), + (ErrorKind::TimedOut, Some(StorageFailure::Unavailable)), + (ErrorKind::ResourceBusy, Some(StorageFailure::Unavailable)), + ( + ErrorKind::PermissionDenied, + Some(StorageFailure::AccessDenied), + ), + ( + ErrorKind::ReadOnlyFilesystem, + Some(StorageFailure::AccessDenied), + ), + (ErrorKind::InvalidInput, None), + (ErrorKind::Other, None), + ]; + for (kind, expected) in io_cases { + assert_eq!( + StorageFailure::in_chain(&std::io::Error::from(kind)), + expected, + "{kind:?}" + ); + } assert!(!StorageFailure::AccessDenied.is_retryable()); - assert_eq!( - StorageFailure::in_chain(&std::io::Error::other("plain")), - None - ); + assert!(StorageFailure::Full.is_retryable()); } } From eea9e760a49ce0e737161b68a6fdbfebdf8efa93 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Wed, 30 Sep 2026 13:57:16 +0000 Subject: [PATCH 29/30] refactor(storage): make upgrade-backup housekeeping best-effort - Open-time tidying logs and continues before and after the open; the current-database probe that let a strict tidy block upgrades is gone. - The backup scan skips symlinks and directories and leaves every other refusal (live database, hard links) to the deletion chokepoint; only the lifecycle lock file keeps its own symlink/hard-link guard. - Backup removal syncs only the directories it actually deleted from. Co-Authored-By: Claude Opus 5.5 --- .../2026-09-10-platform-pin/upgrade-notes.md | 7 +- .../platform_compatibility/engine.rs | 170 ++++----------- .../platform_compatibility/engine/tests.rs | 186 ++++------------ .../platform_compatibility/mod.rs | 206 +++--------------- 4 files changed, 130 insertions(+), 439 deletions(-) diff --git a/docs/ai-design/2026-09-10-platform-pin/upgrade-notes.md b/docs/ai-design/2026-09-10-platform-pin/upgrade-notes.md index 728642cc7..e078e231a 100644 --- a/docs/ai-design/2026-09-10-platform-pin/upgrade-notes.md +++ b/docs/ai-design/2026-09-10-platform-pin/upgrade-notes.md @@ -65,9 +65,10 @@ the rebuild. Opening a database never deletes a published backup except an exact duplicate of a newer backup of the same migration (each failed attempt copies the unchanged original again); it also removes unfinished `.pending` copies left by a crash. Upstream `backups/auto/pre-migration--*.db` snapshots are covered -too. A housekeeping error blocks a new upgrade backup, but not opening a database -that is already current. Backup candidates must have the exact database-specific -name format and be regular files, not symbolic links (or hard links on Unix). +too. Housekeeping is best-effort: an error is logged and retried on the next open, +and never blocks opening. Backup candidates must have the exact database-specific +name format and be regular files; symbolic links and directories are skipped, and +the deletion chokepoint refuses hard links and live databases. Backups older than a user-configurable retention period (default 90 days, or never) are deleted by one global policy covering every database in the data diff --git a/src/wallet_backend/platform_compatibility/engine.rs b/src/wallet_backend/platform_compatibility/engine.rs index f78b9b876..495177eec 100644 --- a/src/wallet_backend/platform_compatibility/engine.rs +++ b/src/wallet_backend/platform_compatibility/engine.rs @@ -323,31 +323,12 @@ fn default_auto_dir(path: &Path) -> PathBuf { platform_wallet_storage::default_auto_backup_dir(path) } -/// Backup candidates that passed validation, plus every rejected one with its reason. -/// -/// Directory-level failures still abort the scan; a single rejected -/// candidate does not, so deletion can remove every valid snapshot before -/// reporting the rejection. -struct BackupScan { - found: Vec, - rejected: Vec<(PathBuf, std::io::Error)>, -} - -/// Strict scan: any rejected candidate fails the whole scan. Retention and -/// publication rely on this to never proceed past an unexpected file. +/// Regular files named like a backup of the database at `path`, next to it or in +/// `auto_dir`. Anything else is skipped; [`remove_backup`] refuses what must never go. fn backups_in(path: &Path, auto_dir: Option<&Path>) -> std::io::Result> { - let scan = scan_backups_in(path, auto_dir)?; - match scan.rejected.into_iter().next() { - Some((_, error)) => Err(error), - None => Ok(scan.found), - } -} - -fn scan_backups_in(path: &Path, auto_dir: Option<&Path>) -> std::io::Result { let mut found = Vec::new(); - let mut rejected = Vec::new(); let Some(parent) = path.parent() else { - return Ok(BackupScan { found, rejected }); + return Ok(found); }; for directory in [Some(parent), auto_dir].into_iter().flatten() { let metadata = match std::fs::symlink_metadata(directory) { @@ -366,56 +347,16 @@ fn scan_backups_in(path: &Path, auto_dir: Option<&Path>) -> std::io::Result found.push(entry.path()), - Err(error) => rejected.push((entry.path(), error)), - } + // Symlinks and directories are skipped; an unreadable type surfaces on use. + let other_type = entry.file_type().is_ok_and(|kind| !kind.is_file()); + if (bridge || upstream) && !other_type { + found.push(entry.path()); } } } found.sort(); found.dedup(); - Ok(BackupScan { found, rejected }) -} - -/// Why a file with a backup (or lifecycle-lock) name is not treated as one. Wrapped -/// in an [`std::io::Error`]; only logs and error details show it. -#[derive(Debug, thiserror::Error)] -enum UnexpectedBackupFile { - #[error("A file named like an upgrade backup is not a regular file.")] - NotRegularFile, - #[error("A file named like an upgrade backup is the live database.")] - LiveDatabase, - #[error("A file named like an upgrade backup has more than one hard link.")] - HardLinked, -} - -/// Scan-time classification of a candidate (also used to vet the lifecycle lock -/// file); the deletion itself is policed by the -/// [`delete_file`](crate::utils::file_deletion::delete_file) chokepoint. -fn validate_backup_file(backup: &Path, database: &Path) -> std::io::Result<()> { - let unexpected = |kind| Err(std::io::Error::other(kind)); - let metadata = std::fs::symlink_metadata(backup)?; - if !metadata.is_file() { - return unexpected(UnexpectedBackupFile::NotRegularFile); - } - if backup == database { - return unexpected(UnexpectedBackupFile::LiveDatabase); - } - if let Ok(live_path) = database.canonicalize() - && backup.canonicalize()? == live_path - { - return unexpected(UnexpectedBackupFile::LiveDatabase); - } - #[cfg(unix)] - { - use std::os::unix::fs::MetadataExt; - if metadata.nlink() != 1 { - return unexpected(UnexpectedBackupFile::HardLinked); - } - } - Ok(()) + Ok(found) } fn remove_backup(backup: &Path, database: &Path) -> std::io::Result<()> { @@ -462,6 +403,22 @@ pub(crate) fn backup_lock_path(database: &Path) -> Option { Some(database.with_file_name(name)) } +/// Refuse an existing lock path that is a symlink, not a regular file, or hard-linked. +fn check_lock_file(lock: &Path) -> std::io::Result<()> { + let metadata = std::fs::symlink_metadata(lock)?; + #[cfg(unix)] + let single_link = std::os::unix::fs::MetadataExt::nlink(&metadata) == 1; + #[cfg(not(unix))] + let single_link = true; + if metadata.is_file() && single_link { + Ok(()) + } else { + Err(std::io::Error::other( + "Upgrade lock path is not a single-link regular file", + )) + } +} + pub(super) fn backup_lock(path: &Path) -> std::io::Result { let Some(lock_path) = backup_lock_path(path) else { return Err(std::io::ErrorKind::InvalidInput.into()); @@ -476,7 +433,7 @@ pub(super) fn backup_lock(path: &Path) -> std::io::Result { let file = match options.open(&lock_path) { Ok(file) => file, Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => { - validate_backup_file(&lock_path, path)?; + check_lock_file(&lock_path)?; std::fs::File::options() .read(true) .write(true) @@ -491,7 +448,6 @@ pub(super) fn backup_lock(path: &Path) -> std::io::Result { } Err(error) => return Err(error), }; - validate_backup_file(&lock_path, path)?; lock_with_grace(&file)?; // Keep the pathname stable: unlinking it could let contenders lock different files. Ok(BackupGuard { @@ -579,10 +535,26 @@ fn same_contents(a: &Path, b: &Path) -> std::io::Result { } } -/// Delete every upgrade backup of the database at `path`; attempts all and returns the -/// first failure. +/// Delete every upgrade backup of the database at `path` and sync the directories +/// deleted from; attempts all and returns the first failure. pub(crate) fn remove_backups(path: &Path) -> std::io::Result<()> { - remove_backups_with_sync(path, crate::utils::backup_prune::sync_directory) + let _guard = backup_lock(path)?; + let mut first_error = None; + let mut touched = BTreeSet::new(); + for backup in backups_in(path, Some(&default_auto_dir(path)))? { + match remove_backup(&backup, path) { + Ok(()) => touched.extend(backup.parent().map(Path::to_owned)), + Err(error) => { + first_error.get_or_insert(error); + } + } + } + for directory in touched { + if let Err(error) = crate::utils::backup_prune::sync_directory(&directory) { + first_error.get_or_insert(error); + } + } + first_error.map_or(Ok(()), Err) } /// [`prune_expired`](crate::utils::backup_prune::prune_expired) over the bridge and @@ -594,8 +566,7 @@ pub(crate) fn prune_expired_backups( now: std::time::SystemTime, ) -> std::io::Result { let auto_dir = default_auto_dir(path); - let unlocked = scan_backups_in(path, Some(&auto_dir))?; - if unlocked.found.is_empty() && unlocked.rejected.is_empty() { + if backups_in(path, Some(&auto_dir))?.is_empty() { return Ok(0); } let _guard = match backup_lock(path) { @@ -606,10 +577,7 @@ pub(crate) fn prune_expired_backups( } Err(error) => return Err(error), }; - let scan = scan_backups_in(path, Some(&auto_dir))?; - let rejected = scan.rejected.into_iter().next().map(|(_, error)| error); - let candidates = scan - .found + let candidates = backups_in(path, Some(&auto_dir))? .into_iter() .filter(|backup| { backup @@ -631,16 +599,12 @@ pub(crate) fn prune_expired_backups( } }) .collect(); - let pruned = crate::utils::backup_prune::prune_expired( + crate::utils::backup_prune::prune_expired( candidates, now, max_age, crate::utils::file_deletion::DeletionIntent::Backup { database: path }, - ); - match (pruned, rejected) { - (Err(error), _) | (Ok(_), Some(error)) => Err(error), - (Ok(removed), None) => Ok(removed), - } + ) } fn backup_created(database: &Path, backup: &Path) -> std::io::Result { @@ -651,44 +615,6 @@ fn backup_created(database: &Path, backup: &Path) -> std::io::Result std::io::Result<()>, -) -> std::io::Result<()> { - let _guard = backup_lock(path)?; - let auto_dir = default_auto_dir(path); - let scan = scan_backups_in(path, Some(&auto_dir))?; - let mut first_error = scan.rejected.into_iter().next().map(|(_, error)| error); - for backup in scan.found { - if let Err(error) = remove_backup(&backup, path) { - first_error.get_or_insert(error); - } - } - // Sync every candidate directory, not just those unlinked from in this call: an - // earlier call may have unlinked successfully and then failed its sync, and a - // retry that finds nothing left to delete must still make that deletion durable. - for directory in [path.parent(), Some(auto_dir.as_path())] - .into_iter() - .flatten() - { - match std::fs::symlink_metadata(directory) { - // No directory means no entry was ever removed from it. - Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, - Err(error) => { - first_error.get_or_insert(error); - continue; - } - // The scan already rejected a non-directory; never open it here. - Ok(metadata) if !metadata.is_dir() => continue, - Ok(_) => {} - } - if let Err(error) = sync_dir(directory) { - first_error.get_or_insert(error); - } - } - first_error.map_or(Ok(()), Err) -} - #[cfg(test)] fn backup(path: &Path) -> Result { backup_with_hook(path, |_| Ok(())) diff --git a/src/wallet_backend/platform_compatibility/engine/tests.rs b/src/wallet_backend/platform_compatibility/engine/tests.rs index 8afa62300..60cb7c1c3 100644 --- a/src/wallet_backend/platform_compatibility/engine/tests.rs +++ b/src/wallet_backend/platform_compatibility/engine/tests.rs @@ -308,72 +308,6 @@ fn platform_compatibility_removes_only_matching_upstream_backups() { assert!(kept.exists()); } -/// An unlink is only durable once its directory is synced: removal must sync -/// every directory it deleted from, and a failed sync must fail the removal so -/// callers keep their retry state instead of -/// retiring it while the deletion could still be lost on power failure. -#[test] -fn platform_compatibility_backup_removal_fails_when_directory_sync_fails() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("det-testnet.sqlite"); - let auto = dir.path().join("backups/auto"); - std::fs::create_dir_all(&auto).unwrap(); - let sibling = dir - .path() - .join("det-testnet.sqlite.platform-67d4ef3-backup-a1.sqlite"); - let upstream = auto.join("pre-migration-det-testnet-1-to-2-20260915T120000Z.db"); - for backup in [&sibling, &upstream] { - std::fs::write(backup, b"backup").unwrap(); - } - - let mut synced = Vec::new(); - let error = remove_backups_with_sync(&path, |directory| { - synced.push(directory.to_owned()); - Err(std::io::Error::other("injected directory sync failure")) - }) - .unwrap_err(); - - assert_eq!(error.kind(), std::io::ErrorKind::Other); - assert!(!sibling.exists()); - assert!(!upstream.exists()); - synced.sort(); - let mut expected = vec![dir.path().to_owned(), auto]; - expected.sort(); - assert_eq!(synced, expected, "every touched directory must be synced"); -} - -/// A retry that finds no backups left must still settle an earlier failed -/// directory sync; otherwise the caller retires its retry state while the -/// earlier unlink could still be lost on power failure. -#[test] -fn platform_compatibility_backup_removal_retries_directory_sync_without_deletions() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("det-testnet.sqlite"); - let sibling = dir - .path() - .join("det-testnet.sqlite.platform-67d4ef3-backup-a1.sqlite"); - std::fs::write(&sibling, b"backup").unwrap(); - - remove_backups_with_sync(&path, |_| { - Err(std::io::Error::other("injected directory sync failure")) - }) - .unwrap_err(); - assert!(!sibling.exists()); - - let mut synced = Vec::new(); - remove_backups_with_sync(&path, |directory| { - synced.push(directory.to_owned()); - Ok(()) - }) - .unwrap(); - assert_eq!( - synced, - vec![dir.path().to_owned()], - "the retry must re-sync the directory whose earlier sync failed; \ - the missing auto-backup directory is skipped" - ); -} - /// The previous recovery snapshot must survive until its replacement is /// published: a failed publication must not leave zero snapshots behind. #[test] @@ -459,86 +393,53 @@ fn platform_compatibility_tidy_removes_only_identical_copies() { assert_eq!(backup_files(dir.path()), vec![copy]); } -/// One invalid candidate must not shield the valid snapshots from deletion: -/// every valid backup is removed and the rejection is still reported. +/// A candidate the deletion chokepoint refuses (a hard link, possibly to a live +/// database) never shields the others: every other backup is removed and the refusal +/// is reported. Directories and symlinks named like backups are skipped. +#[cfg(unix)] #[test] -fn platform_compatibility_invalid_candidate_does_not_block_removal_of_valid_backups() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("det-testnet.sqlite"); - let blocked = dir - .path() - .join("det-testnet.sqlite.platform-67d4ef3-backup-a0.pending"); - std::fs::create_dir(&blocked).unwrap(); - let valid = [ - dir.path() - .join("det-testnet.sqlite.platform-67d4ef3-backup-b1.sqlite"), +fn platform_compatibility_refused_candidate_does_not_block_removal_of_valid_backups() { + let (dir, path, _target) = fixture(); + let bridge = |suffix: &str| { dir.path() - .join("det-testnet.sqlite.platform-67d4ef3-backup-c2.pending"), - ]; + .join(format!("wallet.sqlite.platform-67d4ef3-backup-{suffix}")) + }; + let linked = bridge("link.sqlite"); + std::fs::hard_link(&path, &linked).unwrap(); + let skipped = [bridge("dir.pending"), bridge("symlink.sqlite")]; + std::fs::create_dir(&skipped[0]).unwrap(); + std::os::unix::fs::symlink(&path, &skipped[1]).unwrap(); + let valid = [bridge("b1.sqlite"), bridge("c2.pending")]; for backup in &valid { std::fs::write(backup, b"backup").unwrap(); } let auto = dir.path().join("backups/auto"); std::fs::create_dir_all(&auto).unwrap(); - let upstream = auto.join("pre-migration-det-testnet-1-to-2-20260915T120000Z.db"); + let upstream = auto.join("pre-migration-wallet-1-to-2-20260915T120000Z.db"); std::fs::write(&upstream, b"backup").unwrap(); assert!(remove_backups(&path).is_err()); for backup in valid.iter().chain([&upstream]) { assert!(!backup.exists(), "{} must be removed", backup.display()); } - assert!( - blocked.is_dir(), - "the rejected candidate must be left alone" - ); -} - -#[test] -fn platform_compatibility_tidy_rejects_unexpected_candidate_without_changes() { - let (dir, path, _target) = fixture(); - let before = snapshot(&path); - let pending = dir - .path() - .join("wallet.sqlite.platform-67d4ef3-backup-crash.pending"); - std::fs::copy(&path, &pending).unwrap(); - std::fs::create_dir( - dir.path() - .join("wallet.sqlite.platform-67d4ef3-backup-blocked.sqlite"), - ) - .unwrap(); - assert!(tidy_backups(&path, None).is_err()); - assert!(pending.exists(), "a strict scan failure deletes nothing"); - assert_eq!(snapshot(&path), before); + assert!(linked.exists() && path.exists()); + for entry in &skipped { + assert!(entry.symlink_metadata().is_ok(), "{}", entry.display()); + } + std::fs::remove_file(&linked).unwrap(); + remove_backups(&path).expect("skipped entries are not failures"); } +/// An existing lifecycle lock path that is a symlink is refused, never followed. #[cfg(unix)] #[test] -fn platform_compatibility_rejects_backup_links_to_live_databases() { - let (dir, path, target) = fixture(); - for extension in ["sqlite", "pending"] { - let candidate = dir.path().join(format!( - "wallet.sqlite.platform-67d4ef3-backup-link.{extension}" - )); - for live in [&path, &target] { - std::os::unix::fs::symlink(live, &candidate).unwrap(); - assert!(remove_backups(&path).is_err()); - assert!(live.exists()); - assert!(candidate.symlink_metadata().is_ok()); - std::fs::remove_file(&candidate).unwrap(); - std::fs::hard_link(live, &candidate).unwrap(); - assert!(remove_backups(&path).is_err()); - assert!(live.exists()); - std::fs::remove_file(&candidate).unwrap(); - } - } - let live_alias = dir.path().join("alias.sqlite"); - let live_target = dir - .path() - .join("alias.sqlite.platform-67d4ef3-backup-live.sqlite"); - std::fs::write(&live_target, b"live database").unwrap(); - std::os::unix::fs::symlink(&live_target, &live_alias).unwrap(); - assert!(remove_backups(&live_alias).is_err()); - assert!(live_target.exists()); +fn platform_compatibility_lock_refuses_a_symlinked_lock_file() { + let (dir, path, _target) = fixture(); + let elsewhere = dir.path().join("elsewhere"); + std::fs::write(&elsewhere, b"keep").unwrap(); + std::os::unix::fs::symlink(&elsewhere, backup_lock_path(&path).unwrap()).unwrap(); + assert!(backup_lock(&path).is_err()); + assert_eq!(std::fs::read(&elsewhere).unwrap(), b"keep"); } fn sqlite_failure(code: std::ffi::c_int) -> rusqlite::Error { @@ -1016,31 +917,30 @@ fn platform_compatibility_prune_floor_keeps_latest_migration_snapshot() { assert!(earlier_migration.exists(), "newest by creation time"); } -/// A rejected candidate is reported, but expired valid snapshots are still removed and -/// the rejected entry is left alone. +/// A refused candidate is reported, but expired valid snapshots are still removed and +/// the refused entry is left alone. +#[cfg(unix)] #[test] -fn platform_compatibility_prune_reports_rejected_candidate_after_removing_others() { +fn platform_compatibility_prune_reports_refused_candidate_after_removing_others() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("det-testnet.sqlite"); - let expired = dir - .path() - .join("det-testnet.sqlite.platform-67d4ef3-backup-old.sqlite"); + let bridge = |suffix| bridge_backup_path(&path, suffix); + let expired = bridge("old"); std::fs::write(&expired, b"backup").unwrap(); set_mtime(&expired, -10 * 24 * 60 * 60); - let newest = dir - .path() - .join("det-testnet.sqlite.platform-67d4ef3-backup-new.sqlite"); + let newest = bridge("new"); std::fs::write(&newest, b"newer backup").unwrap(); - let blocked = dir - .path() - .join("det-testnet.sqlite.platform-67d4ef3-backup-blocked.sqlite"); - std::fs::create_dir(&blocked).unwrap(); + let outside = dir.path().join("outside.bin"); + std::fs::write(&outside, b"keep").unwrap(); + let linked = bridge("linked"); + std::fs::hard_link(&outside, &linked).unwrap(); + set_mtime(&linked, -10 * 24 * 60 * 60); prune_expired_backups(&path, RETENTION_DAY, std::time::SystemTime::now()).unwrap_err(); assert!(!expired.exists()); assert!(newest.exists()); - assert!(blocked.is_dir()); + assert!(linked.exists() && outside.exists()); } /// A held lifecycle lock means an open or upgrade is running: pruning skips the diff --git a/src/wallet_backend/platform_compatibility/mod.rs b/src/wallet_backend/platform_compatibility/mod.rs index d56916412..0f3c3c3ab 100644 --- a/src/wallet_backend/platform_compatibility/mod.rs +++ b/src/wallet_backend/platform_compatibility/mod.rs @@ -25,52 +25,22 @@ fn open_with_housekeeping( ) -> Result { let guard = engine::backup_lock(&config.path).map_err(lock_error)?; let auto_dir = config.auto_backup_dir.as_deref(); - // Tidy (strict scan, crash leftovers, identical copies) before another attempt can create - // a snapshot, including after a restart. It stays strict only when a snapshot could - // follow: a database that is already current still opens, since failed housekeeping - // does not endanger it. - if let Err(housekeeping) = tidy(&guard, auto_dir) { - return open_current_only(&config).map_err(|probe| { - tracing::debug!( - error = ?probe, - "Wallet database is not openable without a snapshot; backup housekeeping failure stands" + // Best-effort both times: tidy before a snapshot can be taken, and again to drop a + // copy a failed open just duplicated. A failure never decides the open's result. + let mut tidy_best_effort = || { + if let Err(error) = tidy(&guard, auto_dir) { + tracing::warn!( + ?error, + "Upgrade backup housekeeping failed; retrying on the next open" ); - lock_error(housekeeping) - }); - } + } + }; + tidy_best_effort(); let result = open_inner(&config, &guard); - // Post-open tidying is best-effort: it must neither discard a successful open nor mask - // the open's own error. It also drops a copy a failed open just duplicated. The next - // open retries it before any new snapshot. - if let Err(error) = tidy(&guard, auto_dir) { - tracing::warn!( - ?error, - "Upgrade backup housekeeping failed after opening the wallet database" - ); - } + tidy_best_effort(); result } -/// Open only when no snapshot can be taken: with automatic backups disabled, upstream -/// refuses any pending migration before touching the database, and a brand-new database -/// has nothing to back up. The pinned-profile upgrade is never attempted here. -/// -/// On success the probe is dropped and the database reopened with the caller's config — -/// the database is current, so that open creates no snapshot, while the persister keeps -/// its backup directory for later destructive operations (e.g. wallet deletion). -/// The caller holds the lifecycle guard, so no other DET open can migrate in between. -fn open_current_only(config: &SqlitePersisterConfig) -> Result { - let probe = SqlitePersister::open(config.clone().with_auto_backup_dir(None)) - .map_err(TaskError::from_wallet_storage_open_error)?; - drop(probe); - let persister = - SqlitePersister::open(config.clone()).map_err(TaskError::from_wallet_storage_open_error)?; - tracing::warn!( - "Upgrade backup housekeeping failed; opened the current wallet database and will retry it on the next open" - ); - Ok(persister) -} - fn open_inner( config: &SqlitePersisterConfig, guard: &engine::BackupGuard, @@ -237,42 +207,42 @@ mod tests { } } + /// Housekeeping is best-effort before and after the open: its failure neither + /// blocks nor discards a successful open, nor masks the open's own error. #[test] - fn platform_compatibility_post_open_retention_failure_keeps_open_result() { + fn platform_compatibility_housekeeping_failure_never_decides_the_open() { + fn failing( + calls: &mut usize, + ) -> impl FnMut(&engine::BackupGuard, Option<&std::path::Path>) -> std::io::Result<()> + '_ + { + move |_, _| { + *calls += 1; + Err(std::io::Error::other("housekeeping failure")) + } + } let dir = tempfile::tempdir().unwrap(); crate::app_dir::ensure_data_dir_exists(dir.path()).unwrap(); - let path = dir.path().join("wallet.sqlite"); let mut calls = 0; - let persister = open_with_housekeeping(SqlitePersisterConfig::new(&path), |_, _| { - calls += 1; - if calls == 1 { - Ok(()) - } else { - Err(std::io::Error::other("stray entry rejected by backup scan")) - } - }) - .expect("a post-open retention failure must not discard a successful open"); + let persister = open_with_housekeeping( + SqlitePersisterConfig::new(dir.path().join("wallet.sqlite")), + failing(&mut calls), + ) + .expect("failed housekeeping must not block or discard an open"); assert_eq!(calls, 2); persister.load().unwrap(); let bad = dir.path().join("corrupt.sqlite"); std::fs::write(&bad, b"not a sqlite database, just candy wrappers").unwrap(); let mut calls = 0; - let error = match open_with_housekeeping(SqlitePersisterConfig::new(&bad), |_, _| { - calls += 1; - if calls == 1 { - Ok(()) - } else { - Err(std::io::Error::other("retention failure")) - } - }) { - Ok(_) => panic!("open unexpectedly succeeded on a corrupt database"), - Err(error) => error, - }; + let error = + match open_with_housekeeping(SqlitePersisterConfig::new(&bad), failing(&mut calls)) { + Ok(_) => panic!("open unexpectedly succeeded on a corrupt database"), + Err(error) => error, + }; assert_eq!(calls, 2); assert!( - !matches!(&error, TaskError::FileSystem { source } if source.to_string() == "retention failure"), - "the open error must not be masked by retention: {error:?}" + !matches!(&error, TaskError::FileSystem { source } if source.to_string() == "housekeeping failure"), + "the open error must not be masked by housekeeping: {error:?}" ); } @@ -285,112 +255,6 @@ mod tests { names } - fn failing_retention( - calls: &mut usize, - ) -> impl FnMut(&engine::BackupGuard, Option<&std::path::Path>) -> std::io::Result<()> + '_ - { - move |_, _| { - *calls += 1; - Err(std::io::Error::other("stray entry rejected by backup scan")) - } - } - - #[test] - fn platform_compatibility_pre_open_retention_failure_still_opens_current_database() { - let dir = tempfile::tempdir().unwrap(); - crate::app_dir::ensure_data_dir_exists(dir.path()).unwrap(); - let path = dir.path().join("wallet.sqlite"); - let auto = platform_wallet_storage::default_auto_backup_dir(&path); - // First a brand-new database, then the same database once it is current. - for _ in 0..2 { - let mut calls = 0; - let persister = open_with_housekeeping( - SqlitePersisterConfig::new(&path), - failing_retention(&mut calls), - ) - .expect("failed housekeeping must not block a database that needs no snapshot"); - persister.load().unwrap(); - drop(persister); - assert_eq!(calls, 1); - assert!( - !auto.exists(), - "the probe and reopen must not create the backup directory or any snapshot" - ); - // Only the database itself and its lifecycle lock may exist; the fresh - // profile's file is treated as current by the full reopen. - assert_eq!( - dir_entries(dir.path()), - vec![ - "wallet.sqlite".to_owned(), - format!("wallet.sqlite{}", engine::LOCK_SUFFIX) - ] - ); - } - } - - #[test] - fn platform_compatibility_pre_open_retention_failure_blocks_upgrade() { - let dir = tempfile::tempdir().unwrap(); - crate::app_dir::ensure_data_dir_exists(dir.path()).unwrap(); - let path = dir.path().join("wallet.sqlite"); - rusqlite::Connection::open(&path) - .unwrap() - .execute_batch(include_str!("fixtures/67d4ef3.sql")) - .unwrap(); - let schema = || { - rusqlite::Connection::open(&path) - .unwrap() - .prepare("SELECT type, name, sql FROM sqlite_master ORDER BY type, name") - .unwrap() - .query_map([], |r| { - Ok(format!( - "{}|{}|{:?}", - r.get::<_, String>(0)?, - r.get::<_, String>(1)?, - r.get::<_, Option>(2)? - )) - }) - .unwrap() - .collect::, _>>() - .unwrap() - }; - let before = schema(); - let entries_before = dir_entries(dir.path()); - let mut calls = 0; - let error = match open_with_housekeeping( - SqlitePersisterConfig::new(&path), - failing_retention(&mut calls), - ) { - Ok(_) => panic!("an upgrade must not run while backup retention is failing"), - Err(error) => error, - }; - assert_eq!(calls, 1); - assert!( - matches!(&error, TaskError::FileSystem { source } if source.to_string() == "stray entry rejected by backup scan"), - "the strict retention failure must be reported: {error:?}" - ); - assert_eq!( - schema(), - before, - "the original database must stay untouched" - ); - let mut entries_after = dir_entries(dir.path()); - entries_after.retain(|name| *name != format!("wallet.sqlite{}", engine::LOCK_SUFFIX)); - assert_eq!( - entries_after, entries_before, - "a refused probe must leave no files or directories behind" - ); - let auto = platform_wallet_storage::default_auto_backup_dir(&path); - assert!(!auto.exists() || std::fs::read_dir(&auto).unwrap().next().is_none()); - assert!( - std::fs::read_dir(dir.path()) - .unwrap() - .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) - .all(|name| !name.contains(".platform-67d4ef3-backup-")), - "no bridge snapshot may be published while retention is failing" - ); - } - #[test] fn platform_compatibility_open_reports_lifecycle_lock_as_retryable() { let dir = tempfile::tempdir().unwrap(); From 681ddac0842831765fb27d46f133318acd02dd12 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Wed, 30 Sep 2026 14:04:23 +0000 Subject: [PATCH 30/30] fix(storage): never treat a hard-linked file as an upgrade backup A backup-named hard link (e.g. to the live database) looked like a complete, newest snapshot, so it could become the retention floor and let the real newest backup expire. The backup scan now skips files with more than one link, as it already skips symlinks and directories. Also stop pinning SQLITE_PERM/SQLITE_READONLY migration failures as terminal in the resource-error test; their mapping is not settled here. Co-Authored-By: Claude Opus 5.5 --- src/backend_task/error.rs | 22 ++----- .../platform_compatibility/engine.rs | 24 ++++--- .../platform_compatibility/engine/tests.rs | 66 ++++++++----------- 3 files changed, 49 insertions(+), 63 deletions(-) diff --git a/src/backend_task/error.rs b/src/backend_task/error.rs index 4b3fada71..2a3053720 100644 --- a/src/backend_task/error.rs +++ b/src/backend_task/error.rs @@ -6150,31 +6150,23 @@ mod tests { .expect_err("wrapped SQLite failure") } - /// A migration that ran out of a resource keeps the generic, retryable storage - /// copy; access failures never fix themselves and stay out of that class. + /// A migration that ran out of a resource (disk, OS I/O, memory) keeps the generic, + /// retryable storage copy instead of the terminal incompatible-data guidance. #[test] fn resource_migration_errors_map_to_retryable_wallet_storage() { use rusqlite::ffi; - for (code, retryable) in [ - (ffi::SQLITE_FULL, true), - (ffi::SQLITE_IOERR, true), - (ffi::SQLITE_NOMEM, true), - (ffi::SQLITE_PERM, false), - (ffi::SQLITE_READONLY, false), - ] { + for code in [ffi::SQLITE_FULL, ffi::SQLITE_IOERR, ffi::SQLITE_NOMEM] { let upstream = platform_wallet_storage::WalletStorageError::Migration( resource_migration_error(code), ); let err = TaskError::from_wallet_storage_open_error(upstream); - assert_eq!( + assert!( matches!(err, TaskError::WalletStorage { .. }), - retryable, "SQLite code {code}: {err:?}" ); - assert_eq!( - crate::backend_task::is_terminal_storage_open_error(&err), - !retryable, - "SQLite code {code}" + assert!( + !crate::backend_task::is_terminal_storage_open_error(&err), + "SQLite code {code} must stay retryable" ); } } diff --git a/src/wallet_backend/platform_compatibility/engine.rs b/src/wallet_backend/platform_compatibility/engine.rs index 495177eec..af1fd4f7e 100644 --- a/src/wallet_backend/platform_compatibility/engine.rs +++ b/src/wallet_backend/platform_compatibility/engine.rs @@ -323,8 +323,8 @@ fn default_auto_dir(path: &Path) -> PathBuf { platform_wallet_storage::default_auto_backup_dir(path) } -/// Regular files named like a backup of the database at `path`, next to it or in -/// `auto_dir`. Anything else is skipped; [`remove_backup`] refuses what must never go. +/// Single-link regular files named like a backup of the database at `path`, next to it +/// or in `auto_dir`. Anything else is skipped; [`remove_backup`] refuses what must never go. fn backups_in(path: &Path, auto_dir: Option<&Path>) -> std::io::Result> { let mut found = Vec::new(); let Some(parent) = path.parent() else { @@ -347,8 +347,10 @@ fn backups_in(path: &Path, auto_dir: Option<&Path>) -> std::io::Result Option { Some(database.with_file_name(name)) } -/// Refuse an existing lock path that is a symlink, not a regular file, or hard-linked. -fn check_lock_file(lock: &Path) -> std::io::Result<()> { - let metadata = std::fs::symlink_metadata(lock)?; +/// Whether `metadata` (not followed) is a regular file with no other hard link. +fn is_single_link_file(metadata: &std::fs::Metadata) -> bool { #[cfg(unix)] - let single_link = std::os::unix::fs::MetadataExt::nlink(&metadata) == 1; + let single_link = std::os::unix::fs::MetadataExt::nlink(metadata) == 1; #[cfg(not(unix))] let single_link = true; - if metadata.is_file() && single_link { + metadata.is_file() && single_link +} + +/// Refuse an existing lock path that is a symlink, not a regular file, or hard-linked. +fn check_lock_file(lock: &Path) -> std::io::Result<()> { + if is_single_link_file(&std::fs::symlink_metadata(lock)?) { Ok(()) } else { Err(std::io::Error::other( diff --git a/src/wallet_backend/platform_compatibility/engine/tests.rs b/src/wallet_backend/platform_compatibility/engine/tests.rs index 60cb7c1c3..a6132f4b1 100644 --- a/src/wallet_backend/platform_compatibility/engine/tests.rs +++ b/src/wallet_backend/platform_compatibility/engine/tests.rs @@ -1,8 +1,3 @@ -#![expect( - clippy::disallowed_methods, - reason = "test fixture setup/teardown outside any production deletion path" -)] - use super::*; fn fixture() -> (tempfile::TempDir, PathBuf, PathBuf) { @@ -393,22 +388,24 @@ fn platform_compatibility_tidy_removes_only_identical_copies() { assert_eq!(backup_files(dir.path()), vec![copy]); } -/// A candidate the deletion chokepoint refuses (a hard link, possibly to a live -/// database) never shields the others: every other backup is removed and the refusal -/// is reported. Directories and symlinks named like backups are skipped. +/// Removal deletes every backup and skips entries named like one that are directories, +/// symlinks or hard links (here to the live database), which are never followed. #[cfg(unix)] #[test] -fn platform_compatibility_refused_candidate_does_not_block_removal_of_valid_backups() { +fn platform_compatibility_removal_skips_links_and_directories() { let (dir, path, _target) = fixture(); let bridge = |suffix: &str| { dir.path() .join(format!("wallet.sqlite.platform-67d4ef3-backup-{suffix}")) }; - let linked = bridge("link.sqlite"); - std::fs::hard_link(&path, &linked).unwrap(); - let skipped = [bridge("dir.pending"), bridge("symlink.sqlite")]; - std::fs::create_dir(&skipped[0]).unwrap(); - std::os::unix::fs::symlink(&path, &skipped[1]).unwrap(); + let skipped = [ + bridge("link.sqlite"), + bridge("dir.pending"), + bridge("symlink.sqlite"), + ]; + std::fs::hard_link(&path, &skipped[0]).unwrap(); + std::fs::create_dir(&skipped[1]).unwrap(); + std::os::unix::fs::symlink(&path, &skipped[2]).unwrap(); let valid = [bridge("b1.sqlite"), bridge("c2.pending")]; for backup in &valid { std::fs::write(backup, b"backup").unwrap(); @@ -418,16 +415,14 @@ fn platform_compatibility_refused_candidate_does_not_block_removal_of_valid_back let upstream = auto.join("pre-migration-wallet-1-to-2-20260915T120000Z.db"); std::fs::write(&upstream, b"backup").unwrap(); - assert!(remove_backups(&path).is_err()); + remove_backups(&path).unwrap(); for backup in valid.iter().chain([&upstream]) { assert!(!backup.exists(), "{} must be removed", backup.display()); } - assert!(linked.exists() && path.exists()); + assert!(path.exists()); for entry in &skipped { assert!(entry.symlink_metadata().is_ok(), "{}", entry.display()); } - std::fs::remove_file(&linked).unwrap(); - remove_backups(&path).expect("skipped entries are not failures"); } /// An existing lifecycle lock path that is a symlink is refused, never followed. @@ -917,30 +912,23 @@ fn platform_compatibility_prune_floor_keeps_latest_migration_snapshot() { assert!(earlier_migration.exists(), "newest by creation time"); } -/// A refused candidate is reported, but expired valid snapshots are still removed and -/// the refused entry is left alone. +/// A backup-named hard link (here to the live database) is never a candidate, so it +/// cannot become the floor and let the real newest backup expire. #[cfg(unix)] #[test] -fn platform_compatibility_prune_reports_refused_candidate_after_removing_others() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("det-testnet.sqlite"); - let bridge = |suffix| bridge_backup_path(&path, suffix); - let expired = bridge("old"); - std::fs::write(&expired, b"backup").unwrap(); - set_mtime(&expired, -10 * 24 * 60 * 60); - let newest = bridge("new"); - std::fs::write(&newest, b"newer backup").unwrap(); - let outside = dir.path().join("outside.bin"); - std::fs::write(&outside, b"keep").unwrap(); - let linked = bridge("linked"); - std::fs::hard_link(&outside, &linked).unwrap(); - set_mtime(&linked, -10 * 24 * 60 * 60); - - prune_expired_backups(&path, RETENTION_DAY, std::time::SystemTime::now()).unwrap_err(); +fn platform_compatibility_prune_skips_hard_linked_backup_names() { + let (_dir, path, _target) = fixture(); + let real = backup(&path).unwrap(); + set_mtime(&real, -200 * 24 * 60 * 60); + let linked = bridge_backup_path(&path, "linked"); + std::fs::hard_link(&path, &linked).unwrap(); - assert!(!expired.exists()); - assert!(newest.exists()); - assert!(linked.exists() && outside.exists()); + assert_eq!( + prune_expired_backups(&path, RETENTION_DAY * 90, std::time::SystemTime::now()).unwrap(), + 0 + ); + assert!(real.exists(), "the only real backup is the floor"); + assert!(linked.exists() && path.exists()); } /// A held lifecycle lock means an open or upgrade is running: pruning skips the