From 3589f7d6e65702030e4de3721794da7a366df8a0 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Thu, 9 Dec 2021 14:31:47 +0700 Subject: [PATCH 001/153] Implement Sum trait for Amount --- common/src/primitives/amount.rs | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/common/src/primitives/amount.rs b/common/src/primitives/amount.rs index bc20b493ae..9fa2c99e42 100644 --- a/common/src/primitives/amount.rs +++ b/common/src/primitives/amount.rs @@ -317,6 +317,27 @@ mod tests { assert_eq!(Amount { val: IntType::MAX } + Amount { val: 1 }, None); } + #[test] + fn sum_some() { + let amounts = vec![Amount { val: 1 }, Amount { val: 2 }, Amount { val: 3 }]; + assert_eq!( + amounts.into_iter().sum::>(), + Some(Amount { val: 6 }) + ); + } + + #[test] + fn sum_overflow() { + let amounts = vec![ + Amount { val: 1 }, + Amount { val: 2 }, + Amount { + val: IntType::MAX - 2, + }, + ]; + assert_eq!(amounts.into_iter().sum::>(), None); + } + #[test] fn sub_underflow() { assert_eq!(Amount { val: IntType::MIN } - Amount { val: 1 }, None); From 7d01158bfb984de3e49ba5f6be369b46032967bc Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Mon, 31 Jan 2022 00:40:29 +0700 Subject: [PATCH 002/153] Make Transaction errors thiserror --- common/src/chain/transaction.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/common/src/chain/transaction.rs b/common/src/chain/transaction.rs index 234412f326..f3c0c768cc 100644 --- a/common/src/chain/transaction.rs +++ b/common/src/chain/transaction.rs @@ -17,6 +17,7 @@ use crate::primitives::{Id, Idable}; use serialization::{DirectDecode, DirectEncode, Encode}; +use thiserror::Error; use crate::chain::transaction::transaction_v1::TransactionV1; @@ -54,8 +55,9 @@ impl Idable for Transaction { } } -#[derive(Debug, Clone)] +#[derive(Error, Debug, Clone)] pub enum TransactionCreationError { + #[error("An unknown error has occurred")] Unknown, } From e3154161627de7f81d8d76ef8768a794ee1635e7 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Mon, 31 Jan 2022 00:55:04 +0700 Subject: [PATCH 003/153] Conversion from Amount to underlying u128 type --- common/src/primitives/amount.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/common/src/primitives/amount.rs b/common/src/primitives/amount.rs index 9fa2c99e42..b1af89c6d2 100644 --- a/common/src/primitives/amount.rs +++ b/common/src/primitives/amount.rs @@ -131,6 +131,12 @@ impl Amount { } } +impl From for u128 { + fn from(amount: Amount) -> Self { + amount.val + } +} + impl std::ops::Add for Amount { type Output = Option; From b612cdbe02631ad8af2dbe1673be5f9db7669e82 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Tue, 22 Feb 2022 17:47:48 +0700 Subject: [PATCH 004/153] Basic Mempool Functionality --- Cargo.lock | 466 ++++++----- Cargo.toml | 3 + common/Cargo.toml | 1 + common/src/chain/transaction/input.rs | 9 + common/src/primitives/amount.rs | 5 + mempool/Cargo.toml | 17 + mempool/src/lib.rs | 3 + mempool/src/pool.rs | 1022 +++++++++++++++++++++++++ 8 files changed, 1348 insertions(+), 178 deletions(-) create mode 100644 mempool/Cargo.toml create mode 100644 mempool/src/lib.rs create mode 100644 mempool/src/pool.rs diff --git a/Cargo.lock b/Cargo.lock index e363adb80e..f2e8963616 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -80,9 +80,9 @@ checksum = "bb07d2053ccdbe10e2af2995a2f116c1330396493dc1269f6a91d0ae82e19704" [[package]] name = "arbitrary" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25e0a02cf12f1b1f48b14cb7f8217b876d09992b39c816ffb3b1ba64dd979a87" +checksum = "5a7924531f38b1970ff630f03eb20a2fde69db5c590c93b0f3482e95dcc5fd60" [[package]] name = "arrayref" @@ -149,9 +149,9 @@ dependencies = [ [[package]] name = "async-global-executor" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd8b508d585e01084059b60f06ade4cb7415cd2e4084b71dd1cb44e7d3fb9880" +checksum = "5262ed948da60dd8956c6c5aca4d4163593dddb7b32d73267c93dab7b2e98940" dependencies = [ "async-channel", "async-executor", @@ -159,6 +159,7 @@ dependencies = [ "async-lock", "blocking", "futures-lite", + "num_cpus", "once_cell", ] @@ -192,9 +193,9 @@ dependencies = [ [[package]] name = "async-task" -version = "4.2.0" +version = "4.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30696a84d817107fc028e049980e09d5e140e8da8f1caeb17e8e950658a3cea9" +checksum = "7a40729d2133846d9ed0ea60a8b9541bccddab49cd30f0715a1da672fe9a2524" [[package]] name = "async-trait" @@ -300,9 +301,9 @@ dependencies = [ [[package]] name = "bit-set" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e11e16035ea35e4e5997b393eacbf6f63983188f7a2ad25bfb13465f5ad59de" +checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" dependencies = [ "bit-vec", ] @@ -330,14 +331,26 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitvec" -version = "1.0.0" +version = "0.20.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1489fcb93a5bb47da0462ca93ad252ad6af2145cce58d10d46a83931ba9f016b" +checksum = "7774144344a4faa177370406a7ff5f1da24303817368584c6206c8303eb07848" dependencies = [ - "funty", - "radium", + "funty 1.1.0", + "radium 0.6.2", "tap", - "wyz", + "wyz 0.2.0", +] + +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty 2.0.0", + "radium 0.7.0", + "tap", + "wyz 0.5.0", ] [[package]] @@ -442,9 +455,9 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "bytes" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" +checksum = "f0b3de4a0c5e67e16066a0715723abd91edc2f9001d09c46e1dca929351e130e" [[package]] name = "cache-padded" @@ -485,9 +498,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chacha20" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01b72a433d0cf2aef113ba70f62634c56fddb0f244e6377185c56a7cadbd8f91" +checksum = "5c80e5460aa66fe3b91d40bcbdab953a597b60053e34d684ac6903f863b680a6" dependencies = [ "cfg-if", "cipher", @@ -497,9 +510,9 @@ dependencies = [ [[package]] name = "chacha20poly1305" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b84ed6d1d5f7aa9bdde921a5090e0ca4d934d250ea3b402a5fab3a994e28a2a" +checksum = "a18446b09be63d457bbec447509e85f662f32952b035ce892290396bc0b0cff5" dependencies = [ "aead", "chacha20", @@ -560,7 +573,7 @@ name = "chainstate-types" version = "0.1.0" dependencies = [ "common", - "parity-scale-codec", + "parity-scale-codec 3.1.5", "serialization", "storage", "thiserror", @@ -606,9 +619,9 @@ dependencies = [ [[package]] name = "clap" -version = "3.2.5" +version = "3.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d53da17d37dba964b9b3ecb5c5a1f193a2762c700e6829201e645b9381c99dc7" +checksum = "44bbe24bbd31a185bc2c4f7c2abe80bea13a20d57ee4e55be70ac512bdc76417" dependencies = [ "atty", "bitflags", @@ -623,9 +636,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "3.2.5" +version = "3.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c11d40217d16aee8508cc8e5fde8b4ff24639758608e5374e731b53f85749fb9" +checksum = "9ba52acd3b0a5c33aeada5cdaa3267cdc7c594a98731d4268cdc1532f4264cb4" dependencies = [ "heck 0.4.0", "proc-macro-error", @@ -636,9 +649,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.2.2" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5538cd660450ebeb4234cfecf8f2284b844ffc4c50531e66d584ad5b91293613" +checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" dependencies = [ "os_str_bytes", ] @@ -679,7 +692,7 @@ dependencies = [ "logging", "loom", "merkletree", - "parity-scale-codec", + "parity-scale-codec 3.1.5", "proptest", "rand 0.8.5", "rustc-hex", @@ -696,18 +709,18 @@ dependencies = [ [[package]] name = "concurrent-queue" -version = "1.2.2" +version = "1.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30ed07550be01594c6026cff2a1d7fe9c8f683caa798e12b68694ac9e88286a3" +checksum = "af4780a44ab5696ea9e28294517f1fffb421a83a25af521333c838635509db9c" dependencies = [ "cache-padded", ] [[package]] name = "const_format" -version = "0.2.25" +version = "0.2.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2906f2480cdc015e998deac388331a0f1c1cd88744948c749513020c83c370bc" +checksum = "939dc9e2eb9077e0679d2ce32de1ded8531779360b003b4a972a7a39ec263495" dependencies = [ "const_format_proc_macros", ] @@ -768,9 +781,9 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.5" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c02a4d71819009c192cf4872265391563fd6a84c81ff2c0f2a7026ca4c1d85c" +checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" dependencies = [ "cfg-if", "crossbeam-utils", @@ -778,9 +791,9 @@ dependencies = [ [[package]] name = "crossbeam-deque" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e" +checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc" dependencies = [ "cfg-if", "crossbeam-epoch", @@ -789,9 +802,9 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.9" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07db9d94cbd326813772c968ccd25999e5f8ae22f4f8d1b11effa37ef6ce281d" +checksum = "045ebe27666471bb549370b4b0b3e51b07f56325befa4284db65fc89c02511b1" dependencies = [ "autocfg", "cfg-if", @@ -803,9 +816,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.9" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ff1f980957787286a554052d03c7aee98d99cc32e09f6d45f0a814133c87978" +checksum = "51887d4adc7b564537b15adcfb307936f8075dfcd5f00dde9a9f1d29383682bc" dependencies = [ "cfg-if", "once_cell", @@ -823,7 +836,7 @@ dependencies = [ "num", "num-derive", "num-traits", - "parity-scale-codec", + "parity-scale-codec 3.1.5", "rand 0.8.5", "rand_chacha 0.3.1", "ripemd", @@ -838,9 +851,9 @@ dependencies = [ [[package]] name = "crypto-common" -version = "0.1.3" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57952ca27b5e3606ff4dd79b0020231aaf9d6aa76dc05fd30137538c50bd3ce8" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ "generic-array", "typenum", @@ -1035,15 +1048,15 @@ dependencies = [ [[package]] name = "event-listener" -version = "2.5.2" +version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77f3309417938f28bf8228fcff79a4a37103981e3e186d2ccd19c74b38f4eb71" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" [[package]] name = "expect-test" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dced95c9dcd4e3241f95841aad395f9c8d7933a3b0b524bdeb2440885c72a271" +checksum = "1d4661aca38d826eb7c72fe128e4238220616de4c0cc00db7bfc38e2e1364dd3" dependencies = [ "dissimilar", "once_cell", @@ -1051,18 +1064,18 @@ dependencies = [ [[package]] name = "fastrand" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3fcf0cee53519c866c09b5de1f6c56ff9d647101f81c1964fa632e148896cdf" +checksum = "a7a407cfaa3385c4ae6b23e84623d48c2798d06e3e6a1878f7f59f17b3f86499" dependencies = [ "instant", ] [[package]] name = "fixedbitset" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "279fb028e20b3c4c320317955b77c5e0c9701f05a1d309905d6fc702cdc5053e" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flate2" @@ -1111,6 +1124,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" +[[package]] +name = "funty" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed34cd105917e91daa4da6b3728c47b068749d6a62c59811f06ed2ac71d9da7" + [[package]] name = "funty" version = "2.0.0" @@ -1234,15 +1253,15 @@ dependencies = [ [[package]] name = "generator" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1d9279ca822891c1a4dae06d185612cf8fc6acfe5dff37781b41297811b12ee" +checksum = "cc184cace1cea8335047a471cc1da80f18acf8a76f3bab2028d499e328948ec7" dependencies = [ "cc", "libc", "log", "rustversion", - "winapi 0.3.9", + "windows 0.32.0", ] [[package]] @@ -1302,9 +1321,9 @@ dependencies = [ [[package]] name = "gloo-net" -version = "0.2.0" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6d37f728c2b2b8c568bd2efb34ce9087e347c182db68f101a969b4fe23054d5" +checksum = "351e6f94c76579cc9f9323a15f209086fc7bd428bff4288723d3a417851757b2" dependencies = [ "futures-channel", "futures-core", @@ -1334,9 +1353,9 @@ dependencies = [ [[package]] name = "gloo-utils" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c0bbef55e98d946adbd89f3c65a497cf9adb995a73b99573f30180e8813ab21" +checksum = "929c53c913bb7a88d75d9dc3e9705f963d8c2b9001510b25ddaf671b9fb7049d" dependencies = [ "js-sys", "wasm-bindgen", @@ -1364,19 +1383,13 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.11.2" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" dependencies = [ "ahash", ] -[[package]] -name = "hashbrown" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db0d4cf898abf0081f964436dc980e96670a0f36863e4b83aaacdb65c9d7ccc3" - [[package]] name = "heck" version = "0.3.3" @@ -1461,9 +1474,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.19" +version = "0.14.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42dc3c131584288d375f2d07f822b0cb012d8c6fb899a5b9fdb3cb7eb9b6004f" +checksum = "02c929dc5c39e335a03c405292728118860721b10190d98c2a0f0efd5baafbac" dependencies = [ "bytes", "futures-channel", @@ -1522,9 +1535,9 @@ dependencies = [ [[package]] name = "if-watch" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "774d59367a3d26965d21ac70a86fcb697d83405d1c4312d1d8a6855296af0cf7" +checksum = "015a7df1eb6dda30df37f34b63ada9b7b352984b0e84de2a20ed526345000791" dependencies = [ "async-io", "core-foundation", @@ -1535,7 +1548,7 @@ dependencies = [ "log", "rtnetlink", "system-configuration", - "windows", + "windows 0.34.0", ] [[package]] @@ -1551,12 +1564,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.9.0" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c6392766afd7964e2531940894cffe4bd8d7d17dbc3c1c4857040fd4b33bdb3" +checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e" dependencies = [ "autocfg", - "hashbrown 0.12.1", + "hashbrown", ] [[package]] @@ -1591,9 +1604,9 @@ checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d" [[package]] name = "js-sys" -version = "0.3.58" +version = "0.3.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3fac17f7123a73ca62df411b1bf727ccc805daa070338fda671c86dac1bdc27" +checksum = "258451ab10b34f8af53416d1fdab72c22e805f0c92a1136d59470ec0b11138b2" dependencies = [ "wasm-bindgen", ] @@ -1939,9 +1952,9 @@ dependencies = [ [[package]] name = "libp2p-metrics" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adc4357140141ba9739eee71b20aa735351c0fc642635b2bffc7f57a6b5c1090" +checksum = "564a7e5284d7d9b3140fdfc3cb6567bc32555e86a21de5604c2ec85da05cf384" dependencies = [ "libp2p-core", "libp2p-gossipsub", @@ -2078,7 +2091,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc195aab5b803465bf614a5a4765741abce6c8d64e7d8ca57acd2923661fba9f" dependencies = [ - "clap 3.2.5", + "clap 3.2.15", "crossbeam-channel", "rayon", "termcolor", @@ -2126,11 +2139,11 @@ dependencies = [ [[package]] name = "lru" -version = "0.7.7" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c84e6fe5655adc6ce00787cf7dcaf8dc4f998a0565d23eafc207a8b08ca3349a" +checksum = "e999beba7b6e8345721bd280141ed958096a2e4abdf74f67ff4ce49b4b54e47a" dependencies = [ - "hashbrown 0.11.2", + "hashbrown", ] [[package]] @@ -2173,6 +2186,18 @@ dependencies = [ "autocfg", ] +[[package]] +name = "mempool" +version = "0.1.0" +dependencies = [ + "anyhow", + "common", + "parity-scale-codec 2.3.1", + "rand 0.8.5", + "serialization", + "thiserror", +] + [[package]] name = "merkletree" version = "0.21.0" @@ -2223,6 +2248,7 @@ dependencies = [ "common", "crypto", "logging", + "mempool", "p2p", "rpc", "script", @@ -2256,9 +2282,9 @@ dependencies = [ [[package]] name = "mockall" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5641e476bbaf592a3939a7485fa079f427b4db21407d5ebfd5bba4e07a1f6f4c" +checksum = "e2be9a9090bc1cac2930688fa9478092a64c6a92ddc6ae0692d46b37d9cab709" dependencies = [ "cfg-if", "downcast", @@ -2271,9 +2297,9 @@ dependencies = [ [[package]] name = "mockall_derive" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "262d56735932ee0240d515656e5a7667af3af2a5b0af4da558c4cff2b2aeb0c7" +checksum = "86d702a0530a0141cf4ed147cf5ec7be6f2c187d4e37fcbefc39cf34116bfe8f" dependencies = [ "cfg-if", "proc-macro2", @@ -2360,9 +2386,9 @@ dependencies = [ [[package]] name = "netlink-packet-route" -version = "0.11.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "733ea73609acfd7fa7ddadfb7bf709b0471668c456ad9513685af543a06342b2" +checksum = "d9ea4302b9759a7a88242299225ea3688e63c85ea136371bb6cf94fd674efaab" dependencies = [ "anyhow", "bitflags", @@ -2386,23 +2412,24 @@ dependencies = [ [[package]] name = "netlink-proto" -version = "0.9.2" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef8785b8141e8432aa45fceb922a7e876d7da3fad37fa7e7ec702ace3aa0826b" +checksum = "65b4b14489ab424703c092062176d52ba55485a89c076b4f9db05092b7223aa6" dependencies = [ "bytes", "futures", "log", "netlink-packet-core", "netlink-sys", + "thiserror", "tokio", ] [[package]] name = "netlink-sys" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e4c9f9547a08241bee7b6558b9b98e1f290d187de8b7cfca2bbb4937bcaa8f8" +checksum = "92b654097027250401127914afb37cb1f311df6610a9891ff07a757e94199027" dependencies = [ "async-io", "bytes", @@ -2419,15 +2446,13 @@ checksum = "d36047f46c69ef97b60e7b069a26ce9a15cd8a7852eddb6991ea94a83ba36a78" [[package]] name = "nix" -version = "0.22.3" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4916f159ed8e5de0082076562152a76b7a1f64a01fd9d1e0fea002c37624faf" +checksum = "195cdbc1741b8134346d515b3a56a1c94b0912758009cfd53f99ea0f57b065fc" dependencies = [ "bitflags", - "cc", "cfg-if", "libc", - "memoffset", ] [[package]] @@ -2438,7 +2463,7 @@ dependencies = [ "assert_cmd", "chainstate", "chainstate-storage", - "clap 3.2.5", + "clap 3.2.15", "common", "directories", "jsonrpsee", @@ -2533,9 +2558,9 @@ dependencies = [ [[package]] name = "num-rational" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d41702bd167c2df5520b384281bc111a4b5efcf7fbc4c9c222c815b07e0a6a6a" +checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0" dependencies = [ "autocfg", "num-bigint", @@ -2564,9 +2589,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.12.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7709cef83f0c1f58f666e746a08b21e0085f7440fa6a29cc194d68aac97a4225" +checksum = "18a6dbe30758c9f83eb00cbea4ac95966305f5a7772f3f42ebfc7fc7eddbd8e1" [[package]] name = "opaque-debug" @@ -2582,9 +2607,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "os_str_bytes" -version = "6.1.0" +version = "6.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21326818e99cfe6ce1e524c2a805c189a99b5ae555a35d19f9a284b427d86afa" +checksum = "648001efe5d5c0102d8cea768e348da85d90af8ba91f0bea908f157951493cd4" [[package]] name = "owning_ref" @@ -2611,7 +2636,7 @@ dependencies = [ "libp2p", "logging", "p2p-test-utils", - "parity-scale-codec", + "parity-scale-codec 3.1.5", "portpicker", "rpc", "serde", @@ -2639,6 +2664,20 @@ dependencies = [ "tokio", ] +[[package]] +name = "parity-scale-codec" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "373b1a4c1338d9cd3d1fa53b3a11bdab5ab6bd80a20f7f7becd76953ae2be909" +dependencies = [ + "arrayvec", + "bitvec 0.20.4", + "byte-slice-cast", + "impl-trait-for-tuples", + "parity-scale-codec-derive 2.3.1", + "serde", +] + [[package]] name = "parity-scale-codec" version = "3.1.5" @@ -2646,13 +2685,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9182e4a71cae089267ab03e67c99368db7cd877baf50f931e5d6d4b71e195ac0" dependencies = [ "arrayvec", - "bitvec", + "bitvec 1.0.1", "byte-slice-cast", "impl-trait-for-tuples", - "parity-scale-codec-derive", + "parity-scale-codec-derive 3.1.3", "serde", ] +[[package]] +name = "parity-scale-codec-derive" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1557010476e0595c9b568d16dcfb81b93cdeb157612726f5170d31aa707bed27" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "parity-scale-codec-derive" version = "3.1.3" @@ -2752,18 +2803,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58ad3879ad3baf4e44784bc6a718a8698867bb991f8ce24d1bcbe2cfb4c3a75e" +checksum = "78203e83c48cffbe01e4a2d35d566ca4de445d79a85372fc64e378bfc812a260" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "744b6f092ba29c3650faf274db506afd39944f48420f6c86b17cfe0ee1cb36bb" +checksum = "710faf75e1b33345361201d36d04e98ac1ed8909151a017ed384700836104c74" dependencies = [ "proc-macro2", "quote", @@ -2911,9 +2962,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.40" +version = "1.0.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd96a1e8ed2596c337f8eae5f24924ec83f5ad5ab21ea8e455d3566c69fbcaf7" +checksum = "c278e965f1d8cf32d6e0e96de3d3e79712178ae67986d9cf9151f51e95aac89b" dependencies = [ "unicode-ident", ] @@ -3050,6 +3101,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "radium" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "643f8f41a8ebc4c5dc4515c82bb8abd397b527fc20fd681b7c011c2aee5d44fb" + [[package]] name = "radium" version = "0.7.0" @@ -3199,9 +3256,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.2.13" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62f25bc4c7e55e0b0b7a1d43fb893f4fa1361d0abe38b9ce4f323c2adfe6ef42" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" dependencies = [ "bitflags", ] @@ -3219,9 +3276,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.5.6" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d83f127d94bdbcda4c8cc2e50f6f84f4b611f69c902699ca385a39c3a75f9ff1" +checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b" dependencies = [ "aho-corasick", "memchr", @@ -3239,9 +3296,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.26" +version = "0.6.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49b3de9ec5dc0a3417da371aab17d729997c15010e7fd24ff707773a33bddb64" +checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244" [[package]] name = "remove_dir_all" @@ -3344,9 +3401,9 @@ dependencies = [ [[package]] name = "rtnetlink" -version = "0.9.1" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f54290e54521dac3de4149d83ddf9f62a359b3cc93bcb494a794a41e6f4744b" +checksum = "322c53fd76a18698f1c27381d58091de3a043d356aa5bd0d510608b565f469a0" dependencies = [ "async-global-executor", "futures", @@ -3384,7 +3441,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver 1.0.10", + "semver 1.0.12", ] [[package]] @@ -3422,9 +3479,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0a5f7c728f5d284929a1cccb5bc19884422bfe6ef4d6c409da2c41838983fcf" +checksum = "24c8ad4f0c00e1eb5bc7614d236a7f1300e3dbd76b68cac8e06fb00b015ad8d8" [[package]] name = "rusty-fork" @@ -3503,7 +3560,7 @@ dependencies = [ "hex", "hex-literal", "logging", - "parity-scale-codec", + "parity-scale-codec 3.1.5", "proptest", "serialization", "thiserror", @@ -3554,9 +3611,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.10" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a41d061efea015927ac527063765e73601444cdc344ba855bc7bd44578b25e1c" +checksum = "a2333e6df6d6598f2b1974829f853c2b4c5f4a6e503c10af918081aa6f8564e1" [[package]] name = "semver-parser" @@ -3575,9 +3632,9 @@ checksum = "f638d531eccd6e23b980caf34876660d38e265409d8e99b397ab71eb3612fad0" [[package]] name = "serde" -version = "1.0.137" +version = "1.0.140" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61ea8d54c77f8315140a05f4c7237403bf38b72704d031543aa1d16abbf517d1" +checksum = "fc855a42c7967b7c369eb5860f7164ef1f6f81c20c7cc1141f2a604e18723b03" dependencies = [ "serde_derive", ] @@ -3593,9 +3650,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.137" +version = "1.0.140" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f26faba0c3959972377d3b2d306ee9f71faee9714294e41bb777f83f88578be" +checksum = "6f2122636b9fe3b81f1cb25099fcf2d3f542cdb1d45940d56c713158884a05da" dependencies = [ "proc-macro2", "quote", @@ -3604,9 +3661,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.81" +version = "1.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c" +checksum = "82c2c1fdcd807d1098552c5b9a36e425e42e9fbd7c6a37a8425f390f781f7fa7" dependencies = [ "itoa", "ryu", @@ -3615,9 +3672,9 @@ dependencies = [ [[package]] name = "serde_test" -version = "1.0.137" +version = "1.0.140" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe196827aea34242c314d2f0dd49ed00a129225e80dda71b0dbf65d54d25628d" +checksum = "38b0651a2f427e4a4d74d458947aa5ca36c62c1b503344d143763ec06216a975" dependencies = [ "serde", ] @@ -3662,7 +3719,7 @@ version = "0.1.0" dependencies = [ "arraytools", "hex-literal", - "parity-scale-codec", + "parity-scale-codec 3.1.5", "rand 0.8.5", ] @@ -3670,7 +3727,7 @@ dependencies = [ name = "serialization-tagged" version = "0.1.0" dependencies = [ - "parity-scale-codec", + "parity-scale-codec 3.1.5", "proptest", "serialization", "serialization-core", @@ -3784,9 +3841,12 @@ checksum = "f054c6c1a6e95179d6f23ed974060dcefb2d9388bb7256900badad682c499de4" [[package]] name = "slab" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb703cfe953bccee95685111adeedb76fabe4e97549a58d16f03ea7b9367bb32" +checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" +dependencies = [ + "autocfg", +] [[package]] name = "slave-pool" @@ -3799,9 +3859,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83" +checksum = "2fd0db749597d91ff862fd1d55ea87f7855a744a8425a64695b6fca237d1dad1" [[package]] name = "snow" @@ -3853,9 +3913,9 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] name = "sscanf" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6311683a27f16025db4f8bcf0732662f5fecd1406f53f0aab9adbf6f396f1189" +checksum = "40f3891ecd1b4d716fd674bf238ee743d2cd948c9aeb3b42b1dd9d8f56d0154b" dependencies = [ "const_format", "lazy_static", @@ -3865,9 +3925,9 @@ dependencies = [ [[package]] name = "sscanf_macro" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d1d364d856d72a52b518c7669df864bdf177de242bcb5b45369ec33190c91c3" +checksum = "95d18335301db6e21d35b955cfb58327f3ede99f831a84cad5a07a59c6b1ec5d" dependencies = [ "proc-macro2", "quote", @@ -4191,10 +4251,11 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.19.2" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c51a52ed6686dd62c320f9b89299e9dfb46f730c7a48e635c19f21d116cb1439" +checksum = "7a8325f63a7d4774dd041e363b2409ed1c5cbbd0f867795e661df066b2b0a581" dependencies = [ + "autocfg", "bytes", "libc", "memchr", @@ -4286,9 +4347,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.21" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc6b8ad3567499f98a1db7a752b07a7c8c7c7c34c332ec00effb2b0027974b7c" +checksum = "11c75893af559bc8e10716548bdef5cb2b983f8e637db9d0e15126b61b484ee2" dependencies = [ "proc-macro2", "quote", @@ -4297,9 +4358,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.27" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7709595b8878a4965ce5e87ebf880a7d39c9afc6837721b21a5a816a8117d921" +checksum = "7b7358be39f2f274f322d2aaed611acc57f382e8eb1e5b48cb9ae30933495ce7" dependencies = [ "once_cell", "valuable", @@ -4318,13 +4379,13 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.11" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bc28f93baff38037f64e6f43d34cfa1605f27a49c34e8a04c5e78b0babf2596" +checksum = "60db860322da191b40952ad9affe65ea23e7dd6a5c442c2c42865810c6ab8e6b" dependencies = [ "ansi_term", - "lazy_static", "matchers", + "once_cell", "regex", "sharded-slab", "smallvec", @@ -4348,9 +4409,9 @@ checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" [[package]] name = "ucd-trie" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56dee185309b50d1f11bfedef0fe6d036842e3fb77413abef29f8f8d1c5d4c1c" +checksum = "89570599c4fe5585de2b388aab47e99f7fa4e9238a1399f707a02e356058141c" [[package]] name = "unicase" @@ -4369,15 +4430,15 @@ checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" [[package]] name = "unicode-ident" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bd2fe26506023ed7b5e1e315add59d6f584c621d037f9368fea9cfb988f368c" +checksum = "15c61ba63f9235225a22310255a29b806b907c9b8c964bcbd0a2c70f3f2deea7" [[package]] name = "unicode-normalization" -version = "0.1.19" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54590932941a9e9266f0832deed84ebe1bf2e4c9e4a3554d393d18f5e854bf9" +checksum = "854cbdc4f7bc6ae19c820d44abdc3277ac3e1b2b93db20a636825d9322fb60e6" dependencies = [ "tinyvec", ] @@ -4458,7 +4519,7 @@ dependencies = [ "crypto", "itertools", "logging", - "parity-scale-codec", + "parity-scale-codec 3.1.5", "serialization", "thiserror", ] @@ -4539,9 +4600,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.81" +version = "0.2.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c53b543413a17a202f4be280a7e5c62a1c69345f5de525ee64f8cfdbc954994" +checksum = "fc7652e3f6c4706c8d9cd54832c4a4ccb9b5336e2c3bd154d5cccfbf1c1f5f7d" dependencies = [ "cfg-if", "serde", @@ -4551,13 +4612,13 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.81" +version = "0.2.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5491a68ab4500fa6b4d726bd67408630c3dbe9c4fe7bda16d5c82a1fd8c7340a" +checksum = "662cd44805586bd52971b9586b1df85cdbbd9112e4ef4d8f41559c334dc6ac3f" dependencies = [ "bumpalo", - "lazy_static", "log", + "once_cell", "proc-macro2", "quote", "syn", @@ -4566,9 +4627,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.31" +version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de9a9cec1733468a8c657e57fa2413d2ae2c0129b95e87c5b72b8ace4d13f31f" +checksum = "fa76fb221a1f8acddf5b54ace85912606980ad661ac7a503b4570ffd3a624dad" dependencies = [ "cfg-if", "js-sys", @@ -4578,9 +4639,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.81" +version = "0.2.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c441e177922bc58f1e12c022624b6216378e5febc2f0533e41ba443d505b80aa" +checksum = "b260f13d3012071dfb1512849c033b1925038373aea48ced3012c09df952c602" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4588,9 +4649,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.81" +version = "0.2.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d94ac45fcf608c1f45ef53e748d35660f168490c10b23704c7779ab8f5c3048" +checksum = "5be8e654bdd9b79216c2929ab90721aa82faf65c48cdf08bdc4e7f51357b80da" dependencies = [ "proc-macro2", "quote", @@ -4601,9 +4662,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.81" +version = "0.2.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a89911bd99e5f3659ec4acf9c4d93b0a90fe4a2a11f15328472058edc5261be" +checksum = "6598dd0bd3c7d51095ff6531a5b23e02acdc81804e30d8f07afb77b7215a140a" [[package]] name = "wasm-timer" @@ -4622,9 +4683,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.58" +version = "0.3.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fed94beee57daf8dd7d51f2b15dc2bcde92d7a72304cdf662a4371008b71b90" +checksum = "ed055ab27f941423197eb86b2035720b1a3ce40504df082cac2ecc6ed73335a1" dependencies = [ "js-sys", "wasm-bindgen", @@ -4642,9 +4703,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.22.3" +version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d8de8415c823c8abd270ad483c6feeac771fad964890779f9a8cb24fbbc1bf" +checksum = "f1c760f0d366a6c24a02ed7816e23e691f5d92291f94d15e836006fd11b04daf" dependencies = [ "webpki", ] @@ -4712,6 +4773,19 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbedf6db9096bc2364adce0ae0aa636dcd89f3c3f2cd67947062aaf0ca2a10ec" +dependencies = [ + "windows_aarch64_msvc 0.32.0", + "windows_i686_gnu 0.32.0", + "windows_i686_msvc 0.32.0", + "windows_x86_64_gnu 0.32.0", + "windows_x86_64_msvc 0.32.0", +] + [[package]] name = "windows" version = "0.34.0" @@ -4738,6 +4812,12 @@ dependencies = [ "windows_x86_64_msvc 0.36.1", ] +[[package]] +name = "windows_aarch64_msvc" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8e92753b1c443191654ec532f14c199742964a061be25d77d7a96f09db20bf5" + [[package]] name = "windows_aarch64_msvc" version = "0.34.0" @@ -4750,6 +4830,12 @@ version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" +[[package]] +name = "windows_i686_gnu" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a711c68811799e017b6038e0922cb27a5e2f43a2ddb609fe0b6f3eeda9de615" + [[package]] name = "windows_i686_gnu" version = "0.34.0" @@ -4762,6 +4848,12 @@ version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" +[[package]] +name = "windows_i686_msvc" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "146c11bb1a02615db74680b32a68e2d61f553cc24c4eb5b4ca10311740e44172" + [[package]] name = "windows_i686_msvc" version = "0.34.0" @@ -4774,6 +4866,12 @@ version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" +[[package]] +name = "windows_x86_64_gnu" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c912b12f7454c6620635bbff3450962753834be2a594819bd5e945af18ec64bc" + [[package]] name = "windows_x86_64_gnu" version = "0.34.0" @@ -4786,6 +4884,12 @@ version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" +[[package]] +name = "windows_x86_64_msvc" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "504a2476202769977a040c6364301a3f65d0cc9e3fb08600b2bda150a0488316" + [[package]] name = "windows_x86_64_msvc" version = "0.34.0" @@ -4798,6 +4902,12 @@ version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" +[[package]] +name = "wyz" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85e60b0d1b5f99db2556934e21937020776a5d31520bf169e851ac44e6420214" + [[package]] name = "wyz" version = "0.5.0" @@ -4820,9 +4930,9 @@ dependencies = [ [[package]] name = "zeroize" -version = "1.4.3" +version = "1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d68d9dcec5f9b43a30d38c49f91dfedfaac384cb8f085faca366c26207dd1619" +checksum = "c394b5bd0c6f669e7275d9c20aa90ae064cb22e75a1cad54e1b34088034b149f" dependencies = [ "zeroize_derive", ] diff --git a/Cargo.toml b/Cargo.toml index 18f98b0450..c634b2c467 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,7 @@ members = [ "chainstate", # code on chainstate of blocks and transactions "script", # bitcoin script and its interfaces "logging", # logging engine and its interfaces + "mempool", # mempool interface and implementation "p2p", # p2p communication interfaces and protocols "rpc", # rpc abstraction and implementation "serialization", # full featured serialization interfaces and implementations @@ -40,6 +41,7 @@ default-members = [ "chainstate", "script", "logging", + "mempool", "p2p", "rpc", "serialization", @@ -61,6 +63,7 @@ chainstate = { path = "chainstate"} chainstate-types = { path = "chainstate-types"} script = { path = "script"} logging = { path = "logging"} +mempool = { path = "mempool"} p2p = { path = "p2p"} rpc = { path = "rpc"} serialization = { path = "serialization"} diff --git a/common/Cargo.toml b/common/Cargo.toml index 4786214592..16d7a047a4 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -23,6 +23,7 @@ sscanf = "0.2" static_assertions = "1.1" thiserror = "1.0" hex = "0.4" +rand = "0.8.5" # for fixed_hash arbitrary = "1.1" diff --git a/common/src/chain/transaction/input.rs b/common/src/chain/transaction/input.rs index bee1185350..f2e20cb6b8 100644 --- a/common/src/chain/transaction/input.rs +++ b/common/src/chain/transaction/input.rs @@ -52,6 +52,15 @@ impl From> for OutPointSourceId { } } +impl OutPointSourceId { + pub fn get_tx_id(&self) -> Option<&Id> { + match self { + OutPointSourceId::Transaction(id) => Some(id), + _ => None, + } + } +} + #[derive(Debug, Clone, PartialEq, Eq, Encode, Decode)] pub struct OutPoint { id: OutPointSourceId, diff --git a/common/src/primitives/amount.rs b/common/src/primitives/amount.rs index b1af89c6d2..4b9bc4e370 100644 --- a/common/src/primitives/amount.rs +++ b/common/src/primitives/amount.rs @@ -15,6 +15,7 @@ #![allow(clippy::eq_op)] +use rand::Rng; use serialization::{Decode, Encode}; use std::iter::Sum; @@ -129,6 +130,10 @@ impl Amount { atoms_str.parse::().ok().map(|v| Amount { val: v }) } } + + pub fn random(range: std::ops::RangeInclusive) -> Amount { + Amount::from_atoms(rand::thread_rng().gen_range(range.start().val..=range.end().val)) + } } impl From for u128 { diff --git a/mempool/Cargo.toml b/mempool/Cargo.toml new file mode 100644 index 0000000000..e510a9c86c --- /dev/null +++ b/mempool/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "mempool" +version = "0.1.0" +edition = "2021" +license = "MIT" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +parity-scale-codec = {version = "2.3.1", features = ["derive", "chain-error"]} +serialization = { path = '../serialization' } +common = { path = '../common' } +anyhow = "1.0" +thiserror = "1.0" + +[dev-dependencies] +rand = "0.8.4" diff --git a/mempool/src/lib.rs b/mempool/src/lib.rs new file mode 100644 index 0000000000..167730954a --- /dev/null +++ b/mempool/src/lib.rs @@ -0,0 +1,3 @@ +#![deny(clippy::clone_on_ref_ptr)] + +pub mod pool; diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs new file mode 100644 index 0000000000..dcc4ff6284 --- /dev/null +++ b/mempool/src/pool.rs @@ -0,0 +1,1022 @@ +use std::cmp::Ord; +use std::cmp::Ordering; +use std::collections::BTreeMap; +use std::collections::BTreeSet; +use std::collections::HashMap; +use std::fmt::Debug; +use std::rc::Rc; + +use serialization::Encode; +use thiserror::Error; + +use common::chain::transaction::Transaction; +use common::chain::transaction::TxInput; +use common::chain::OutPoint; +use common::primitives::amount::Amount; +use common::primitives::Id; +use common::primitives::Idable; +use common::primitives::H256; + +// TODO this willbe defined elsewhere (some of limits.rs file) +const MAX_BLOCK_SIZE_BYTES: usize = 1_000_000; + +const MEMPOOL_MAX_TXS: usize = 1_000_000; + +pub trait Mempool { + fn create(chain_state: C) -> Self; + fn add_transaction(&mut self, tx: Transaction) -> Result<(), MempoolError>; + fn get_all(&self) -> Vec<&Transaction>; + fn contains_transaction(&self, tx: &Id) -> bool; + fn drop_transaction(&mut self, tx: &Id); + fn new_tip_set(&mut self) -> Result<(), MempoolError>; +} + +pub trait ChainState { + fn contains_outpoint(&self, outpoint: &OutPoint) -> bool; +} + +#[derive(Debug, PartialEq, Eq, Clone)] +struct TxMempoolEntry { + tx: Transaction, + fee: Amount, + parents: BTreeSet>, +} + +trait TryGetFee { + fn try_get_fee(&self) -> Option; +} + +//TODO this should really be sum of inputs minus sum of outputs +//But we don't yet have a way of summing of the inputs +impl TryGetFee for Transaction { + fn try_get_fee(&self) -> Option { + self.outputs().iter().map(|output| output.value()).sum::>() + } +} + +impl TxMempoolEntry { + fn new(tx: Transaction, pool: &MempoolStore) -> Option { + let parents = tx + .inputs() + .iter() + .filter_map(|input| { + pool.txs_by_id + .get(&input.outpoint().tx_id().get_tx_id().expect("Not coinbase").get()) + }) + .cloned() + .collect::>(); + let fee = tx.try_get_fee()?; + + Some(Self { tx, fee, parents }) + } + + fn is_replaceable(&self) -> bool { + self.tx.is_replaceable() + || self.unconfirmed_ancestors().iter().any(|ancestor| ancestor.tx.is_replaceable()) + } + + fn unconfirmed_ancestors(&self) -> BTreeSet> { + let mut visited = BTreeSet::new(); + self.unconfirmed_ancestors_inner(&mut visited); + visited + } + + fn unconfirmed_ancestors_inner(&self, visited: &mut BTreeSet>) { + for parent in self.parents.iter() { + if visited.contains(parent) { + break; + } else { + visited.insert(Rc::clone(parent)); + parent.unconfirmed_ancestors_inner(visited); + } + } + } + + fn get_fee(&self) -> Amount { + self.fee + } +} + +impl PartialOrd for TxMempoolEntry { + fn partial_cmp(&self, other: &Self) -> Option { + Some(other.get_fee().cmp(&self.get_fee())) + } +} + +impl Ord for TxMempoolEntry { + fn cmp(&self, other: &Self) -> Ordering { + other.get_fee().cmp(&self.get_fee()) + } +} + +#[derive(Debug)] +pub struct MempoolImpl { + store: MempoolStore, + chain_state: C, +} + +#[derive(Debug)] +struct MempoolStore { + txs_by_id: HashMap>, + txs_by_fee: BTreeSet>, + spender_txs: BTreeMap>, +} + +impl MempoolStore { + fn new() -> Self { + Self { + txs_by_fee: BTreeSet::new(), + txs_by_id: HashMap::new(), + spender_txs: BTreeMap::new(), + } + } + + // Checks whether the outpoint is to be created by an unconfirmed tx + fn contains_outpoint(&self, outpoint: &OutPoint) -> bool { + matches!(self.txs_by_id.get(&outpoint.tx_id().get_tx_id().expect("Not Coinbase").get()), + Some(entry) if entry.tx.outputs().len() > outpoint.output_index() as usize) + } + + fn add_tx(&mut self, tx: Transaction) -> Result<(), MempoolError> { + let id = tx.get_id().get(); + let entry = TxMempoolEntry::new(tx, self) + .ok_or_else(|| MempoolError::from(TxValidationError::TransactionFeeOverflow))?; + let entry = Rc::new(entry); + self.txs_by_id.insert(id, Rc::clone(&entry)); + self.txs_by_fee.insert(Rc::clone(&entry)); + + for outpoint in entry.tx.inputs().iter().map(|input| input.outpoint()) { + self.spender_txs.insert(outpoint.clone(), Rc::clone(&entry)); + } + Ok(()) + } + + fn drop_tx(&mut self, tx_id: &Id) { + if let Some(tx) = self.txs_by_id.remove(&tx_id.get()) { + self.txs_by_fee.remove(&tx).then(|| ()).expect("Inconsistent mempool store"); + self.spender_txs.retain(|_, entry| entry.tx.get_id() != *tx_id) + } else { + assert!(!self.txs_by_fee.iter().any(|entry| entry.tx.get_id() == *tx_id)); + assert!(!self.spender_txs.iter().any(|(_, entry)| entry.tx.get_id() == *tx_id)); + } + } + + fn find_conflicting_tx(&self, outpoint: &OutPoint) -> Option> { + self.spender_txs.get(outpoint).cloned() + } +} + +#[derive(Debug, Error)] +pub enum MempoolError { + #[error("Mempool is full")] + MempoolFull, + #[error(transparent)] + TxValidationError(TxValidationError), +} + +#[derive(Debug, Error)] +pub enum TxValidationError { + #[error("No Inputs")] + NoInputs, + #[error("No Ouputs")] + NoOutputs, + #[error("DuplicateInputs")] + DuplicateInputs, + #[error("OutPointNotFound {outpoint:?}")] + OutPointNotFound { outpoint: OutPoint, tx: Transaction }, + #[error("ExceedsMaxBlockSize")] + ExceedsMaxBlockSize, + #[error("TransactionAlreadyInMempool")] + TransactionAlreadyInMempool, + #[error("ConflictWithIrreplaceableTransaction")] + ConflictWithIrreplaceableTransaction, + #[error("TransactionFeeOverflow")] + TransactionFeeOverflow, +} + +impl From for MempoolError { + fn from(e: TxValidationError) -> Self { + MempoolError::TxValidationError(e) + } +} + +impl MempoolImpl { + fn verify_inputs_available(&self, tx: &Transaction) -> Result<(), TxValidationError> { + tx.inputs() + .iter() + .map(TxInput::outpoint) + .find(|outpoint| !self.outpoint_available(outpoint)) + .map_or_else( + || Ok(()), + |outpoint| { + Err(TxValidationError::OutPointNotFound { + outpoint: outpoint.clone(), + tx: tx.clone(), + }) + }, + ) + } + + fn outpoint_available(&self, outpoint: &OutPoint) -> bool { + self.store.contains_outpoint(outpoint) || self.chain_state.contains_outpoint(outpoint) + } + + fn validate_transaction(&self, tx: &Transaction) -> Result<(), TxValidationError> { + if tx.inputs().is_empty() { + return Err(TxValidationError::NoInputs); + } + + if tx.outputs().is_empty() { + return Err(TxValidationError::NoOutputs); + } + + // TODO consier a MAX_MONEY check reminiscent of bitcoin's + // TODO consider rejecting non-standard transactions (for some definition of standard) + + let outpoints = tx.inputs().iter().map(|input| input.outpoint()).cloned(); + + if has_duplicate_entry(outpoints) { + return Err(TxValidationError::DuplicateInputs); + } + + if tx.encoded_size() > MAX_BLOCK_SIZE_BYTES { + return Err(TxValidationError::ExceedsMaxBlockSize); + } + + if self.contains_transaction(&tx.get_id()) { + return Err(TxValidationError::TransactionAlreadyInMempool); + } + + tx.inputs() + .iter() + .filter_map(|input| self.store.find_conflicting_tx(input.outpoint())) + .all(|tx| tx.is_replaceable()) + .then(|| ()) + .ok_or(TxValidationError::ConflictWithIrreplaceableTransaction)?; + + self.verify_inputs_available(tx)?; + + Ok(()) + } +} + +impl Mempool for MempoolImpl { + fn create(chain_state: C) -> Self { + Self { + store: MempoolStore::new(), + chain_state, + } + } + + fn new_tip_set(&mut self) -> Result<(), MempoolError> { + unimplemented!() + } + // + + fn add_transaction(&mut self, tx: Transaction) -> Result<(), MempoolError> { + // TODO (1). First, we need to decide on criteria for the Mempool to be considered full. Maybe number + // of transactions is not a good enough indicator. Consider checking mempool size as well + // TODO (2) What to do when the mempool is full. Instead of rejecting Do incoming transaction we probably want to evict a low-score transaction + if self.store.txs_by_fee.len() >= MEMPOOL_MAX_TXS { + return Err(MempoolError::MempoolFull); + } + self.validate_transaction(&tx)?; + self.store.add_tx(tx)?; + Ok(()) + } + + fn get_all(&self) -> Vec<&Transaction> { + self.store.txs_by_fee.iter().map(|mempool_tx| &mempool_tx.tx).collect() + } + + fn contains_transaction(&self, tx_id: &Id) -> bool { + self.store.txs_by_id.contains_key(&tx_id.get()) + } + + // TODO Consider returning an error + fn drop_transaction(&mut self, tx_id: &Id) { + self.store.drop_tx(tx_id); + } +} + +fn has_duplicate_entry(iter: T) -> bool +where + T: IntoIterator, + T::Item: Ord, +{ + let mut uniq = BTreeSet::new(); + iter.into_iter().any(move |x| !uniq.insert(x)) +} + +#[cfg(test)] +mod tests { + use super::*; + use common::chain::signature::inputsig::InputWitness; + use common::chain::transaction::{Destination, TxInput, TxOutput}; + use common::chain::OutPointSourceId; + use common::chain::OutputPurpose; + use rand::Rng; + + const DUMMY_WITNESS_MSG: &[u8] = b"dummy_witness_msg"; + + #[derive(PartialEq, Eq, PartialOrd, Ord, Clone)] + struct ValuedOutPoint { + outpoint: OutPoint, + value: Amount, + } + + fn valued_outpoint( + tx_id: &Id, + outpoint_index: u32, + output: &TxOutput, + ) -> ValuedOutPoint { + let outpoint_source_id = OutPointSourceId::Transaction(*tx_id); + let outpoint = OutPoint::new(outpoint_source_id, outpoint_index); + let value = output.value(); + ValuedOutPoint { outpoint, value } + } + + pub(crate) fn create_genesis_tx() -> Transaction { + const TOTAL_SUPPLY: u128 = 10_000_000_000_000; + let genesis_message = b"".to_vec(); + let outpoint_source_id = OutPointSourceId::Transaction(Id::new(H256::zero())); + let input = TxInput::new( + outpoint_source_id, + 0, + InputWitness::NoSignature(Some(genesis_message)), + ); + let output = TxOutput::new( + Amount::from_atoms(TOTAL_SUPPLY), + OutputPurpose::Transfer(Destination::AnyoneCanSpend), + ); + Transaction::new(0, vec![input], vec![output], 0) + .expect("Failed to create genesis coinbase transaction") + } + + impl TxMempoolEntry { + fn outpoints_created(&self) -> BTreeSet { + let id = self.tx.get_id(); + std::iter::repeat(id) + .zip(self.tx.outputs().iter().enumerate()) + .map(|(id, (index, output))| valued_outpoint(&id, index as u32, output)) + .collect() + } + } + + impl MempoolStore { + fn unconfirmed_outpoints(&self) -> BTreeSet { + self.txs_by_id + .values() + .cloned() + .flat_map(|entry| entry.outpoints_created()) + .collect() + } + } + + impl MempoolImpl { + fn available_outpoints(&self) -> BTreeSet { + self.store + .unconfirmed_outpoints() + .into_iter() + .chain(self.chain_state.confirmed_outpoints()) + .collect() + } + } + + #[derive(Debug, Clone)] + pub(crate) struct ChainStateMock { + txs: HashMap, + outpoints: BTreeSet, + } + + impl ChainStateMock { + pub(crate) fn new() -> Self { + let genesis_tx = create_genesis_tx(); + let outpoint_source_id = OutPointSourceId::Transaction(genesis_tx.get_id()); + let outpoints = genesis_tx + .outputs() + .iter() + .enumerate() + .map(|(index, _)| OutPoint::new(outpoint_source_id.clone(), index as u32)) + .collect(); + Self { + txs: std::iter::once((genesis_tx.get_id().get(), genesis_tx)).collect(), + outpoints, + } + } + + fn unspent_outpoints(&self) -> BTreeSet { + self.outpoints + .iter() + .map(|outpoint| { + let value = + self.get_outpoint_value(outpoint).expect("Inconsistent Chain State"); + ValuedOutPoint { + outpoint: outpoint.clone(), + value, + } + }) + .collect() + } + + fn confirmed_txs(&self) -> &HashMap { + &self.txs + } + + fn get_outpoint_value(&self, outpoint: &OutPoint) -> Result { + self.txs + .get(&outpoint.tx_id().get_tx_id().expect("Not Coinbase").get()) + .ok_or_else(|| anyhow::anyhow!("tx for outpoint sought in chain state, not found")) + .and_then(|tx| { + tx.outputs() + .get(outpoint.output_index() as usize) + .ok_or_else(|| anyhow::anyhow!("outpoint index out of bounds")) + .map(|output| output.value()) + }) + } + + fn confirmed_outpoints(&self) -> BTreeSet { + self.txs + .values() + .flat_map(|tx| { + std::iter::repeat(tx.get_id()) + .zip(tx.outputs().iter().enumerate()) + .map(move |(tx_id, (i, output))| valued_outpoint(&tx_id, i as u32, output)) + }) + .collect() + } + } + + impl ChainState for ChainStateMock { + fn contains_outpoint(&self, outpoint: &OutPoint) -> bool { + self.outpoints.iter().any(|value| *value == *outpoint) + } + } + + struct TxGenerator { + coin_pool: BTreeSet, + num_inputs: usize, + num_outputs: usize, + } + + impl TxGenerator { + fn new( + mempool: &MempoolImpl, + num_inputs: usize, + num_outputs: usize, + ) -> Self { + let unconfirmed_outputs = BTreeSet::new(); + Self::create_tx_generator( + &mempool.chain_state, + &unconfirmed_outputs, + num_inputs, + num_outputs, + ) + } + + fn new_with_unconfirmed( + mempool: &MempoolImpl, + num_inputs: usize, + num_outputs: usize, + ) -> Self { + let unconfirmed_outputs = mempool.available_outpoints(); + Self::create_tx_generator( + &mempool.chain_state, + &unconfirmed_outputs, + num_inputs, + num_outputs, + ) + } + + fn create_tx_generator( + chain_state: &ChainStateMock, + unconfirmed_outputs: &BTreeSet, + num_inputs: usize, + num_outputs: usize, + ) -> Self { + let coin_pool = chain_state + .unspent_outpoints() + .iter() + .chain(unconfirmed_outputs) + .cloned() + .collect(); + + Self { + coin_pool, + num_inputs, + num_outputs, + } + } + + fn generate_tx(&mut self) -> anyhow::Result { + let valued_inputs = self.generate_tx_inputs(); + let outputs = self.generate_tx_outputs(&valued_inputs)?; + let locktime = 0; + let flags = 0; + let (inputs, _): (Vec, Vec) = valued_inputs.into_iter().unzip(); + let spent_outpoints = + inputs.iter().map(|input| input.outpoint()).collect::>(); + self.coin_pool.retain(|outpoint| { + !spent_outpoints.iter().any(|spent| **spent == outpoint.outpoint) + }); + let tx = Transaction::new(flags, inputs, outputs.clone(), locktime) + .map_err(anyhow::Error::from)?; + self.coin_pool.extend( + std::iter::repeat(tx.get_id()) + .zip(outputs.iter().enumerate()) + .map(|(id, (i, output))| valued_outpoint(&id, i as u32, output)), + ); + Ok(tx) + } + + fn generate_replaceable_tx(mut self) -> anyhow::Result { + let valued_inputs = self.generate_tx_inputs(); + let outputs = self.generate_tx_outputs(&valued_inputs)?; + let locktime = 0; + let flags = 1; + let (inputs, _values): (Vec, Vec) = valued_inputs.into_iter().unzip(); + let tx = Transaction::new(flags, inputs, outputs, locktime)?; + assert!(tx.is_replaceable()); + Ok(tx) + } + + fn generate_tx_inputs(&mut self) -> Vec<(TxInput, Amount)> { + std::iter::repeat(()) + .take(self.num_inputs) + .filter_map(|_| self.generate_input().ok()) + .collect() + } + + fn generate_tx_outputs( + &self, + inputs: &[(TxInput, Amount)], + ) -> anyhow::Result> { + if self.num_outputs == 0 { + return Ok(vec![]); + } + + let inputs: Vec<_> = inputs.to_owned(); + let (inputs, values): (Vec, Vec) = inputs.into_iter().unzip(); + if inputs.is_empty() { + return Ok(vec![]); + } + let max_spend = + values.into_iter().sum::>().expect("Overflow in sum of input values"); + + let mut left_to_spend = max_spend; + let mut outputs = Vec::new(); + + let max_output_value = Amount::from_atoms(1_000); + for _ in 0..self.num_outputs { + let max_output_value = std::cmp::min( + (left_to_spend / 2).expect("division failed"), + max_output_value, + ); + if max_output_value == Amount::from_atoms(0) { + return Err(anyhow::Error::msg("No more funds to spend")); + } + let value = Amount::random(Amount::from_atoms(1)..=max_output_value); + outputs.push(TxOutput::new( + value, + OutputPurpose::Transfer(Destination::AnyoneCanSpend), + )); + left_to_spend = (left_to_spend - value).expect("subtraction failed"); + } + Ok(outputs) + } + + fn generate_input(&self) -> anyhow::Result<(TxInput, Amount)> { + let ValuedOutPoint { outpoint, value } = self.random_unspent_outpoint()?; + Ok(( + TxInput::new( + outpoint.tx_id(), + outpoint.output_index(), + InputWitness::NoSignature(None), + ), + value, + )) + } + + fn random_unspent_outpoint(&self) -> anyhow::Result { + let num_outpoints = self.coin_pool.len(); + (num_outpoints > 0) + .then(|| { + let index = rand::thread_rng().gen_range(0..num_outpoints); + self.coin_pool + .iter() + .nth(index) + .cloned() + .expect("Outpoint set should not be empty") + }) + .ok_or_else(|| anyhow::anyhow!("no outpoints left")) + } + } + + #[test] + fn add_single_tx() -> anyhow::Result<()> { + let mut mempool = MempoolImpl::create(ChainStateMock::new()); + + let genesis_tx = mempool + .chain_state + .confirmed_txs() + .values() + .next() + .expect("genesis tx not found"); + + let outpoint_source_id = OutPointSourceId::Transaction(genesis_tx.get_id()); + let input = TxInput::new( + outpoint_source_id, + 0, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ); + let outputs = spend_input(&mempool, &input)?; + + let flags = 0; + let inputs = vec![input]; + let locktime = 0; + let tx = Transaction::new(flags, inputs, outputs, locktime) + .map_err(|e| anyhow::anyhow!("failed to create transaction: {:?}", e))?; + + let tx_clone = tx.clone(); + let tx_id = tx.get_id(); + mempool.add_transaction(tx)?; + assert!(mempool.contains_transaction(&tx_id)); + let all_txs = mempool.get_all(); + assert_eq!(all_txs, vec![&tx_clone]); + mempool.drop_transaction(&tx_id); + assert!(!mempool.contains_transaction(&tx_id)); + let all_txs = mempool.get_all(); + assert_eq!(all_txs, Vec::<&Transaction>::new()); + Ok(()) + } + + // The "fees" now a are calculated as sum of the outputs + // This test creates transactions with a single input and a single output to check that the + // mempool sorts txs by fee + #[test] + fn txs_sorted() -> anyhow::Result<()> { + let chain_state = ChainStateMock::new(); + let num_inputs = 1; + let num_outputs = 1; + let mut mempool = MempoolImpl::create(chain_state); + let mut tx_generator = TxGenerator::new(&mempool, num_inputs, num_outputs); + let target_txs = 100; + + for _ in 0..target_txs { + match tx_generator.generate_tx() { + Ok(tx) => { + mempool.add_transaction(tx.clone())?; + } + _ => break, + } + } + + let fees = mempool + .get_all() + .iter() + .map(|tx| tx.outputs().first().expect("TX should have exactly one output").value()) + .collect::>(); + let mut fees_sorted = fees.clone(); + fees_sorted.sort_by(|a, b| b.cmp(a)); + assert_eq!(fees, fees_sorted); + Ok(()) + } + + #[test] + fn tx_no_inputs() -> anyhow::Result<()> { + let mut mempool = setup(); + let num_inputs = 0; + let num_outputs = 1; + let tx = TxGenerator::new(&mempool, num_inputs, num_outputs) + .generate_tx() + .expect("generate_tx failed"); + assert!(matches!( + mempool.add_transaction(tx), + Err(MempoolError::TxValidationError(TxValidationError::NoInputs)) + )); + Ok(()) + } + + fn setup() -> MempoolImpl { + MempoolImpl::create(ChainStateMock::new()) + } + + #[test] + fn tx_no_outputs() -> anyhow::Result<()> { + let mut mempool = setup(); + let num_inputs = 1; + let num_outputs = 0; + let tx = TxGenerator::new(&mempool, num_inputs, num_outputs) + .generate_tx() + .expect("generate_tx failed"); + assert!(matches!( + mempool.add_transaction(tx), + Err(MempoolError::TxValidationError( + TxValidationError::NoOutputs + )) + )); + Ok(()) + } + + fn spend_input( + mempool: &MempoolImpl, + input: &TxInput, + ) -> anyhow::Result> { + let outpoint = input.outpoint(); + let input_value = mempool.chain_state.get_outpoint_value(outpoint).or_else(|_| { + mempool + .available_outpoints() + .iter() + .find_map(|valued_outpoint| { + (valued_outpoint.outpoint == *outpoint).then(|| valued_outpoint.value) + }) + .ok_or_else(|| anyhow::anyhow!("No such unconfirmed output")) + })?; + let output_value = (input_value / 2).expect("failed to divide input"); + let output_pay = TxOutput::new( + output_value, + OutputPurpose::Transfer(Destination::AnyoneCanSpend), + ); + let output_change_amount = (input_value - output_value).expect("underflow"); + let output_change = TxOutput::new( + output_change_amount, + OutputPurpose::Transfer(Destination::AnyoneCanSpend), + ); + Ok(vec![output_pay, output_change]) + } + + #[test] + fn tx_duplicate_inputs() -> anyhow::Result<()> { + let mut mempool = MempoolImpl::create(ChainStateMock::new()); + + let genesis_tx = mempool + .chain_state + .confirmed_txs() + .values() + .next() + .expect("genesis tx not found"); + + let outpoint_source_id = OutPointSourceId::Transaction(genesis_tx.get_id()); + let input = TxInput::new( + outpoint_source_id.clone(), + 0, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ); + let witness = b"attempted_double_spend".to_vec(); + let duplicate_input = TxInput::new( + outpoint_source_id, + 0, + InputWitness::NoSignature(Some(witness)), + ); + let outputs = spend_input(&mempool, &input)?; + let inputs = vec![input, duplicate_input]; + let flags = 0; + let locktime = 0; + let tx = Transaction::new(flags, inputs, outputs, locktime) + .map_err(|e| anyhow::anyhow!("failed to create transaction: {:?}", e))?; + + assert!(matches!( + mempool.add_transaction(tx), + Err(MempoolError::TxValidationError( + TxValidationError::DuplicateInputs + )) + )); + Ok(()) + } + + #[test] + fn tx_already_in_mempool() -> anyhow::Result<()> { + let mut mempool = MempoolImpl::create(ChainStateMock::new()); + + let genesis_tx = mempool + .chain_state + .confirmed_txs() + .values() + .next() + .expect("genesis tx not found"); + + let outpoint_source_id = OutPointSourceId::Transaction(genesis_tx.get_id()); + let input = TxInput::new( + outpoint_source_id, + 0, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ); + let outputs = spend_input(&mempool, &input)?; + + let flags = 0; + let inputs = vec![input]; + let locktime = 0; + let tx = Transaction::new(flags, inputs, outputs, locktime)?; + + mempool.add_transaction(tx.clone())?; + assert!(matches!( + mempool.add_transaction(tx), + Err(MempoolError::TxValidationError( + TxValidationError::TransactionAlreadyInMempool + )) + )); + Ok(()) + } + + #[test] + fn outpoint_not_found() -> anyhow::Result<()> { + let mut mempool = MempoolImpl::create(ChainStateMock::new()); + + let genesis_tx = mempool + .chain_state + .confirmed_txs() + .values() + .next() + .expect("genesis tx not found"); + + let outpoint_source_id = OutPointSourceId::Transaction(genesis_tx.get_id()); + let good_input = TxInput::new( + outpoint_source_id.clone(), + 0, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ); + let outputs = spend_input(&mempool, &good_input)?; + let bad_outpoint_index = 1; + let bad_input = TxInput::new( + outpoint_source_id, + bad_outpoint_index, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ); + + let flags = 0; + let inputs = vec![bad_input]; + let locktime = 0; + let tx = Transaction::new(flags, inputs, outputs, locktime)?; + + assert!(matches!( + mempool.add_transaction(tx), + Err(MempoolError::TxValidationError( + TxValidationError::OutPointNotFound { .. } + )) + )); + + Ok(()) + } + + #[test] + fn tx_too_big() -> anyhow::Result<()> { + let mut mempool = setup(); + let num_inputs = 1; + let num_outputs = 400_000; + let tx = TxGenerator::new(&mempool, num_inputs, num_outputs) + .generate_tx() + .expect("generate_tx failed"); + assert!(matches!( + mempool.add_transaction(tx), + Err(MempoolError::TxValidationError( + TxValidationError::ExceedsMaxBlockSize + )) + )); + Ok(()) + } + + #[test] + fn tx_replace() -> anyhow::Result<()> { + let mut mempool = setup(); + let num_inputs = 1; + let num_outputs = 1; + let tx = TxGenerator::new(&mempool, num_inputs, num_outputs) + .generate_replaceable_tx() + .expect("generate_replaceable_tx"); + mempool.add_transaction(tx)?; + + let tx = TxGenerator::new(&mempool, num_inputs, num_outputs) + .generate_tx() + .expect("generate_tx_failed"); + + mempool.add_transaction(tx)?; + Ok(()) + } + + #[test] + fn tx_replace_child() -> anyhow::Result<()> { + let mut mempool = setup(); + let num_inputs = 1; + let num_outputs = 1; + let tx = TxGenerator::new_with_unconfirmed(&mempool, num_inputs, num_outputs) + .generate_replaceable_tx() + .expect("generate_replaceable_tx"); + mempool.add_transaction(tx.clone())?; + + let outpoint_source_id = OutPointSourceId::Transaction(tx.get_id()); + let child_tx_input = TxInput::new( + outpoint_source_id, + 0, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ); + // We want to test that even though it doesn't signal replaceability directly, the child tx is replaceable because it's parent signalled replaceability + // replaced + let flags = 0; + let locktime = 0; + let outputs = spend_input(&mempool, &child_tx_input)?; + let inputs = vec![child_tx_input]; + let child_tx = Transaction::new(flags, inputs, outputs, locktime)?; + mempool.add_transaction(child_tx)?; + Ok(()) + } + + fn tx_spend_input( + mempool: &MempoolImpl, + input: TxInput, + flags: u32, + locktime: u32, + ) -> anyhow::Result { + let input_value = mempool + .available_outpoints() + .iter() + .find_map(|valued_outpoint| { + (valued_outpoint.outpoint == *input.outpoint()).then(|| valued_outpoint.value) + }) + .ok_or_else(|| anyhow::anyhow!("No such unconfirmed output"))?; + let output_value = (input_value / 2).expect("failed to divide input"); + let output_pay = TxOutput::new( + output_value, + OutputPurpose::Transfer(Destination::AnyoneCanSpend), + ); + let output_change_amount = (input_value - output_value).expect("underflow"); + let output_change = TxOutput::new( + output_change_amount, + OutputPurpose::Transfer(Destination::AnyoneCanSpend), + ); + let outputs = vec![output_pay, output_change]; + let inputs = vec![input]; + Transaction::new(flags, inputs, outputs, locktime).map_err(Into::into) + } + + #[test] + fn one_ancestor_signal_is_enough() -> anyhow::Result<()> { + let mut mempool = setup(); + let num_inputs = 1; + let num_outputs = 2; + let tx = TxGenerator::new_with_unconfirmed(&mempool, num_inputs, num_outputs) + .generate_tx() + .expect("generate_replaceable_tx"); + + mempool.add_transaction(tx.clone())?; + + let outpoint_source_id = OutPointSourceId::Transaction(tx.get_id()); + let replaceable_input = TxInput::new( + outpoint_source_id.clone(), + 0, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ); + let other_input = TxInput::new( + outpoint_source_id, + 1, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ); + + let flags_replaceable = 1; + let flags_irreplaceable = 0; + let locktime = 0; + + let ancestor_with_signal = + tx_spend_input(&mempool, replaceable_input, flags_replaceable, locktime)?; + + let ancestor_without_signal = + tx_spend_input(&mempool, other_input, flags_irreplaceable, locktime)?; + + mempool.add_transaction(ancestor_with_signal.clone())?; + mempool.add_transaction(ancestor_without_signal.clone())?; + + let input_with_replaceable_parent = TxInput::new( + OutPointSourceId::Transaction(ancestor_with_signal.get_id()), + 0, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ); + let input_with_irreplaceable_parent = TxInput::new( + OutPointSourceId::Transaction(ancestor_without_signal.get_id()), + 0, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ); + let dummy_output = TxOutput::new( + Amount::from_atoms(0), + OutputPurpose::Transfer(Destination::AnyoneCanSpend), + ); + + let replaced_tx = Transaction::new( + flags_irreplaceable, + vec![input_with_irreplaceable_parent.clone(), input_with_replaceable_parent], + vec![dummy_output.clone()], + locktime, + )?; + + mempool.add_transaction(replaced_tx)?; + + let replacing_tx = Transaction::new( + flags_irreplaceable, + vec![input_with_irreplaceable_parent], + vec![dummy_output], + locktime, + )?; + + mempool.add_transaction(replacing_tx)?; + + Ok(()) + } +} From 35930a23cf904ec26adb89e1fc591551a9f3dabc Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Fri, 11 Feb 2022 22:04:02 +0700 Subject: [PATCH 005/153] Move get outpoint value (Test pass) --- mempool/src/pool.rs | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index dcc4ff6284..dae36151ab 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -33,6 +33,7 @@ pub trait Mempool { pub trait ChainState { fn contains_outpoint(&self, outpoint: &OutPoint) -> bool; + fn get_outpoint_value(&self, outpoint: &OutPoint) -> Result; } #[derive(Debug, PartialEq, Eq, Clone)] @@ -183,7 +184,10 @@ pub enum TxValidationError { #[error("DuplicateInputs")] DuplicateInputs, #[error("OutPointNotFound {outpoint:?}")] - OutPointNotFound { outpoint: OutPoint, tx: Transaction }, + OutPointNotFound { + outpoint: OutPoint, + tx_id: Id, + }, #[error("ExceedsMaxBlockSize")] ExceedsMaxBlockSize, #[error("TransactionAlreadyInMempool")] @@ -211,7 +215,7 @@ impl MempoolImpl { |outpoint| { Err(TxValidationError::OutPointNotFound { outpoint: outpoint.clone(), - tx: tx.clone(), + tx_id: tx.get_id(), }) }, ) @@ -451,6 +455,20 @@ mod tests { fn contains_outpoint(&self, outpoint: &OutPoint) -> bool { self.outpoints.iter().any(|value| *value == *outpoint) } + + fn get_outpoint_value(&self, outpoint: &OutPoint) -> Result { + self.txs + .get(&outpoint.get_tx_id().get_tx_id().expect("Not coinbase").get()) + .ok_or(anyhow::anyhow!( + "tx for outpoint sought in chain state, not found" + )) + .and_then(|tx| { + tx.get_outputs() + .get(outpoint.get_output_index() as usize) + .ok_or(anyhow::anyhow!("outpoint index out of bounds")) + .map(|output| output.get_value()) + }) + } } struct TxGenerator { From 40cf3a0749c264ece8fa9a0601148af84a439162 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Mon, 14 Feb 2022 14:49:13 +0700 Subject: [PATCH 006/153] Order TxMempoolEntry by Hash, not Fee --- mempool/src/pool.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index dae36151ab..8d7624153c 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -92,21 +92,17 @@ impl TxMempoolEntry { } } } - - fn get_fee(&self) -> Amount { - self.fee - } } impl PartialOrd for TxMempoolEntry { fn partial_cmp(&self, other: &Self) -> Option { - Some(other.get_fee().cmp(&self.get_fee())) + Some(other.tx.get_id().get().cmp(&self.tx.get_id().get())) } } impl Ord for TxMempoolEntry { fn cmp(&self, other: &Self) -> Ordering { - other.get_fee().cmp(&self.get_fee()) + other.tx.get_id().get().cmp(&self.tx.get_id().get()) } } From 4444a8690eb12b8792cc06f8fe802606ed3dd564 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Mon, 14 Feb 2022 14:56:53 +0700 Subject: [PATCH 007/153] Fix DFS in unconfirmed ancestors_search --- mempool/src/pool.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 8d7624153c..2273ea0469 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -85,7 +85,7 @@ impl TxMempoolEntry { fn unconfirmed_ancestors_inner(&self, visited: &mut BTreeSet>) { for parent in self.parents.iter() { if visited.contains(parent) { - break; + continue; } else { visited.insert(Rc::clone(parent)); parent.unconfirmed_ancestors_inner(visited); From 82e3adf6f5d3413d5eaf0498f298a5ead9408447 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Mon, 14 Feb 2022 19:25:12 +0700 Subject: [PATCH 008/153] Allow multiple txs with the same fee --- mempool/src/pool.rs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 2273ea0469..56f42ce18e 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -115,14 +115,14 @@ pub struct MempoolImpl { #[derive(Debug)] struct MempoolStore { txs_by_id: HashMap>, - txs_by_fee: BTreeSet>, + txs_by_fee: BTreeMap>>, spender_txs: BTreeMap>, } impl MempoolStore { fn new() -> Self { Self { - txs_by_fee: BTreeSet::new(), + txs_by_fee: BTreeMap::new(), txs_by_id: HashMap::new(), spender_txs: BTreeMap::new(), } @@ -140,7 +140,7 @@ impl MempoolStore { .ok_or_else(|| MempoolError::from(TxValidationError::TransactionFeeOverflow))?; let entry = Rc::new(entry); self.txs_by_id.insert(id, Rc::clone(&entry)); - self.txs_by_fee.insert(Rc::clone(&entry)); + self.txs_by_fee.entry(entry.fee).or_default().insert(Rc::clone(&entry)); for outpoint in entry.tx.inputs().iter().map(|input| input.outpoint()) { self.spender_txs.insert(outpoint.clone(), Rc::clone(&entry)); @@ -149,11 +149,13 @@ impl MempoolStore { } fn drop_tx(&mut self, tx_id: &Id) { - if let Some(tx) = self.txs_by_id.remove(&tx_id.get()) { - self.txs_by_fee.remove(&tx).then(|| ()).expect("Inconsistent mempool store"); + if let Some(entry) = self.txs_by_id.remove(&tx_id.get()) { + self.txs_by_fee.entry(entry.fee).and_modify(|entries| { + entries.remove(&entry).then(|| ()).expect("Inconsistent mempool store") + }); self.spender_txs.retain(|_, entry| entry.tx.get_id() != *tx_id) } else { - assert!(!self.txs_by_fee.iter().any(|entry| entry.tx.get_id() == *tx_id)); + assert!(!self.txs_by_fee.values().flatten().any(|entry| entry.tx.get_id() == *tx_id)); assert!(!self.spender_txs.iter().any(|(_, entry)| entry.tx.get_id() == *tx_id)); } } @@ -286,7 +288,7 @@ impl Mempool for MempoolImpl { } fn get_all(&self) -> Vec<&Transaction> { - self.store.txs_by_fee.iter().map(|mempool_tx| &mempool_tx.tx).collect() + self.store.txs_by_fee.values().flatten().map(|entry| &entry.tx).collect() } fn contains_transaction(&self, tx_id: &Id) -> bool { From 736cd4b71bba8b77f3e171ca65cba86962870672 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Mon, 14 Feb 2022 19:31:11 +0700 Subject: [PATCH 009/153] Refactor fee calculation --- mempool/src/pool.rs | 101 +++++++++++++++++++++++++++++++------------- 1 file changed, 71 insertions(+), 30 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 56f42ce18e..4b46a68ca7 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -22,6 +22,33 @@ const MAX_BLOCK_SIZE_BYTES: usize = 1_000_000; const MEMPOOL_MAX_TXS: usize = 1_000_000; +impl TryGetFee for MempoolImpl { + fn try_get_fee(&self, tx: &Transaction) -> Result { + let inputs = tx + .inputs() + .iter() + .map(|input| { + let outpoint = input.outpoint(); + self.chain_state + .get_outpoint_value(outpoint) + .or_else(|_| self.store.get_unconfirmed_outpoint_value(outpoint)) + }) + .collect::, _>>()?; + let sum_inputs = inputs + .iter() + .cloned() + .sum::>() + .ok_or(TxValidationError::TransactionFeeOverflow)?; + let sum_outputs = tx + .outputs() + .iter() + .map(|output| output.value()) + .sum::>() + .ok_or(TxValidationError::TransactionFeeOverflow)?; + (sum_inputs - sum_outputs).ok_or(TxValidationError::TransactionFeeOverflow) + } +} + pub trait Mempool { fn create(chain_state: C) -> Self; fn add_transaction(&mut self, tx: Transaction) -> Result<(), MempoolError>; @@ -44,33 +71,26 @@ struct TxMempoolEntry { } trait TryGetFee { - fn try_get_fee(&self) -> Option; -} - -//TODO this should really be sum of inputs minus sum of outputs -//But we don't yet have a way of summing of the inputs -impl TryGetFee for Transaction { - fn try_get_fee(&self) -> Option { - self.outputs().iter().map(|output| output.value()).sum::>() - } + fn try_get_fee(&self, tx: &Transaction) -> Result; } impl TxMempoolEntry { - fn new(tx: Transaction, pool: &MempoolStore) -> Option { + fn new(tx: Transaction, pool: &MempoolImpl) -> Option { let parents = tx .inputs() .iter() .filter_map(|input| { - pool.txs_by_id + pool.store + .txs_by_id .get(&input.outpoint().tx_id().get_tx_id().expect("Not coinbase").get()) }) .cloned() .collect::>(); - let fee = tx.try_get_fee()?; + + let fee = pool.try_get_fee(&tx).ok()?; Some(Self { tx, fee, parents }) } - fn is_replaceable(&self) -> bool { self.tx.is_replaceable() || self.unconfirmed_ancestors().iter().any(|ancestor| ancestor.tx.is_replaceable()) @@ -134,10 +154,26 @@ impl MempoolStore { Some(entry) if entry.tx.outputs().len() > outpoint.output_index() as usize) } - fn add_tx(&mut self, tx: Transaction) -> Result<(), MempoolError> { - let id = tx.get_id().get(); - let entry = TxMempoolEntry::new(tx, self) - .ok_or_else(|| MempoolError::from(TxValidationError::TransactionFeeOverflow))?; + fn get_unconfirmed_outpoint_value( + &self, + outpoint: &OutPoint, + ) -> Result { + let tx_id = outpoint.tx_id().get_tx_id().expect("Not coinbase").clone(); + let err = || TxValidationError::OutPointNotFound { + outpoint: outpoint.clone(), + tx_id: tx_id.clone(), + }; + self.txs_by_id + .get(&tx_id.get()) + .ok_or_else(err) + .and_then(|entry| { + entry.tx.outputs().get(outpoint.output_index() as usize).ok_or_else(err) + }) + .map(|output| output.value()) + } + + fn add_tx(&mut self, entry: TxMempoolEntry) -> Result<(), MempoolError> { + let id = entry.tx.get_id().get(); let entry = Rc::new(entry); self.txs_by_id.insert(id, Rc::clone(&entry)); self.txs_by_fee.entry(entry.fee).or_default().insert(Rc::clone(&entry)); @@ -249,12 +285,17 @@ impl MempoolImpl { return Err(TxValidationError::TransactionAlreadyInMempool); } - tx.inputs() + let conflicts = tx + .inputs() .iter() .filter_map(|input| self.store.find_conflicting_tx(input.outpoint())) - .all(|tx| tx.is_replaceable()) - .then(|| ()) - .ok_or(TxValidationError::ConflictWithIrreplaceableTransaction)?; + .collect::>(); + + for entry in conflicts { + if !entry.is_replaceable() { + return Err(TxValidationError::ConflictWithIrreplaceableTransaction); + } + } self.verify_inputs_available(tx)?; @@ -283,7 +324,9 @@ impl Mempool for MempoolImpl { return Err(MempoolError::MempoolFull); } self.validate_transaction(&tx)?; - self.store.add_tx(tx)?; + let entry = TxMempoolEntry::new(tx, self) + .ok_or_else(|| MempoolError::from(TxValidationError::TransactionFeeOverflow))?; + self.store.add_tx(entry)?; Ok(()) } @@ -456,15 +499,13 @@ mod tests { fn get_outpoint_value(&self, outpoint: &OutPoint) -> Result { self.txs - .get(&outpoint.get_tx_id().get_tx_id().expect("Not coinbase").get()) - .ok_or(anyhow::anyhow!( - "tx for outpoint sought in chain state, not found" - )) + .get(&outpoint.tx_id().get_tx_id().expect("Not coinbase").get()) + .ok_or_else(|| anyhow::anyhow!("tx for outpoint sought in chain state, not found")) .and_then(|tx| { - tx.get_outputs() - .get(outpoint.get_output_index() as usize) - .ok_or(anyhow::anyhow!("outpoint index out of bounds")) - .map(|output| output.get_value()) + tx.outputs() + .get(outpoint.output_index() as usize) + .ok_or_else(|| anyhow::anyhow!("outpoint index out of bounds")) + .map(|output| output.value()) }) } } From 76e2d117b403a62ff0de9035db9024b9d1b05d54 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Mon, 14 Feb 2022 20:40:26 +0700 Subject: [PATCH 010/153] Support txs with prescribed fees in tests --- mempool/src/pool.rs | 71 +++++++++++++++++++++++++++++++-------------- 1 file changed, 50 insertions(+), 21 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 4b46a68ca7..327f9dcb43 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -91,6 +91,7 @@ impl TxMempoolEntry { Some(Self { tx, fee, parents }) } + fn is_replaceable(&self) -> bool { self.tx.is_replaceable() || self.unconfirmed_ancestors().iter().any(|ancestor| ancestor.tx.is_replaceable()) @@ -514,6 +515,7 @@ mod tests { coin_pool: BTreeSet, num_inputs: usize, num_outputs: usize, + tx_fee: Option, } impl TxGenerator { @@ -531,6 +533,11 @@ mod tests { ) } + fn with_fee(mut self, fee: Amount) -> Self { + self.tx_fee = Some(fee); + self + } + fn new_with_unconfirmed( mempool: &MempoolImpl, num_inputs: usize, @@ -562,12 +569,13 @@ mod tests { coin_pool, num_inputs, num_outputs, + tx_fee: None, } } fn generate_tx(&mut self) -> anyhow::Result { let valued_inputs = self.generate_tx_inputs(); - let outputs = self.generate_tx_outputs(&valued_inputs)?; + let outputs = self.generate_tx_outputs(&valued_inputs, self.tx_fee)?; let locktime = 0; let flags = 0; let (inputs, _): (Vec, Vec) = valued_inputs.into_iter().unzip(); @@ -583,12 +591,13 @@ mod tests { .zip(outputs.iter().enumerate()) .map(|(id, (i, output))| valued_outpoint(&id, i as u32, output)), ); + Ok(tx) } fn generate_replaceable_tx(mut self) -> anyhow::Result { let valued_inputs = self.generate_tx_inputs(); - let outputs = self.generate_tx_outputs(&valued_inputs)?; + let outputs = self.generate_tx_outputs(&valued_inputs, self.tx_fee)?; let locktime = 0; let flags = 1; let (inputs, _values): (Vec, Vec) = valued_inputs.into_iter().unzip(); @@ -607,6 +616,7 @@ mod tests { fn generate_tx_outputs( &self, inputs: &[(TxInput, Amount)], + tx_fee: Option, ) -> anyhow::Result> { if self.num_outputs == 0 { return Ok(vec![]); @@ -617,14 +627,20 @@ mod tests { if inputs.is_empty() { return Ok(vec![]); } - let max_spend = + let sum_of_inputs = values.into_iter().sum::>().expect("Overflow in sum of input values"); - let mut left_to_spend = max_spend; + let total_to_spend = if let Some(fee) = tx_fee { + (sum_of_inputs - fee).expect("underflow") + } else { + sum_of_inputs + }; + + let mut left_to_spend = total_to_spend; let mut outputs = Vec::new(); let max_output_value = Amount::from_atoms(1_000); - for _ in 0..self.num_outputs { + for _ in 0..self.num_outputs - 1 { let max_output_value = std::cmp::min( (left_to_spend / 2).expect("division failed"), max_output_value, @@ -639,6 +655,8 @@ mod tests { )); left_to_spend = (left_to_spend - value).expect("subtraction failed"); } + + outputs.push(TxOutput::new(left_to_spend, Destination::AnyoneCanSpend)); Ok(outputs) } @@ -937,12 +955,17 @@ mod tests { let mut mempool = setup(); let num_inputs = 1; let num_outputs = 1; + let original_fee = Amount::from_atoms(10); let tx = TxGenerator::new(&mempool, num_inputs, num_outputs) + .with_fee(original_fee) .generate_replaceable_tx() .expect("generate_replaceable_tx"); mempool.add_transaction(tx)?; + let fee_delta = Amount::from_atoms(5); + let replacement_fee = (original_fee + fee_delta).expect("overflow"); let tx = TxGenerator::new(&mempool, num_inputs, num_outputs) + .with_fee(replacement_fee) .generate_tx() .expect("generate_tx_failed"); @@ -1016,27 +1039,32 @@ mod tests { mempool.add_transaction(tx.clone())?; - let outpoint_source_id = OutPointSourceId::Transaction(tx.get_id()); - let replaceable_input = TxInput::new( - outpoint_source_id.clone(), - 0, - InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), - ); - let other_input = TxInput::new( - outpoint_source_id, - 1, - InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), - ); - let flags_replaceable = 1; let flags_irreplaceable = 0; let locktime = 0; - let ancestor_with_signal = - tx_spend_input(&mempool, replaceable_input, flags_replaceable, locktime)?; + let outpoint_source_id = OutPointSourceId::Transaction(tx.get_id()); + let ancestor_with_signal = tx_spend_input( + &mempool, + TxInput::new( + outpoint_source_id.clone(), + 0, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ), + flags_replaceable, + locktime, + )?; - let ancestor_without_signal = - tx_spend_input(&mempool, other_input, flags_irreplaceable, locktime)?; + let ancestor_without_signal = tx_spend_input( + &mempool, + TxInput::new( + outpoint_source_id, + 1, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ), + flags_irreplaceable, + locktime, + )?; mempool.add_transaction(ancestor_with_signal.clone())?; mempool.add_transaction(ancestor_without_signal.clone())?; @@ -1046,6 +1074,7 @@ mod tests { 0, InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), ); + let input_with_irreplaceable_parent = TxInput::new( OutPointSourceId::Transaction(ancestor_without_signal.get_id()), 0, From 5f6b788e5b39e62f0c6eb42e0175d3d39e26f771 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Tue, 15 Feb 2022 09:17:57 +0700 Subject: [PATCH 011/153] Refactor TxMempoolEntry to make it unit-testable --- mempool/src/pool.rs | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 327f9dcb43..dbdf1886c6 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -75,21 +75,8 @@ trait TryGetFee { } impl TxMempoolEntry { - fn new(tx: Transaction, pool: &MempoolImpl) -> Option { - let parents = tx - .inputs() - .iter() - .filter_map(|input| { - pool.store - .txs_by_id - .get(&input.outpoint().tx_id().get_tx_id().expect("Not coinbase").get()) - }) - .cloned() - .collect::>(); - - let fee = pool.try_get_fee(&tx).ok()?; - - Some(Self { tx, fee, parents }) + fn new(tx: Transaction, fee: Amount, parents: BTreeSet>) -> TxMempoolEntry { + Self { tx, fee, parents } } fn is_replaceable(&self) -> bool { @@ -260,6 +247,22 @@ impl MempoolImpl { self.store.contains_outpoint(outpoint) || self.chain_state.contains_outpoint(outpoint) } + fn create_entry(&self, tx: Transaction) -> Result { + let parents = tx + .inputs() + .iter() + .filter_map(|input| { + self.store + .txs_by_id + .get(&input.outpoint().tx_id().get_tx_id().expect("Not Coinbase").get()) + }) + .cloned() + .collect::>(); + + let fee = self.try_get_fee(&tx)?; + Ok(TxMempoolEntry::new(tx, fee, parents)) + } + fn validate_transaction(&self, tx: &Transaction) -> Result<(), TxValidationError> { if tx.inputs().is_empty() { return Err(TxValidationError::NoInputs); @@ -325,8 +328,7 @@ impl Mempool for MempoolImpl { return Err(MempoolError::MempoolFull); } self.validate_transaction(&tx)?; - let entry = TxMempoolEntry::new(tx, self) - .ok_or_else(|| MempoolError::from(TxValidationError::TransactionFeeOverflow))?; + let entry = self.create_entry(tx)?; self.store.add_tx(entry)?; Ok(()) } From 4f177927ce90ba28beaf621c4d6966dcfb842e3f Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Tue, 15 Feb 2022 09:55:47 +0700 Subject: [PATCH 012/153] Add unit test for TxMempoolEntry --- mempool/src/pool.rs | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index dbdf1886c6..e4e90bcefb 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -1107,4 +1107,30 @@ mod tests { Ok(()) } + + #[test] + fn tx_mempool_entry_num_ancestors() -> anyhow::Result<()> { + // Input different flag values just to make the hashes of these dummy transactions + // different + let tx1 = Transaction::new(1, vec![], vec![], 0).map_err(anyhow::Error::from)?; + let tx2 = Transaction::new(2, vec![], vec![], 0).map_err(anyhow::Error::from)?; + let tx3 = Transaction::new(3, vec![], vec![], 0).map_err(anyhow::Error::from)?; + let tx4 = Transaction::new(4, vec![], vec![], 0).map_err(anyhow::Error::from)?; + let fee = Amount::from(0); + + let entry1 = Rc::new(TxMempoolEntry::new(tx1, fee, BTreeSet::default())); + let tx2_parents = vec![Rc::clone(&entry1)].into_iter().collect(); + let entry2 = Rc::new(TxMempoolEntry::new(tx2, fee, tx2_parents)); + + let tx3_parents = vec![Rc::clone(&entry1), Rc::clone(&entry2)].into_iter().collect(); + let entry3 = Rc::new(TxMempoolEntry::new(tx3, fee, tx3_parents)); + + let tx4_parents = vec![Rc::clone(&entry3)].into_iter().collect(); + let entry4 = TxMempoolEntry::new(tx4, fee, tx4_parents); + assert_eq!(entry4.unconfirmed_ancestors().len(), 3); + assert_eq!(entry3.unconfirmed_ancestors().len(), 2); + assert_eq!(entry2.unconfirmed_ancestors().len(), 1); + assert_eq!(entry1.unconfirmed_ancestors().len(), 0); + Ok(()) + } } From da027483090809cccdf9f01e3506018df035cd62 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Tue, 15 Feb 2022 10:28:24 +0700 Subject: [PATCH 013/153] Added unit test of TxMempoolEntry DFS --- mempool/src/pool.rs | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index e4e90bcefb..6703a5698b 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -1116,21 +1116,39 @@ mod tests { let tx2 = Transaction::new(2, vec![], vec![], 0).map_err(anyhow::Error::from)?; let tx3 = Transaction::new(3, vec![], vec![], 0).map_err(anyhow::Error::from)?; let tx4 = Transaction::new(4, vec![], vec![], 0).map_err(anyhow::Error::from)?; + let tx5 = Transaction::new(5, vec![], vec![], 0).map_err(anyhow::Error::from)?; + + let tx6 = Transaction::new(6, vec![], vec![], 0).map_err(anyhow::Error::from)?; let fee = Amount::from(0); - let entry1 = Rc::new(TxMempoolEntry::new(tx1, fee, BTreeSet::default())); - let tx2_parents = vec![Rc::clone(&entry1)].into_iter().collect(); + // Generation 1 + let tx1_parents = BTreeSet::default(); + let entry1 = Rc::new(TxMempoolEntry::new(tx1, fee, tx1_parents)); + let tx2_parents = BTreeSet::default(); let entry2 = Rc::new(TxMempoolEntry::new(tx2, fee, tx2_parents)); + // Generation 2 let tx3_parents = vec![Rc::clone(&entry1), Rc::clone(&entry2)].into_iter().collect(); let entry3 = Rc::new(TxMempoolEntry::new(tx3, fee, tx3_parents)); + // Generation 3 let tx4_parents = vec![Rc::clone(&entry3)].into_iter().collect(); - let entry4 = TxMempoolEntry::new(tx4, fee, tx4_parents); - assert_eq!(entry4.unconfirmed_ancestors().len(), 3); - assert_eq!(entry3.unconfirmed_ancestors().len(), 2); - assert_eq!(entry2.unconfirmed_ancestors().len(), 1); + let tx5_parents = vec![Rc::clone(&entry3)].into_iter().collect(); + let entry4 = Rc::new(TxMempoolEntry::new(tx4, fee, tx4_parents)); + let entry5 = Rc::new(TxMempoolEntry::new(tx5, fee, tx5_parents)); + + // Generation 4 + let tx6_parents = vec![Rc::clone(&entry3), Rc::clone(&entry4), Rc::clone(&entry5)] + .into_iter() + .collect(); + let entry6 = Rc::new(TxMempoolEntry::new(tx6, fee, tx6_parents)); + assert_eq!(entry1.unconfirmed_ancestors().len(), 0); + assert_eq!(entry2.unconfirmed_ancestors().len(), 0); + assert_eq!(entry3.unconfirmed_ancestors().len(), 2); + assert_eq!(entry4.unconfirmed_ancestors().len(), 3); + assert_eq!(entry5.unconfirmed_ancestors().len(), 3); + assert_eq!(entry6.unconfirmed_ancestors().len(), 5); Ok(()) } } From eb592928ecac1c7cf8c608aa9075bb024df0fb33 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Tue, 15 Feb 2022 15:09:52 +0700 Subject: [PATCH 014/153] Refactor test utilityfunction tx_spend_input --- mempool/src/pool.rs | 148 +++++++++++++++++++++++--------------------- 1 file changed, 78 insertions(+), 70 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 6703a5698b..bf03187d23 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -429,6 +429,15 @@ mod tests { .chain(self.chain_state.confirmed_outpoints()) .collect() } + + fn get_input_value(&self, input: &TxInput) -> anyhow::Result { + self.available_outpoints() + .iter() + .find_map(|valued_outpoint| { + (valued_outpoint.outpoint == *input.outpoint()).then(|| valued_outpoint.value) + }) + .ok_or_else(|| anyhow::anyhow!("No such unconfirmed output")) + } } #[derive(Debug, Clone)] @@ -658,7 +667,10 @@ mod tests { left_to_spend = (left_to_spend - value).expect("subtraction failed"); } - outputs.push(TxOutput::new(left_to_spend, Destination::AnyoneCanSpend)); + outputs.push(TxOutput::new( + left_to_spend, + OutputPurpose::Transfer(Destination::AnyoneCanSpend), + )); Ok(outputs) } @@ -701,18 +713,15 @@ mod tests { .expect("genesis tx not found"); let outpoint_source_id = OutPointSourceId::Transaction(genesis_tx.get_id()); + + let flags = 0; + let locktime = 0; let input = TxInput::new( outpoint_source_id, 0, InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), ); - let outputs = spend_input(&mempool, &input)?; - - let flags = 0; - let inputs = vec![input]; - let locktime = 0; - let tx = Transaction::new(flags, inputs, outputs, locktime) - .map_err(|e| anyhow::anyhow!("failed to create transaction: {:?}", e))?; + let tx = tx_spend_input(&mempool, input, None, flags, locktime)?; let tx_clone = tx.clone(); let tx_id = tx.get_id(); @@ -795,33 +804,6 @@ mod tests { Ok(()) } - fn spend_input( - mempool: &MempoolImpl, - input: &TxInput, - ) -> anyhow::Result> { - let outpoint = input.outpoint(); - let input_value = mempool.chain_state.get_outpoint_value(outpoint).or_else(|_| { - mempool - .available_outpoints() - .iter() - .find_map(|valued_outpoint| { - (valued_outpoint.outpoint == *outpoint).then(|| valued_outpoint.value) - }) - .ok_or_else(|| anyhow::anyhow!("No such unconfirmed output")) - })?; - let output_value = (input_value / 2).expect("failed to divide input"); - let output_pay = TxOutput::new( - output_value, - OutputPurpose::Transfer(Destination::AnyoneCanSpend), - ); - let output_change_amount = (input_value - output_value).expect("underflow"); - let output_change = TxOutput::new( - output_change_amount, - OutputPurpose::Transfer(Destination::AnyoneCanSpend), - ); - Ok(vec![output_pay, output_change]) - } - #[test] fn tx_duplicate_inputs() -> anyhow::Result<()> { let mut mempool = MempoolImpl::create(ChainStateMock::new()); @@ -845,12 +827,13 @@ mod tests { 0, InputWitness::NoSignature(Some(witness)), ); - let outputs = spend_input(&mempool, &input)?; - let inputs = vec![input, duplicate_input]; let flags = 0; let locktime = 0; - let tx = Transaction::new(flags, inputs, outputs, locktime) - .map_err(|e| anyhow::anyhow!("failed to create transaction: {:?}", e))?; + let outputs = tx_spend_input(&mempool, input.clone(), None, flags, locktime)? + .outputs() + .clone(); + let inputs = vec![input, duplicate_input]; + let tx = Transaction::new(flags, inputs, outputs, locktime)?; assert!(matches!( mempool.add_transaction(tx), @@ -878,12 +861,10 @@ mod tests { 0, InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), ); - let outputs = spend_input(&mempool, &input)?; let flags = 0; - let inputs = vec![input]; let locktime = 0; - let tx = Transaction::new(flags, inputs, outputs, locktime)?; + let tx = tx_spend_input(&mempool, input, None, flags, locktime)?; mempool.add_transaction(tx.clone())?; assert!(matches!( @@ -907,12 +888,17 @@ mod tests { .expect("genesis tx not found"); let outpoint_source_id = OutPointSourceId::Transaction(genesis_tx.get_id()); + let good_input = TxInput::new( outpoint_source_id.clone(), 0, InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), ); - let outputs = spend_input(&mempool, &good_input)?; + let flags = 0; + let locktime = 0; + let outputs = + tx_spend_input(&mempool, good_input, None, flags, locktime)?.outputs().clone(); + let bad_outpoint_index = 1; let bad_input = TxInput::new( outpoint_source_id, @@ -920,9 +906,7 @@ mod tests { InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), ); - let flags = 0; let inputs = vec![bad_input]; - let locktime = 0; let tx = Transaction::new(flags, inputs, outputs, locktime)?; assert!(matches!( @@ -995,9 +979,7 @@ mod tests { // replaced let flags = 0; let locktime = 0; - let outputs = spend_input(&mempool, &child_tx_input)?; - let inputs = vec![child_tx_input]; - let child_tx = Transaction::new(flags, inputs, outputs, locktime)?; + let child_tx = tx_spend_input(&mempool, child_tx_input, None, flags, locktime)?; mempool.add_transaction(child_tx)?; Ok(()) } @@ -1005,29 +987,52 @@ mod tests { fn tx_spend_input( mempool: &MempoolImpl, input: TxInput, + fee: Option, flags: u32, locktime: u32, ) -> anyhow::Result { - let input_value = mempool - .available_outpoints() + tx_spend_several_inputs(mempool, &[input], fee, flags, locktime) + } + + fn tx_spend_several_inputs( + mempool: &MempoolImpl, + inputs: &[TxInput], + fee: impl Into>, + flags: u32, + locktime: u32, + ) -> anyhow::Result { + let input_value = inputs .iter() - .find_map(|valued_outpoint| { - (valued_outpoint.outpoint == *input.outpoint()).then(|| valued_outpoint.value) - }) - .ok_or_else(|| anyhow::anyhow!("No such unconfirmed output"))?; - let output_value = (input_value / 2).expect("failed to divide input"); - let output_pay = TxOutput::new( - output_value, - OutputPurpose::Transfer(Destination::AnyoneCanSpend), - ); - let output_change_amount = (input_value - output_value).expect("underflow"); - let output_change = TxOutput::new( - output_change_amount, - OutputPurpose::Transfer(Destination::AnyoneCanSpend), - ); - let outputs = vec![output_pay, output_change]; - let inputs = vec![input]; - Transaction::new(flags, inputs, outputs, locktime).map_err(Into::into) + .map(|input| mempool.get_input_value(input)) + .collect::, _>>()? + .into_iter() + .sum::>() + .expect("tx_spend_input: overflow"); + + let output_value = if let Some(fee) = fee.into() { + (fee <= input_value) + .then(|| (input_value - fee).expect("tx_spend_input: subtraction error")) + .ok_or_else(|| anyhow::anyhow!("Not enough funds"))? + } else { + (input_value / 2).expect("tx_spend_input: division error") + }; + + Transaction::new( + flags, + inputs.to_owned(), + vec![ + TxOutput::new( + output_value, + OutputPurpose::Transfer(Destination::AnyoneCanSpend), + ), + TxOutput::new( + (input_value - output_value).expect("underflow"), + OutputPurpose::Transfer(Destination::AnyoneCanSpend), + ), + ], + locktime, + ) + .map_err(Into::into) } #[test] @@ -1053,6 +1058,7 @@ mod tests { 0, InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), ), + None, flags_replaceable, locktime, )?; @@ -1064,6 +1070,7 @@ mod tests { 1, InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), ), + None, flags_irreplaceable, locktime, )?; @@ -1082,11 +1089,12 @@ mod tests { 0, InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), ); + + let original_fee = Amount::from_atoms(10); let dummy_output = TxOutput::new( - Amount::from_atoms(0), + original_fee, OutputPurpose::Transfer(Destination::AnyoneCanSpend), ); - let replaced_tx = Transaction::new( flags_irreplaceable, vec![input_with_irreplaceable_parent.clone(), input_with_replaceable_parent], @@ -1119,7 +1127,7 @@ mod tests { let tx5 = Transaction::new(5, vec![], vec![], 0).map_err(anyhow::Error::from)?; let tx6 = Transaction::new(6, vec![], vec![], 0).map_err(anyhow::Error::from)?; - let fee = Amount::from(0); + let fee = Amount::from_atoms(0); // Generation 1 let tx1_parents = BTreeSet::default(); From b93eb8713acca90a4f747effcfe68d791a998236 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Tue, 15 Feb 2022 15:58:06 +0700 Subject: [PATCH 015/153] replacement fee higher than original in unit test --- mempool/src/pool.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index bf03187d23..029415b9bc 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -1095,10 +1095,11 @@ mod tests { original_fee, OutputPurpose::Transfer(Destination::AnyoneCanSpend), ); - let replaced_tx = Transaction::new( + let replaced_tx = tx_spend_several_inputs( + &mempool, + &[input_with_irreplaceable_parent.clone(), input_with_replaceable_parent], + original_fee, flags_irreplaceable, - vec![input_with_irreplaceable_parent.clone(), input_with_replaceable_parent], - vec![dummy_output.clone()], locktime, )?; From 7514284c72fad2aa4824620f5a74efd7c2fb3929 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Tue, 15 Feb 2022 16:03:59 +0700 Subject: [PATCH 016/153] Refactor call to is_replaceable in validation --- mempool/src/pool.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 029415b9bc..d7be9dab99 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -295,10 +295,11 @@ impl MempoolImpl { .filter_map(|input| self.store.find_conflicting_tx(input.outpoint())) .collect::>(); - for entry in conflicts { - if !entry.is_replaceable() { - return Err(TxValidationError::ConflictWithIrreplaceableTransaction); - } + for entry in &conflicts { + entry + .is_replaceable() + .then(|| ()) + .ok_or(TxValidationError::ConflictWithIrreplaceableTransaction)?; } self.verify_inputs_available(tx)?; From b77fb9a496d84e8b716a48910badcbcfa8fdc8da Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Tue, 15 Feb 2022 16:42:12 +0700 Subject: [PATCH 017/153] Add children to TxMempoolEntry --- mempool/src/pool.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index d7be9dab99..50a683df73 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -68,6 +68,7 @@ struct TxMempoolEntry { tx: Transaction, fee: Amount, parents: BTreeSet>, + children: BTreeSet>, } trait TryGetFee { @@ -76,7 +77,12 @@ trait TryGetFee { impl TxMempoolEntry { fn new(tx: Transaction, fee: Amount, parents: BTreeSet>) -> TxMempoolEntry { - Self { tx, fee, parents } + Self { + tx, + fee, + parents, + children: BTreeSet::default(), + } } fn is_replaceable(&self) -> bool { From 94667e74b2f387a31969006bfc41fe55cd4bf394 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Tue, 15 Feb 2022 16:59:10 +0700 Subject: [PATCH 018/153] Need to figure out how to update parents --- mempool/src/pool.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 50a683df73..70f08e2b18 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -175,6 +175,14 @@ impl MempoolStore { for outpoint in entry.tx.inputs().iter().map(|input| input.outpoint()) { self.spender_txs.insert(outpoint.clone(), Rc::clone(&entry)); } + + for mut parent in entry.parents.clone() { + assert!(Rc::get_mut(&mut parent) + .expect("exclusive access to parent") + .children + .insert(Rc::clone(&entry))) + } + Ok(()) } From 476d2e4065db3f8f6c4a99005fac018603940a43 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Tue, 15 Feb 2022 10:44:23 +0700 Subject: [PATCH 019/153] Check that replacement tx pays more than conflict --- mempool/src/pool.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 70f08e2b18..7bdc272698 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -232,6 +232,8 @@ pub enum TxValidationError { ConflictWithIrreplaceableTransaction, #[error("TransactionFeeOverflow")] TransactionFeeOverflow, + #[error("ReplacementFeeLowerThanOriginal")] + ReplacementFeeLowerThanOriginal, } impl From for MempoolError { @@ -314,12 +316,27 @@ impl MempoolImpl { .is_replaceable() .then(|| ()) .ok_or(TxValidationError::ConflictWithIrreplaceableTransaction)?; + + self.pays_more_than_conflicts(tx, &conflicts)?; } self.verify_inputs_available(tx)?; Ok(()) } + + fn pays_more_than_conflicts( + &self, + tx: &Transaction, + conflicts: &[Rc], + ) -> Result<(), TxValidationError> { + let replacement_fee = self.try_get_fee(tx)?; + conflicts + .iter() + .all(|conflict| conflict.fee < replacement_fee) + .then(|| ()) + .ok_or(TxValidationError::ReplacementFeeLowerThanOriginal) + } } impl Mempool for MempoolImpl { From a7c619c9d85b174420f7f706e0905fbfa58a9ecf Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Wed, 16 Feb 2022 13:25:42 +0700 Subject: [PATCH 020/153] Add TODO to test tx_replace_child --- mempool/src/pool.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 7bdc272698..e9c15297c8 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -993,6 +993,7 @@ mod tests { #[test] fn tx_replace_child() -> anyhow::Result<()> { + // TODO this test doesn't test what it's supposed to test! let mut mempool = setup(); let num_inputs = 1; let num_outputs = 1; From 4b26371c141ac0ced7074aef92f8ebf8d33fead6 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Thu, 17 Feb 2022 10:57:50 +0700 Subject: [PATCH 021/153] Use H256 instead of Rc to reference entries in the mempool --- mempool/src/pool.rs | 135 +++++++++++++++++++++++++------------------- 1 file changed, 78 insertions(+), 57 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index e9c15297c8..fcc203baf1 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -4,7 +4,6 @@ use std::collections::BTreeMap; use std::collections::BTreeSet; use std::collections::HashMap; use std::fmt::Debug; -use std::rc::Rc; use serialization::Encode; use thiserror::Error; @@ -67,8 +66,8 @@ pub trait ChainState { struct TxMempoolEntry { tx: Transaction, fee: Amount, - parents: BTreeSet>, - children: BTreeSet>, + parents: BTreeSet, + children: BTreeSet, } trait TryGetFee { @@ -76,7 +75,7 @@ trait TryGetFee { } impl TxMempoolEntry { - fn new(tx: Transaction, fee: Amount, parents: BTreeSet>) -> TxMempoolEntry { + fn new(tx: Transaction, fee: Amount, parents: BTreeSet) -> TxMempoolEntry { Self { tx, fee, @@ -85,24 +84,30 @@ impl TxMempoolEntry { } } - fn is_replaceable(&self) -> bool { + fn is_replaceable(&self, store: &MempoolStore) -> bool { self.tx.is_replaceable() - || self.unconfirmed_ancestors().iter().any(|ancestor| ancestor.tx.is_replaceable()) + || self + .unconfirmed_ancestors(store) + .iter() + .any(|ancestor| store.get_entry(ancestor).expect("entry").tx.is_replaceable()) } - fn unconfirmed_ancestors(&self) -> BTreeSet> { + fn unconfirmed_ancestors(&self, store: &MempoolStore) -> BTreeSet { let mut visited = BTreeSet::new(); - self.unconfirmed_ancestors_inner(&mut visited); + self.unconfirmed_ancestors_inner(&mut visited, store); visited } - fn unconfirmed_ancestors_inner(&self, visited: &mut BTreeSet>) { + fn unconfirmed_ancestors_inner(&self, visited: &mut BTreeSet, store: &MempoolStore) { for parent in self.parents.iter() { if visited.contains(parent) { continue; } else { - visited.insert(Rc::clone(parent)); - parent.unconfirmed_ancestors_inner(visited); + visited.insert(*parent); + store + .get_entry(parent) + .expect("entry") + .unconfirmed_ancestors_inner(visited, store); } } } @@ -128,9 +133,9 @@ pub struct MempoolImpl { #[derive(Debug)] struct MempoolStore { - txs_by_id: HashMap>, - txs_by_fee: BTreeMap>>, - spender_txs: BTreeMap>, + txs_by_id: HashMap, + txs_by_fee: BTreeMap>, + spender_txs: BTreeMap, } impl MempoolStore { @@ -166,22 +171,22 @@ impl MempoolStore { .map(|output| output.value()) } + fn get_entry(&self, id: &H256) -> Option<&TxMempoolEntry> { + self.txs_by_id.get(id) + } + fn add_tx(&mut self, entry: TxMempoolEntry) -> Result<(), MempoolError> { let id = entry.tx.get_id().get(); - let entry = Rc::new(entry); - self.txs_by_id.insert(id, Rc::clone(&entry)); - self.txs_by_fee.entry(entry.fee).or_default().insert(Rc::clone(&entry)); + for parent in &entry.parents { + self.txs_by_id.get_mut(parent).map(|parent| parent.children.insert(id)); + } for outpoint in entry.tx.inputs().iter().map(|input| input.outpoint()) { - self.spender_txs.insert(outpoint.clone(), Rc::clone(&entry)); + self.spender_txs.insert(outpoint.clone(), id); } - for mut parent in entry.parents.clone() { - assert!(Rc::get_mut(&mut parent) - .expect("exclusive access to parent") - .children - .insert(Rc::clone(&entry))) - } + self.txs_by_fee.entry(entry.fee).or_default().insert(id); + self.txs_by_id.insert(id, entry); Ok(()) } @@ -189,16 +194,16 @@ impl MempoolStore { fn drop_tx(&mut self, tx_id: &Id) { if let Some(entry) = self.txs_by_id.remove(&tx_id.get()) { self.txs_by_fee.entry(entry.fee).and_modify(|entries| { - entries.remove(&entry).then(|| ()).expect("Inconsistent mempool store") + entries.remove(&tx_id.get()).then(|| ()).expect("Inconsistent mempool store") }); - self.spender_txs.retain(|_, entry| entry.tx.get_id() != *tx_id) + self.spender_txs.retain(|_, id| *id != tx_id.get()) } else { - assert!(!self.txs_by_fee.values().flatten().any(|entry| entry.tx.get_id() == *tx_id)); - assert!(!self.spender_txs.iter().any(|(_, entry)| entry.tx.get_id() == *tx_id)); + assert!(!self.txs_by_fee.values().flatten().any(|id| *id == tx_id.get())); + assert!(!self.spender_txs.iter().any(|(_, id)| *id == tx_id.get())); } } - fn find_conflicting_tx(&self, outpoint: &OutPoint) -> Option> { + fn find_conflicting_tx(&self, outpoint: &OutPoint) -> Option { self.spender_txs.get(outpoint).cloned() } } @@ -267,12 +272,8 @@ impl MempoolImpl { let parents = tx .inputs() .iter() - .filter_map(|input| { - self.store - .txs_by_id - .get(&input.outpoint().tx_id().get_tx_id().expect("Not Coinbase").get()) - }) - .cloned() + .map(|input| input.outpoint().tx_id().get_tx_id().expect("Not coinbase").get()) + .filter_map(|id| self.store.txs_by_id.contains_key(&id).then(|| id)) .collect::>(); let fee = self.try_get_fee(&tx)?; @@ -309,11 +310,12 @@ impl MempoolImpl { .inputs() .iter() .filter_map(|input| self.store.find_conflicting_tx(input.outpoint())) + .map(|id_conflict| self.store.get_entry(&id_conflict).expect("entry for id")) .collect::>(); for entry in &conflicts { entry - .is_replaceable() + .is_replaceable(&self.store) .then(|| ()) .ok_or(TxValidationError::ConflictWithIrreplaceableTransaction)?; @@ -328,7 +330,7 @@ impl MempoolImpl { fn pays_more_than_conflicts( &self, tx: &Transaction, - conflicts: &[Rc], + conflicts: &[&TxMempoolEntry], ) -> Result<(), TxValidationError> { let replacement_fee = self.try_get_fee(tx)?; conflicts @@ -366,7 +368,12 @@ impl Mempool for MempoolImpl { } fn get_all(&self) -> Vec<&Transaction> { - self.store.txs_by_fee.values().flatten().map(|entry| &entry.tx).collect() + self.store + .txs_by_fee + .values() + .flatten() + .map(|id| &self.store.get_entry(id).expect("entry").tx) + .collect() } fn contains_transaction(&self, tx_id: &Id) -> bool { @@ -1152,6 +1159,7 @@ mod tests { #[test] fn tx_mempool_entry_num_ancestors() -> anyhow::Result<()> { + let mut mempool = setup(); // Input different flag values just to make the hashes of these dummy transactions // different let tx1 = Transaction::new(1, vec![], vec![], 0).map_err(anyhow::Error::from)?; @@ -1165,32 +1173,45 @@ mod tests { // Generation 1 let tx1_parents = BTreeSet::default(); - let entry1 = Rc::new(TxMempoolEntry::new(tx1, fee, tx1_parents)); + let entry1 = TxMempoolEntry::new(tx1, fee, tx1_parents); let tx2_parents = BTreeSet::default(); - let entry2 = Rc::new(TxMempoolEntry::new(tx2, fee, tx2_parents)); + let entry2 = TxMempoolEntry::new(tx2, fee, tx2_parents); // Generation 2 - let tx3_parents = vec![Rc::clone(&entry1), Rc::clone(&entry2)].into_iter().collect(); - let entry3 = Rc::new(TxMempoolEntry::new(tx3, fee, tx3_parents)); + let tx3_parents = + vec![entry1.tx.get_id().get(), entry2.tx.get_id().get()].into_iter().collect(); + let entry3 = TxMempoolEntry::new(tx3, fee, tx3_parents); // Generation 3 - let tx4_parents = vec![Rc::clone(&entry3)].into_iter().collect(); - let tx5_parents = vec![Rc::clone(&entry3)].into_iter().collect(); - let entry4 = Rc::new(TxMempoolEntry::new(tx4, fee, tx4_parents)); - let entry5 = Rc::new(TxMempoolEntry::new(tx5, fee, tx5_parents)); + let tx4_parents = vec![entry3.tx.get_id().get()].into_iter().collect(); + let tx5_parents = vec![entry3.tx.get_id().get()].into_iter().collect(); + let entry4 = TxMempoolEntry::new(tx4, fee, tx4_parents); + let entry5 = TxMempoolEntry::new(tx5, fee, tx5_parents); // Generation 4 - let tx6_parents = vec![Rc::clone(&entry3), Rc::clone(&entry4), Rc::clone(&entry5)] - .into_iter() - .collect(); - let entry6 = Rc::new(TxMempoolEntry::new(tx6, fee, tx6_parents)); - - assert_eq!(entry1.unconfirmed_ancestors().len(), 0); - assert_eq!(entry2.unconfirmed_ancestors().len(), 0); - assert_eq!(entry3.unconfirmed_ancestors().len(), 2); - assert_eq!(entry4.unconfirmed_ancestors().len(), 3); - assert_eq!(entry5.unconfirmed_ancestors().len(), 3); - assert_eq!(entry6.unconfirmed_ancestors().len(), 5); + let tx6_parents = + vec![entry3.tx.get_id().get(), entry4.tx.get_id().get(), entry5.tx.get_id().get()] + .into_iter() + .collect(); + let entry6 = TxMempoolEntry::new(tx6, fee, tx6_parents); + + for entry in vec![ + entry1.clone(), + entry2.clone(), + entry3.clone(), + entry4.clone(), + entry5.clone(), + entry6.clone(), + ] { + mempool.store.add_tx(entry)? + } + + assert_eq!(entry1.unconfirmed_ancestors(&mempool.store).len(), 0); + assert_eq!(entry2.unconfirmed_ancestors(&mempool.store).len(), 0); + assert_eq!(entry3.unconfirmed_ancestors(&mempool.store).len(), 2); + assert_eq!(entry4.unconfirmed_ancestors(&mempool.store).len(), 3); + assert_eq!(entry5.unconfirmed_ancestors(&mempool.store).len(), 3); + assert_eq!(entry6.unconfirmed_ancestors(&mempool.store).len(), 5); Ok(()) } } From 150c9bd814601617afd748379820b35cab70f929 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Thu, 17 Feb 2022 13:25:38 +0700 Subject: [PATCH 022/153] Getters for parents and children --- mempool/src/pool.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index fcc203baf1..7ef28756f5 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -84,6 +84,14 @@ impl TxMempoolEntry { } } + fn get_parents(&self) -> impl Iterator { + self.parents.iter() + } + + fn get_children_mut(&mut self) -> &mut BTreeSet { + &mut self.children + } + fn is_replaceable(&self, store: &MempoolStore) -> bool { self.tx.is_replaceable() || self @@ -177,8 +185,10 @@ impl MempoolStore { fn add_tx(&mut self, entry: TxMempoolEntry) -> Result<(), MempoolError> { let id = entry.tx.get_id().get(); - for parent in &entry.parents { - self.txs_by_id.get_mut(parent).map(|parent| parent.children.insert(id)); + for parent in entry.get_parents() { + self.txs_by_id + .get_mut(parent) + .map(|parent| parent.get_children_mut().insert(id)); } for outpoint in entry.tx.inputs().iter().map(|input| input.outpoint()) { From 5ffaa813ed8caa36652ef5d1191c5ce6d774245a Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Thu, 17 Feb 2022 13:32:50 +0700 Subject: [PATCH 023/153] Add convenience method TxMempoolEntry::tx_id --- mempool/src/pool.rs | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 7ef28756f5..b5f25e754e 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -84,6 +84,10 @@ impl TxMempoolEntry { } } + fn tx_id(&self) -> H256 { + self.tx.get_id().get() + } + fn get_parents(&self) -> impl Iterator { self.parents.iter() } @@ -123,13 +127,13 @@ impl TxMempoolEntry { impl PartialOrd for TxMempoolEntry { fn partial_cmp(&self, other: &Self) -> Option { - Some(other.tx.get_id().get().cmp(&self.tx.get_id().get())) + Some(other.tx_id().cmp(&self.tx_id())) } } impl Ord for TxMempoolEntry { fn cmp(&self, other: &Self) -> Ordering { - other.tx.get_id().get().cmp(&self.tx.get_id().get()) + other.tx_id().cmp(&self.tx_id()) } } @@ -184,7 +188,7 @@ impl MempoolStore { } fn add_tx(&mut self, entry: TxMempoolEntry) -> Result<(), MempoolError> { - let id = entry.tx.get_id().get(); + let id = entry.tx_id(); for parent in entry.get_parents() { self.txs_by_id .get_mut(parent) @@ -1188,21 +1192,18 @@ mod tests { let entry2 = TxMempoolEntry::new(tx2, fee, tx2_parents); // Generation 2 - let tx3_parents = - vec![entry1.tx.get_id().get(), entry2.tx.get_id().get()].into_iter().collect(); + let tx3_parents = vec![entry1.tx_id(), entry2.tx_id()].into_iter().collect(); let entry3 = TxMempoolEntry::new(tx3, fee, tx3_parents); // Generation 3 - let tx4_parents = vec![entry3.tx.get_id().get()].into_iter().collect(); - let tx5_parents = vec![entry3.tx.get_id().get()].into_iter().collect(); + let tx4_parents = vec![entry3.tx_id()].into_iter().collect(); + let tx5_parents = vec![entry3.tx_id()].into_iter().collect(); let entry4 = TxMempoolEntry::new(tx4, fee, tx4_parents); let entry5 = TxMempoolEntry::new(tx5, fee, tx5_parents); // Generation 4 let tx6_parents = - vec![entry3.tx.get_id().get(), entry4.tx.get_id().get(), entry5.tx.get_id().get()] - .into_iter() - .collect(); + vec![entry3.tx_id(), entry4.tx_id(), entry5.tx_id()].into_iter().collect(); let entry6 = TxMempoolEntry::new(tx6, fee, tx6_parents); for entry in vec![ From 787200e36734a2364941b7053a1f87ab2cec325c Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Thu, 17 Feb 2022 13:41:22 +0700 Subject: [PATCH 024/153] Refactor test tx_replace_child --- mempool/src/pool.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index b5f25e754e..41df345379 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -989,20 +989,16 @@ mod tests { Ok(()) } - #[test] - fn tx_replace() -> anyhow::Result<()> { + fn test_replace_tx(original_fee: Amount, replacement_fee: Amount) -> Result<(), MempoolError> { let mut mempool = setup(); let num_inputs = 1; let num_outputs = 1; - let original_fee = Amount::from_atoms(10); let tx = TxGenerator::new(&mempool, num_inputs, num_outputs) .with_fee(original_fee) .generate_replaceable_tx() .expect("generate_replaceable_tx"); mempool.add_transaction(tx)?; - let fee_delta = Amount::from_atoms(5); - let replacement_fee = (original_fee + fee_delta).expect("overflow"); let tx = TxGenerator::new(&mempool, num_inputs, num_outputs) .with_fee(replacement_fee) .generate_tx() @@ -1012,6 +1008,11 @@ mod tests { Ok(()) } + #[test] + fn tx_replace() -> anyhow::Result<()> { + test_replace_tx(Amount::from_atoms(10), Amount::from_atoms(15)).map_err(anyhow::Error::from) + } + #[test] fn tx_replace_child() -> anyhow::Result<()> { // TODO this test doesn't test what it's supposed to test! From de77505b9154e27bf5863e641ceda96894d22356 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Thu, 17 Feb 2022 13:45:52 +0700 Subject: [PATCH 025/153] Expand tx_replace test --- mempool/src/pool.rs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 41df345379..bde217c12b 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -1010,7 +1010,20 @@ mod tests { #[test] fn tx_replace() -> anyhow::Result<()> { - test_replace_tx(Amount::from_atoms(10), Amount::from_atoms(15)).map_err(anyhow::Error::from) + assert!(test_replace_tx(Amount::from_atoms(10), Amount::from_atoms(15)).is_ok()); + assert!(matches!( + test_replace_tx(Amount::from_atoms(10), Amount::from_atoms(10)), + Err(MempoolError::TxValidationError( + TxValidationError::ReplacementFeeLowerThanOriginal + )) + )); + assert!(matches!( + test_replace_tx(Amount::from_atoms(10), Amount::from_atoms(5)), + Err(MempoolError::TxValidationError( + TxValidationError::ReplacementFeeLowerThanOriginal + )) + )); + Ok(()) } #[test] From 2b502e92b9248123d43eaf2417d596795d25f6d0 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Thu, 17 Feb 2022 14:07:04 +0700 Subject: [PATCH 026/153] Move TxMepoolEntry next to impl --- mempool/src/pool.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index bde217c12b..e71495b7ec 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -62,6 +62,10 @@ pub trait ChainState { fn get_outpoint_value(&self, outpoint: &OutPoint) -> Result; } +trait TryGetFee { + fn try_get_fee(&self, tx: &Transaction) -> Result; +} + #[derive(Debug, PartialEq, Eq, Clone)] struct TxMempoolEntry { tx: Transaction, @@ -70,10 +74,6 @@ struct TxMempoolEntry { children: BTreeSet, } -trait TryGetFee { - fn try_get_fee(&self, tx: &Transaction) -> Result; -} - impl TxMempoolEntry { fn new(tx: Transaction, fee: Amount, parents: BTreeSet) -> TxMempoolEntry { Self { From ba417d61d7689140208d65901ec69b9d6ba8a2b6 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Fri, 18 Feb 2022 12:11:58 +0700 Subject: [PATCH 027/153] Support count_with_descendants in TxMempoolEntry --- mempool/src/pool.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index e71495b7ec..8f8faf1ef9 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -72,6 +72,7 @@ struct TxMempoolEntry { fee: Amount, parents: BTreeSet, children: BTreeSet, + count_with_descendants: usize, } impl TxMempoolEntry { @@ -81,9 +82,15 @@ impl TxMempoolEntry { fee, parents, children: BTreeSet::default(), + count_with_descendants: 1, } } + #[allow(unused)] + fn count_with_descendants(&self) -> usize { + self.count_with_descendants + } + fn tx_id(&self) -> H256 { self.tx.get_id().get() } @@ -199,6 +206,11 @@ impl MempoolStore { self.spender_txs.insert(outpoint.clone(), id); } + for ancestor in entry.unconfirmed_ancestors(self) { + let ancestor = self.txs_by_id.get_mut(&ancestor).expect("ancestor"); + ancestor.count_with_descendants += 1; + } + self.txs_by_fee.entry(entry.fee).or_default().insert(id); self.txs_by_id.insert(id, entry); From c5bab45db83a30f454846a6cc2f013e3f1ea9c19 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Fri, 18 Feb 2022 12:14:34 +0700 Subject: [PATCH 028/153] Validate number of potential replacements --- mempool/src/pool.rs | 62 ++++++++++++++++++++++++++++++++------------- 1 file changed, 45 insertions(+), 17 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 8f8faf1ef9..c29275f57c 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -86,7 +86,6 @@ impl TxMempoolEntry { } } - #[allow(unused)] fn count_with_descendants(&self) -> usize { self.count_with_descendants } @@ -197,13 +196,7 @@ impl MempoolStore { fn add_tx(&mut self, entry: TxMempoolEntry) -> Result<(), MempoolError> { let id = entry.tx_id(); for parent in entry.get_parents() { - self.txs_by_id - .get_mut(parent) - .map(|parent| parent.get_children_mut().insert(id)); - } - - for outpoint in entry.tx.inputs().iter().map(|input| input.outpoint()) { - self.spender_txs.insert(outpoint.clone(), id); + self.txs_by_id.get_mut(parent).expect("be there").get_children_mut().insert(id); } for ancestor in entry.unconfirmed_ancestors(self) { @@ -211,6 +204,10 @@ impl MempoolStore { ancestor.count_with_descendants += 1; } + for outpoint in entry.tx.inputs().iter().map(|input| input.outpoint()) { + self.spender_txs.insert(outpoint.clone(), id); + } + self.txs_by_fee.entry(entry.fee).or_default().insert(id); self.txs_by_id.insert(id, entry); @@ -265,6 +262,8 @@ pub enum TxValidationError { TransactionFeeOverflow, #[error("ReplacementFeeLowerThanOriginal")] ReplacementFeeLowerThanOriginal, + #[error("TooManyPotentialReplacements")] + TooManyPotentialReplacements, } impl From for MempoolError { @@ -346,6 +345,7 @@ impl MempoolImpl { .ok_or(TxValidationError::ConflictWithIrreplaceableTransaction)?; self.pays_more_than_conflicts(tx, &conflicts)?; + self.potential_replacements_within_limit(&conflicts)?; } self.verify_inputs_available(tx)?; @@ -365,6 +365,21 @@ impl MempoolImpl { .then(|| ()) .ok_or(TxValidationError::ReplacementFeeLowerThanOriginal) } + + fn potential_replacements_within_limit( + &self, + conflicts: &[&TxMempoolEntry], + ) -> Result<(), TxValidationError> { + const MAX_BIP125_REPLACEMENT_CANDIATES: usize = 100; + let mut num_potential_replacements = 0; + for conflict in conflicts { + num_potential_replacements += conflict.count_with_descendants(); + if num_potential_replacements > MAX_BIP125_REPLACEMENT_CANDIATES { + return Err(TxValidationError::TooManyPotentialReplacements); + } + } + Ok(()) + } } impl Mempool for MempoolImpl { @@ -1232,23 +1247,36 @@ mod tests { vec![entry3.tx_id(), entry4.tx_id(), entry5.tx_id()].into_iter().collect(); let entry6 = TxMempoolEntry::new(tx6, fee, tx6_parents); - for entry in vec![ - entry1.clone(), - entry2.clone(), - entry3.clone(), - entry4.clone(), - entry5.clone(), - entry6.clone(), - ] { - mempool.store.add_tx(entry)? + let entries = vec![entry1, entry2, entry3, entry4, entry5, entry6]; + let ids = entries.clone().into_iter().map(|entry| entry.tx_id()).collect::>(); + + for (index, entry) in entries.into_iter().enumerate() { + println!("adding {}, (entry {})", entry.tx_id(), index + 1); + mempool.store.add_tx(entry)?; + println!( + "after adding entry {}, store looks like this {:#?}", + index + 1, + mempool.store.txs_by_id + ); } + let entry1 = mempool.store.get_entry(ids.get(0).expect("index")).expect("entry"); + let entry2 = mempool.store.get_entry(ids.get(1).expect("index")).expect("entry"); + let entry3 = mempool.store.get_entry(ids.get(2).expect("index")).expect("entry"); + let entry4 = mempool.store.get_entry(ids.get(3).expect("index")).expect("entry"); + let entry5 = mempool.store.get_entry(ids.get(4).expect("index")).expect("entry"); + let entry6 = mempool.store.get_entry(ids.get(5).expect("index")).expect("entry"); assert_eq!(entry1.unconfirmed_ancestors(&mempool.store).len(), 0); assert_eq!(entry2.unconfirmed_ancestors(&mempool.store).len(), 0); assert_eq!(entry3.unconfirmed_ancestors(&mempool.store).len(), 2); assert_eq!(entry4.unconfirmed_ancestors(&mempool.store).len(), 3); assert_eq!(entry5.unconfirmed_ancestors(&mempool.store).len(), 3); assert_eq!(entry6.unconfirmed_ancestors(&mempool.store).len(), 5); + + //println!("entry1 {:#?}", entry1); + assert_eq!(entry1.count_with_descendants(), 5); + //assert_eq!(entry2.count_with_descendants(&mempool.store), 4); + Ok(()) } } From 5ca71c9065ee241255eeaccf048f5fb656995ff7 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Fri, 18 Feb 2022 12:29:45 +0700 Subject: [PATCH 029/153] Rename 'get_parents' to 'unconfirmed_parents' --- mempool/src/pool.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index c29275f57c..f2674a56ef 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -94,7 +94,7 @@ impl TxMempoolEntry { self.tx.get_id().get() } - fn get_parents(&self) -> impl Iterator { + fn unconfirmed_parents(&self) -> impl Iterator { self.parents.iter() } @@ -195,7 +195,7 @@ impl MempoolStore { fn add_tx(&mut self, entry: TxMempoolEntry) -> Result<(), MempoolError> { let id = entry.tx_id(); - for parent in entry.get_parents() { + for parent in entry.unconfirmed_parents() { self.txs_by_id.get_mut(parent).expect("be there").get_children_mut().insert(id); } From f58473fc1cecd3e76c8c347606a9706bf6a6de4f Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Fri, 18 Feb 2022 12:39:58 +0700 Subject: [PATCH 030/153] Refactor parent update --- mempool/src/pool.rs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index f2674a56ef..bb9bec3bc8 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -193,11 +193,19 @@ impl MempoolStore { self.txs_by_id.get(id) } - fn add_tx(&mut self, entry: TxMempoolEntry) -> Result<(), MempoolError> { - let id = entry.tx_id(); + fn update_parents(&mut self, entry: &TxMempoolEntry) { for parent in entry.unconfirmed_parents() { - self.txs_by_id.get_mut(parent).expect("be there").get_children_mut().insert(id); + self.txs_by_id + .get_mut(parent) + .expect("be there") + .get_children_mut() + .insert(entry.tx_id()); } + } + + fn add_tx(&mut self, entry: TxMempoolEntry) -> Result<(), MempoolError> { + let id = entry.tx_id(); + self.update_parents(&entry); for ancestor in entry.unconfirmed_ancestors(self) { let ancestor = self.txs_by_id.get_mut(&ancestor).expect("ancestor"); From 8f84402c0813f081360f040eae1ba99716aa65ee Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Fri, 18 Feb 2022 12:41:39 +0700 Subject: [PATCH 031/153] Refactor ancestor count update --- mempool/src/pool.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index bb9bec3bc8..640bde9343 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -193,7 +193,7 @@ impl MempoolStore { self.txs_by_id.get(id) } - fn update_parents(&mut self, entry: &TxMempoolEntry) { + fn append_to_parents(&mut self, entry: &TxMempoolEntry) { for parent in entry.unconfirmed_parents() { self.txs_by_id .get_mut(parent) @@ -203,14 +203,17 @@ impl MempoolStore { } } - fn add_tx(&mut self, entry: TxMempoolEntry) -> Result<(), MempoolError> { - let id = entry.tx_id(); - self.update_parents(&entry); - + fn update_ancestor_count(&mut self, entry: &TxMempoolEntry) { for ancestor in entry.unconfirmed_ancestors(self) { let ancestor = self.txs_by_id.get_mut(&ancestor).expect("ancestor"); ancestor.count_with_descendants += 1; } + } + + fn add_tx(&mut self, entry: TxMempoolEntry) -> Result<(), MempoolError> { + let id = entry.tx_id(); + self.append_to_parents(&entry); + self.update_ancestor_count(&entry); for outpoint in entry.tx.inputs().iter().map(|input| input.outpoint()) { self.spender_txs.insert(outpoint.clone(), id); From 98e8912b3a82bd8c8936c58dbc5a8adcd7a760ad Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Fri, 18 Feb 2022 12:43:33 +0700 Subject: [PATCH 032/153] Refactor marking outpoints as spent --- mempool/src/pool.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 640bde9343..2ef8d1418c 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -210,14 +210,18 @@ impl MempoolStore { } } - fn add_tx(&mut self, entry: TxMempoolEntry) -> Result<(), MempoolError> { + fn mark_outpoints_as_spent(&mut self, entry: &TxMempoolEntry) { let id = entry.tx_id(); - self.append_to_parents(&entry); - self.update_ancestor_count(&entry); - for outpoint in entry.tx.inputs().iter().map(|input| input.outpoint()) { self.spender_txs.insert(outpoint.clone(), id); } + } + + fn add_tx(&mut self, entry: TxMempoolEntry) -> Result<(), MempoolError> { + let id = entry.tx_id(); + self.append_to_parents(&entry); + self.update_ancestor_count(&entry); + self.mark_outpoints_as_spent(&entry); self.txs_by_fee.entry(entry.fee).or_default().insert(id); self.txs_by_id.insert(id, entry); From dcbc97b72959f5f0300ec7de01b2169362b64cba Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Fri, 18 Feb 2022 12:45:01 +0700 Subject: [PATCH 033/153] Remove local variable in add_tx --- mempool/src/pool.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 2ef8d1418c..492530c520 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -218,13 +218,12 @@ impl MempoolStore { } fn add_tx(&mut self, entry: TxMempoolEntry) -> Result<(), MempoolError> { - let id = entry.tx_id(); self.append_to_parents(&entry); self.update_ancestor_count(&entry); self.mark_outpoints_as_spent(&entry); - self.txs_by_fee.entry(entry.fee).or_default().insert(id); - self.txs_by_id.insert(id, entry); + self.txs_by_fee.entry(entry.fee).or_default().insert(entry.tx_id()); + self.txs_by_id.insert(entry.tx_id(), entry); Ok(()) } From f2d69b0e96bb0e97942f62f9df7358bc32a04697 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Fri, 18 Feb 2022 13:45:50 +0700 Subject: [PATCH 034/153] Make ReplacementFeeLowerThanOriginal error more descriptive --- mempool/src/pool.rs | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 492530c520..9f47292092 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -275,7 +275,12 @@ pub enum TxValidationError { #[error("TransactionFeeOverflow")] TransactionFeeOverflow, #[error("ReplacementFeeLowerThanOriginal")] - ReplacementFeeLowerThanOriginal, + ReplacementFeeLowerThanOriginal { + replacement_tx: H256, + replacement_fee: Amount, + original_tx: H256, + original_fee: Amount, + }, #[error("TooManyPotentialReplacements")] TooManyPotentialReplacements, } @@ -373,11 +378,17 @@ impl MempoolImpl { conflicts: &[&TxMempoolEntry], ) -> Result<(), TxValidationError> { let replacement_fee = self.try_get_fee(tx)?; - conflicts - .iter() - .all(|conflict| conflict.fee < replacement_fee) - .then(|| ()) - .ok_or(TxValidationError::ReplacementFeeLowerThanOriginal) + conflicts.iter().find(|conflict| conflict.fee >= replacement_fee).map_or_else( + || Ok(()), + |conflict| { + Err(TxValidationError::ReplacementFeeLowerThanOriginal { + replacement_tx: tx.get_id().get(), + replacement_fee, + original_fee: conflict.fee, + original_tx: conflict.tx_id(), + }) + }, + ) } fn potential_replacements_within_limit( @@ -1055,13 +1066,13 @@ mod tests { assert!(matches!( test_replace_tx(Amount::from_atoms(10), Amount::from_atoms(10)), Err(MempoolError::TxValidationError( - TxValidationError::ReplacementFeeLowerThanOriginal + TxValidationError::ReplacementFeeLowerThanOriginal { .. } )) )); assert!(matches!( test_replace_tx(Amount::from_atoms(10), Amount::from_atoms(5)), Err(MempoolError::TxValidationError( - TxValidationError::ReplacementFeeLowerThanOriginal + TxValidationError::ReplacementFeeLowerThanOriginal { .. } )) )); Ok(()) From 9f7499d8cf5a33186ad19a3b5744091359920fb1 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Fri, 18 Feb 2022 13:45:12 +0700 Subject: [PATCH 035/153] Fixed tx_spend_inputs --- mempool/src/pool.rs | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 9f47292092..5b9c641d74 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -1121,6 +1121,7 @@ mod tests { flags: u32, locktime: u32, ) -> anyhow::Result { + let fee = fee.into().map_or(Amount::from_atoms(0), std::convert::identity); let input_value = inputs .iter() .map(|input| mempool.get_input_value(input)) @@ -1129,26 +1130,17 @@ mod tests { .sum::>() .expect("tx_spend_input: overflow"); - let output_value = if let Some(fee) = fee.into() { - (fee <= input_value) - .then(|| (input_value - fee).expect("tx_spend_input: subtraction error")) - .ok_or_else(|| anyhow::anyhow!("Not enough funds"))? - } else { - (input_value / 2).expect("tx_spend_input: division error") - }; + let available_for_spending = (input_value - fee).expect("underflow"); + let spent = (available_for_spending / 2).expect("division error"); + + let change = (available_for_spending - spent).expect("underflow"); Transaction::new( flags, inputs.to_owned(), vec![ - TxOutput::new( - output_value, - OutputPurpose::Transfer(Destination::AnyoneCanSpend), - ), - TxOutput::new( - (input_value - output_value).expect("underflow"), - OutputPurpose::Transfer(Destination::AnyoneCanSpend), - ), + TxOutput::new(spent, OutputPurpose::Transfer(Destination::AnyoneCanSpend)), + TxOutput::new(change, OutputPurpose::Transfer(Destination::AnyoneCanSpend)), ], locktime, ) From 9495d19da905c9092532b7c1500289e5d0d58435 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Fri, 18 Feb 2022 13:23:19 +0700 Subject: [PATCH 036/153] Fix tx_replace_child test --- mempool/src/pool.rs | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 5b9c641d74..72bac2d78c 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -1080,7 +1080,6 @@ mod tests { #[test] fn tx_replace_child() -> anyhow::Result<()> { - // TODO this test doesn't test what it's supposed to test! let mut mempool = setup(); let num_inputs = 1; let num_outputs = 1; @@ -1095,12 +1094,27 @@ mod tests { 0, InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), ); - // We want to test that even though it doesn't signal replaceability directly, the child tx is replaceable because it's parent signalled replaceability + // We want to test that even though child_tx doesn't signal replaceability directly, it is replaceable because its parent signalled replaceability // replaced let flags = 0; let locktime = 0; - let child_tx = tx_spend_input(&mempool, child_tx_input, None, flags, locktime)?; + let child_tx = tx_spend_input( + &mempool, + child_tx_input.clone(), + Amount::from_atoms(10).into(), + flags, + locktime, + )?; mempool.add_transaction(child_tx)?; + + let replacement_tx = tx_spend_input( + &mempool, + child_tx_input, + Amount::from_atoms(15).into(), + flags, + locktime, + )?; + mempool.add_transaction(replacement_tx)?; Ok(()) } From 29db54c77776ede6720b374bc3ea88e461b956d3 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Fri, 18 Feb 2022 15:30:02 +0700 Subject: [PATCH 037/153] Refactor, expand, and rename tx_mempool_entry test --- mempool/src/pool.rs | 40 +++++++++++++++++----------------------- 1 file changed, 17 insertions(+), 23 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 72bac2d78c..99b6b62273 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -1244,51 +1244,42 @@ mod tests { } #[test] - fn tx_mempool_entry_num_ancestors() -> anyhow::Result<()> { + fn tx_mempool_entry() -> anyhow::Result<()> { let mut mempool = setup(); // Input different flag values just to make the hashes of these dummy transactions // different - let tx1 = Transaction::new(1, vec![], vec![], 0).map_err(anyhow::Error::from)?; - let tx2 = Transaction::new(2, vec![], vec![], 0).map_err(anyhow::Error::from)?; - let tx3 = Transaction::new(3, vec![], vec![], 0).map_err(anyhow::Error::from)?; - let tx4 = Transaction::new(4, vec![], vec![], 0).map_err(anyhow::Error::from)?; - let tx5 = Transaction::new(5, vec![], vec![], 0).map_err(anyhow::Error::from)?; - - let tx6 = Transaction::new(6, vec![], vec![], 0).map_err(anyhow::Error::from)?; + let txs = (1..=6) + .into_iter() + .map(|i| Transaction::new(i, vec![], vec![], 0).unwrap_or_else(|_| panic!("tx {}", i))) + .collect::>(); let fee = Amount::from_atoms(0); // Generation 1 let tx1_parents = BTreeSet::default(); - let entry1 = TxMempoolEntry::new(tx1, fee, tx1_parents); + let entry1 = TxMempoolEntry::new(txs.get(0).unwrap().clone(), fee, tx1_parents); let tx2_parents = BTreeSet::default(); - let entry2 = TxMempoolEntry::new(tx2, fee, tx2_parents); + let entry2 = TxMempoolEntry::new(txs.get(1).unwrap().clone(), fee, tx2_parents); // Generation 2 let tx3_parents = vec![entry1.tx_id(), entry2.tx_id()].into_iter().collect(); - let entry3 = TxMempoolEntry::new(tx3, fee, tx3_parents); + let entry3 = TxMempoolEntry::new(txs.get(2).unwrap().clone(), fee, tx3_parents); // Generation 3 let tx4_parents = vec![entry3.tx_id()].into_iter().collect(); let tx5_parents = vec![entry3.tx_id()].into_iter().collect(); - let entry4 = TxMempoolEntry::new(tx4, fee, tx4_parents); - let entry5 = TxMempoolEntry::new(tx5, fee, tx5_parents); + let entry4 = TxMempoolEntry::new(txs.get(3).unwrap().clone(), fee, tx4_parents); + let entry5 = TxMempoolEntry::new(txs.get(4).unwrap().clone(), fee, tx5_parents); // Generation 4 let tx6_parents = vec![entry3.tx_id(), entry4.tx_id(), entry5.tx_id()].into_iter().collect(); - let entry6 = TxMempoolEntry::new(tx6, fee, tx6_parents); + let entry6 = TxMempoolEntry::new(txs.get(5).unwrap().clone(), fee, tx6_parents); let entries = vec![entry1, entry2, entry3, entry4, entry5, entry6]; let ids = entries.clone().into_iter().map(|entry| entry.tx_id()).collect::>(); - for (index, entry) in entries.into_iter().enumerate() { - println!("adding {}, (entry {})", entry.tx_id(), index + 1); + for entry in entries.into_iter() { mempool.store.add_tx(entry)?; - println!( - "after adding entry {}, store looks like this {:#?}", - index + 1, - mempool.store.txs_by_id - ); } let entry1 = mempool.store.get_entry(ids.get(0).expect("index")).expect("entry"); @@ -1304,9 +1295,12 @@ mod tests { assert_eq!(entry5.unconfirmed_ancestors(&mempool.store).len(), 3); assert_eq!(entry6.unconfirmed_ancestors(&mempool.store).len(), 5); - //println!("entry1 {:#?}", entry1); assert_eq!(entry1.count_with_descendants(), 5); - //assert_eq!(entry2.count_with_descendants(&mempool.store), 4); + assert_eq!(entry2.count_with_descendants(), 5); + assert_eq!(entry3.count_with_descendants(), 4); + assert_eq!(entry4.count_with_descendants(), 2); + assert_eq!(entry5.count_with_descendants(), 2); + assert_eq!(entry6.count_with_descendants(), 1); Ok(()) } From d1253b97ccc3560e9acd8d70c43430d184c93f02 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Fri, 18 Feb 2022 15:47:58 +0700 Subject: [PATCH 038/153] fee member of TxGenerator does not need to be an option --- mempool/src/pool.rs | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 99b6b62273..32b1258a8e 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -631,7 +631,7 @@ mod tests { coin_pool: BTreeSet, num_inputs: usize, num_outputs: usize, - tx_fee: Option, + tx_fee: Amount, } impl TxGenerator { @@ -650,7 +650,7 @@ mod tests { } fn with_fee(mut self, fee: Amount) -> Self { - self.tx_fee = Some(fee); + self.tx_fee = fee; self } @@ -685,13 +685,13 @@ mod tests { coin_pool, num_inputs, num_outputs, - tx_fee: None, + tx_fee: 0.into(), } } fn generate_tx(&mut self) -> anyhow::Result { let valued_inputs = self.generate_tx_inputs(); - let outputs = self.generate_tx_outputs(&valued_inputs, self.tx_fee)?; + let outputs = self.generate_tx_outputs(&valued_inputs)?; let locktime = 0; let flags = 0; let (inputs, _): (Vec, Vec) = valued_inputs.into_iter().unzip(); @@ -713,7 +713,7 @@ mod tests { fn generate_replaceable_tx(mut self) -> anyhow::Result { let valued_inputs = self.generate_tx_inputs(); - let outputs = self.generate_tx_outputs(&valued_inputs, self.tx_fee)?; + let outputs = self.generate_tx_outputs(&valued_inputs)?; let locktime = 0; let flags = 1; let (inputs, _values): (Vec, Vec) = valued_inputs.into_iter().unzip(); @@ -732,7 +732,6 @@ mod tests { fn generate_tx_outputs( &self, inputs: &[(TxInput, Amount)], - tx_fee: Option, ) -> anyhow::Result> { if self.num_outputs == 0 { return Ok(vec![]); @@ -746,11 +745,7 @@ mod tests { let sum_of_inputs = values.into_iter().sum::>().expect("Overflow in sum of input values"); - let total_to_spend = if let Some(fee) = tx_fee { - (sum_of_inputs - fee).expect("underflow") - } else { - sum_of_inputs - }; + let total_to_spend = (sum_of_inputs - self.tx_fee).expect("underflow"); let mut left_to_spend = total_to_spend; let mut outputs = Vec::new(); From 0eb6ec91c4037b500d675270caf7ad5b101221a6 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Fri, 18 Feb 2022 16:38:56 +0700 Subject: [PATCH 039/153] Move MAX_BIP125_REPLACEMENT_CANDIDATES to module scope, for testing --- mempool/src/pool.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 32b1258a8e..4842ed2862 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -21,6 +21,8 @@ const MAX_BLOCK_SIZE_BYTES: usize = 1_000_000; const MEMPOOL_MAX_TXS: usize = 1_000_000; +const MAX_BIP125_REPLACEMENT_CANDIATES: usize = 100; + impl TryGetFee for MempoolImpl { fn try_get_fee(&self, tx: &Transaction) -> Result { let inputs = tx @@ -395,7 +397,6 @@ impl MempoolImpl { &self, conflicts: &[&TxMempoolEntry], ) -> Result<(), TxValidationError> { - const MAX_BIP125_REPLACEMENT_CANDIATES: usize = 100; let mut num_potential_replacements = 0; for conflict in conflicts { num_potential_replacements += conflict.count_with_descendants(); From 232f9b0a206b522acddec554613b6dbc232eb83a Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Fri, 18 Feb 2022 15:48:18 +0700 Subject: [PATCH 040/153] BIP125 max replacements test --- mempool/src/pool.rs | 58 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 2 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 4842ed2862..dc7a530e85 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -21,7 +21,7 @@ const MAX_BLOCK_SIZE_BYTES: usize = 1_000_000; const MEMPOOL_MAX_TXS: usize = 1_000_000; -const MAX_BIP125_REPLACEMENT_CANDIATES: usize = 100; +const MAX_BIP125_REPLACEMENT_CANDIDATES: usize = 100; impl TryGetFee for MempoolImpl { fn try_get_fee(&self, tx: &Transaction) -> Result { @@ -400,7 +400,7 @@ impl MempoolImpl { let mut num_potential_replacements = 0; for conflict in conflicts { num_potential_replacements += conflict.count_with_descendants(); - if num_potential_replacements > MAX_BIP125_REPLACEMENT_CANDIATES { + if num_potential_replacements > MAX_BIP125_REPLACEMENT_CANDIDATES { return Err(TxValidationError::TooManyPotentialReplacements); } } @@ -1300,4 +1300,58 @@ mod tests { Ok(()) } + + fn test_bip125_max_replacements( + mempool: &mut MempoolImpl, + num_potential_replacements: usize, + ) -> anyhow::Result<()> { + let num_inputs = 1; + let num_outputs = num_potential_replacements - 1; + let tx = TxGenerator::new(mempool, num_inputs, num_outputs) + .generate_replaceable_tx() + .expect("generate_tx failed"); + let input = tx.get_inputs().first().expect("one input").to_owned(); + let outputs = tx.get_outputs().clone(); + let tx_id = tx.get_id(); + mempool.add_transaction(tx)?; + + let flags = 0; + let locktime = 0; + let outpoint_source_id = OutPointSourceId::Transaction(tx_id); + for (index, _) in outputs.iter().enumerate() { + let input = TxInput::new( + outpoint_source_id.clone(), + index.try_into().unwrap(), + DUMMY_WITNESS_MSG.to_vec(), + ); + let fee = Amount::from(0).into(); + let tx = tx_spend_input(mempool, input, fee, flags, locktime)?; + mempool.add_transaction(tx)?; + } + + let replacement_tx = + tx_spend_input(mempool, input, Amount::from(100).into(), flags, locktime)?; + mempool.add_transaction(replacement_tx).map_err(anyhow::Error::from) + } + + #[test] + fn too_many_conflicts() -> anyhow::Result<()> { + let mut mempool = setup(); + let num_potential_replacements = MAX_BIP125_REPLACEMENT_CANDIDATES + 1; + let err = test_bip125_max_replacements(&mut mempool, num_potential_replacements) + .expect_err("too many replacements"); + let real_err = anyhow::Error::downcast::(err).expect("failed to downcast"); + assert!(matches!( + real_err, + MempoolError::TxValidationError(TxValidationError::TooManyPotentialReplacements) + )); + Ok(()) + } + + #[test] + fn not_too_many_conflicts() -> anyhow::Result<()> { + let mut mempool = setup(); + let num_potential_replacements = MAX_BIP125_REPLACEMENT_CANDIDATES; + test_bip125_max_replacements(&mut mempool, num_potential_replacements) + } } From d6cf6c683bf7fd42ecd80dd128e5f32bb3e5e552 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Fri, 18 Feb 2022 17:02:19 +0700 Subject: [PATCH 041/153] Refactor rbf checks into separate function --- mempool/src/pool.rs | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index dc7a530e85..3d1f590cbc 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -352,6 +352,13 @@ impl MempoolImpl { return Err(TxValidationError::TransactionAlreadyInMempool); } + self.rbf_checks(tx)?; + self.verify_inputs_available(tx)?; + + Ok(()) + } + + fn rbf_checks(&self, tx: &Transaction) -> Result<(), TxValidationError> { let conflicts = tx .inputs() .iter() @@ -367,10 +374,8 @@ impl MempoolImpl { self.pays_more_than_conflicts(tx, &conflicts)?; self.potential_replacements_within_limit(&conflicts)?; + // TODO no new unconfirmed } - - self.verify_inputs_available(tx)?; - Ok(()) } @@ -1339,10 +1344,11 @@ mod tests { let mut mempool = setup(); let num_potential_replacements = MAX_BIP125_REPLACEMENT_CANDIDATES + 1; let err = test_bip125_max_replacements(&mut mempool, num_potential_replacements) - .expect_err("too many replacements"); - let real_err = anyhow::Error::downcast::(err).expect("failed to downcast"); + .expect_err("expected error TooManyPotentialReplacements") + .downcast() + .expect("failed to downcast"); assert!(matches!( - real_err, + err, MempoolError::TxValidationError(TxValidationError::TooManyPotentialReplacements) )); Ok(()) From ee5ab37c25e6ac211aaa607a59e6b203c91ded01 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Mon, 21 Feb 2022 18:21:06 +0700 Subject: [PATCH 042/153] Refactor tx_spend_several_inputs --- mempool/src/pool.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 3d1f590cbc..b8168e2cec 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -691,7 +691,7 @@ mod tests { coin_pool, num_inputs, num_outputs, - tx_fee: 0.into(), + tx_fee: Amount::from_atoms(0), } } @@ -1102,7 +1102,7 @@ mod tests { let child_tx = tx_spend_input( &mempool, child_tx_input.clone(), - Amount::from_atoms(10).into(), + Amount::from_atoms(10), flags, locktime, )?; @@ -1111,7 +1111,7 @@ mod tests { let replacement_tx = tx_spend_input( &mempool, child_tx_input, - Amount::from_atoms(15).into(), + Amount::from_atoms(15), flags, locktime, )?; @@ -1122,7 +1122,7 @@ mod tests { fn tx_spend_input( mempool: &MempoolImpl, input: TxInput, - fee: Option, + fee: impl Into>, flags: u32, locktime: u32, ) -> anyhow::Result { @@ -1329,13 +1329,13 @@ mod tests { index.try_into().unwrap(), DUMMY_WITNESS_MSG.to_vec(), ); - let fee = Amount::from(0).into(); + let fee = Amount::from_atoms(0); let tx = tx_spend_input(mempool, input, fee, flags, locktime)?; mempool.add_transaction(tx)?; } let replacement_tx = - tx_spend_input(mempool, input, Amount::from(100).into(), flags, locktime)?; + tx_spend_input(mempool, input, Amount::from_atoms(100), flags, locktime)?; mempool.add_transaction(replacement_tx).map_err(anyhow::Error::from) } From ccf59f3c86444982fb5de5045774256b6ace93e9 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Mon, 21 Feb 2022 19:15:53 +0700 Subject: [PATCH 043/153] ChainState trait inherits Debug --- mempool/src/pool.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index b8168e2cec..704cde3a46 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -59,7 +59,7 @@ pub trait Mempool { fn new_tip_set(&mut self) -> Result<(), MempoolError>; } -pub trait ChainState { +pub trait ChainState: Debug { fn contains_outpoint(&self, outpoint: &OutPoint) -> bool; fn get_outpoint_value(&self, outpoint: &OutPoint) -> Result; } @@ -293,7 +293,7 @@ impl From for MempoolError { } } -impl MempoolImpl { +impl MempoolImpl { fn verify_inputs_available(&self, tx: &Transaction) -> Result<(), TxValidationError> { tx.inputs() .iter() @@ -413,7 +413,7 @@ impl MempoolImpl { } } -impl Mempool for MempoolImpl { +impl Mempool for MempoolImpl { fn create(chain_state: C) -> Self { Self { store: MempoolStore::new(), From d4af46a3da161f3506a7ec2278bd6d9a3ba50d5b Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Mon, 21 Feb 2022 17:43:07 +0700 Subject: [PATCH 044/153] Validation - Replacing Tx spends no new unconfirmed outputs --- mempool/src/pool.rs | 77 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 76 insertions(+), 1 deletion(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 704cde3a46..a075746cea 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -285,6 +285,8 @@ pub enum TxValidationError { }, #[error("TooManyPotentialReplacements")] TooManyPotentialReplacements, + #[error("SpendsNewUnconfirmedInput")] + SpendsNewUnconfirmedOutput, } impl From for MempoolError { @@ -374,11 +376,34 @@ impl MempoolImpl { self.pays_more_than_conflicts(tx, &conflicts)?; self.potential_replacements_within_limit(&conflicts)?; - // TODO no new unconfirmed + self.spends_no_new_unconfirmed_outputs(tx, &conflicts)?; } Ok(()) } + fn spends_no_new_unconfirmed_outputs( + &self, + tx: &Transaction, + conflicts: &[&TxMempoolEntry], + ) -> Result<(), TxValidationError> { + let outpoints_spent_by_conflicts = conflicts + .iter() + .flat_map(|conflict| conflict.tx.get_inputs().iter().map(|input| input.get_outpoint())) + .collect::>(); + + tx.get_inputs() + .iter() + .find(|input| { + // input spends an unconfirmed output + input.spends_unconfirmed(self) && + // this unconfirmed output is not spent by one of the conflicts + !outpoints_spent_by_conflicts.contains(&input.get_outpoint()) + }) + .map_or(Ok(()), |_| { + Err(TxValidationError::SpendsNewUnconfirmedOutput) + }) + } + fn pays_more_than_conflicts( &self, tx: &Transaction, @@ -413,6 +438,18 @@ impl MempoolImpl { } } +trait SpendsUnconfirmed { + fn spends_unconfirmed(&self, mempool: &MempoolImpl) -> bool; +} + +impl SpendsUnconfirmed for TxInput { + fn spends_unconfirmed(&self, mempool: &MempoolImpl) -> bool { + mempool.contains_transaction( + self.get_outpoint().get_tx_id().get_tx_id().expect("Not coinbase"), + ) + } +} + impl Mempool for MempoolImpl { fn create(chain_state: C) -> Self { Self { @@ -1360,4 +1397,42 @@ mod tests { let num_potential_replacements = MAX_BIP125_REPLACEMENT_CANDIDATES; test_bip125_max_replacements(&mut mempool, num_potential_replacements) } + + #[test] + fn spends_new_unconfirmed() -> anyhow::Result<()> { + let mut mempool = setup(); + let num_inputs = 1; + let num_outputs = 2; + let tx = TxGenerator::new(&mempool, num_inputs, num_outputs) + .generate_replaceable_tx() + .expect("generate_replaceable_tx"); + let outpoint_source_id = OutPointSourceId::Transaction(tx.get_id()); + mempool.add_transaction(tx)?; + + let input1 = TxInput::new(outpoint_source_id.clone(), 0, DUMMY_WITNESS_MSG.to_vec()); + let input2 = TxInput::new(outpoint_source_id, 1, DUMMY_WITNESS_MSG.to_vec()); + + let locktime = 0; + let flags = 0; + let original_fee = Amount::from_atoms(0); + let replaced_tx = tx_spend_input(&mempool, input1.clone(), original_fee, flags, locktime)?; + mempool.add_transaction(replaced_tx)?; + let replacement_fee = Amount::from_atoms(10); + let incoming_tx = tx_spend_several_inputs( + &mempool, + &[input1, input2], + replacement_fee, + flags, + locktime, + )?; + + let res = mempool.add_transaction(incoming_tx); + assert!(matches!( + res, + Err(MempoolError::TxValidationError( + TxValidationError::SpendsNewUnconfirmedOutput + )) + )); + Ok(()) + } } From ea0de8e3fd0901505f32ad780db97a1a4c3fe579 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Tue, 22 Feb 2022 19:09:44 +0700 Subject: [PATCH 045/153] Clarity: Renaming and Comments --- mempool/src/pool.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index a075746cea..5201f6cfc3 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -374,8 +374,16 @@ impl MempoolImpl { .then(|| ()) .ok_or(TxValidationError::ConflictWithIrreplaceableTransaction)?; - self.pays_more_than_conflicts(tx, &conflicts)?; + // It's possible that the replacement pays more fees than its direct conflicts but not more + // than all conflicts (i.e. the direct conflicts have high-fee descendants). However, if the + // replacement doesn't pay more fees than its direct conflicts, then we can be sure it's not + // more economically rational to mine. Before we go digging through the mempool for all + // transactions that would need to be removed (direct conflicts and all descendants), check + // that the replacement transaction pays more than its direct conflicts. + self.pays_more_than_direct_conflicts(tx, &conflicts)?; + // Enforce BIP125 Rule #5. self.potential_replacements_within_limit(&conflicts)?; + // Enforce BIP125 Rule #2. self.spends_no_new_unconfirmed_outputs(tx, &conflicts)?; } Ok(()) @@ -404,7 +412,7 @@ impl MempoolImpl { }) } - fn pays_more_than_conflicts( + fn pays_more_than_direct_conflicts( &self, tx: &Transaction, conflicts: &[&TxMempoolEntry], From d6f518a2b34990a59d078a476acc8756336b7937 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Tue, 22 Feb 2022 19:10:00 +0700 Subject: [PATCH 046/153] Validation: Replacement Tx pays more than conflicts together with their descendants --- mempool/src/pool.rs | 59 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 56 insertions(+), 3 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 5201f6cfc3..476bb4aacf 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -131,6 +131,26 @@ impl TxMempoolEntry { } } } + + fn unconfirmed_descendants(&self, store: &MempoolStore) -> BTreeSet { + let mut visited = BTreeSet::new(); + self.unconfirmed_descendants_inner(&mut visited, store); + visited + } + + fn unconfirmed_descendants_inner(&self, visited: &mut BTreeSet, store: &MempoolStore) { + for child in self.children.iter() { + if visited.contains(child) { + continue; + } else { + visited.insert(child.to_owned()); + store + .get_entry(child) + .expect("entry") + .unconfirmed_descendants_inner(visited, store); + } + } + } } impl PartialOrd for TxMempoolEntry { @@ -287,6 +307,10 @@ pub enum TxValidationError { TooManyPotentialReplacements, #[error("SpendsNewUnconfirmedInput")] SpendsNewUnconfirmedOutput, + #[error("ConflictsFeeOverflow")] + ConflictsFeeOverflow, + #[error("TransactionFeeLowerThanConflictsWithDescendants")] + TransactionFeeLowerThanConflictsWithDescendants, } impl From for MempoolError { @@ -382,13 +406,36 @@ impl MempoolImpl { // that the replacement transaction pays more than its direct conflicts. self.pays_more_than_direct_conflicts(tx, &conflicts)?; // Enforce BIP125 Rule #5. - self.potential_replacements_within_limit(&conflicts)?; + let conflicts_with_descendants = + self.potential_replacements_within_limit(&conflicts)?; // Enforce BIP125 Rule #2. self.spends_no_new_unconfirmed_outputs(tx, &conflicts)?; + // Enforce BIP125 Rule #3. + self.pays_more_than_conflicts_with_descendants(tx, &conflicts_with_descendants)?; } Ok(()) } + fn pays_more_than_conflicts_with_descendants( + &self, + tx: &Transaction, + conflicts_with_descendants: &BTreeSet, + ) -> Result<(), TxValidationError> { + let conflicts_with_descendants = conflicts_with_descendants.iter().map(|conflict_id| { + self.store.txs_by_id.get(conflict_id).expect("tx should exist in mempool") + }); + + let total_conflict_fees = conflicts_with_descendants + .map(|conflict| conflict.fee) + .sum::>() + .ok_or(TxValidationError::ConflictsFeeOverflow)?; + let replacement_fee = self.try_get_fee(tx)?; + (replacement_fee > total_conflict_fees) + .then(|| ()) + .ok_or(TxValidationError::TransactionFeeLowerThanConflictsWithDescendants)?; + Ok(()) + } + fn spends_no_new_unconfirmed_outputs( &self, tx: &Transaction, @@ -434,7 +481,7 @@ impl MempoolImpl { fn potential_replacements_within_limit( &self, conflicts: &[&TxMempoolEntry], - ) -> Result<(), TxValidationError> { + ) -> Result, TxValidationError> { let mut num_potential_replacements = 0; for conflict in conflicts { num_potential_replacements += conflict.count_with_descendants(); @@ -442,7 +489,13 @@ impl MempoolImpl { return Err(TxValidationError::TooManyPotentialReplacements); } } - Ok(()) + let replacements_with_descendants = conflicts + .iter() + .flat_map(|conflict| conflict.unconfirmed_descendants(&self.store)) + .chain(conflicts.iter().map(|conflict| conflict.tx_id())) + .collect(); + + Ok(replacements_with_descendants) } } From 99e356448169ffb15f84bd34e4e265ce7bbe5731 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Tue, 22 Feb 2022 19:53:27 +0700 Subject: [PATCH 047/153] Test pays_more_than_conflicts_with_descendants --- mempool/src/pool.rs | 77 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 476bb4aacf..8abcf35c94 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -1496,4 +1496,81 @@ mod tests { )); Ok(()) } + + #[test] + fn pays_more_than_conflicts_with_descendants() -> anyhow::Result<()> { + let mut mempool = setup(); + let num_inputs = 1; + let num_outputs = 1; + let tx = TxGenerator::new(&mempool, num_inputs, num_outputs) + .generate_replaceable_tx() + .expect("generate_replaceable_tx"); + let tx_id = tx.get_id(); + mempool.add_transaction(tx)?; + + let outpoint_source_id = OutPointSourceId::Transaction(tx_id); + let input = TxInput::new(outpoint_source_id, 0, DUMMY_WITNESS_MSG.to_vec()); + + let locktime = 0; + let rbf = 1; + let no_rbf = 0; + + // Create transaction that we will attempt to replace + let original_fee = Amount::from(10); + let replaced_tx = tx_spend_input(&mempool, input.clone(), original_fee, rbf, locktime)?; + let replaced_id = replaced_tx.get_id(); + mempool.add_transaction(replaced_tx)?; + + // Create some children for this transaction + let descendant_outpoint_source_id = OutPointSourceId::Transaction(replaced_id); + + let descendant1_fee = Amount::from(10); + let descendant1 = tx_spend_input( + &mempool, + TxInput::new( + descendant_outpoint_source_id.clone(), + 0, + DUMMY_WITNESS_MSG.to_vec(), + ), + descendant1_fee, + no_rbf, + locktime, + )?; + mempool.add_transaction(descendant1)?; + + let descendant2_fee = Amount::from(10); + let descendant2 = tx_spend_input( + &mempool, + TxInput::new(descendant_outpoint_source_id, 1, DUMMY_WITNESS_MSG.to_vec()), + descendant2_fee, + no_rbf, + locktime, + )?; + mempool.add_transaction(descendant2)?; + + //Create a new incoming transaction that conflicts with `replaced_tx` because it spends + //`input`. It will be rejected because its fee exactly equals (so is not greater than) the + //sum of the fees of the conflict together with its descendants + let insufficient_rbf_fee = Amount::from(30); + let incoming_tx = tx_spend_input( + &mempool, + input.clone(), + insufficient_rbf_fee, + no_rbf, + locktime, + )?; + + assert!(matches!( + mempool.add_transaction(incoming_tx), + Err(MempoolError::TxValidationError( + TxValidationError::TransactionFeeLowerThanConflictsWithDescendants + )) + )); + + let sufficient_rbf_fee = Amount::from(31); + let incoming_tx = tx_spend_input(&mempool, input, sufficient_rbf_fee, no_rbf, locktime)?; + assert!(mempool.add_transaction(incoming_tx).is_ok()); + + Ok(()) + } } From bcb05775a1f8aa103d9668182e3196d74fb39cd5 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Wed, 23 Feb 2022 14:05:03 +0700 Subject: [PATCH 048/153] derive(Debug) for test struct ValuedOutPoint --- mempool/src/pool.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 8abcf35c94..a204e5f83b 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -576,7 +576,7 @@ mod tests { const DUMMY_WITNESS_MSG: &[u8] = b"dummy_witness_msg"; - #[derive(PartialEq, Eq, PartialOrd, Ord, Clone)] + #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)] struct ValuedOutPoint { outpoint: OutPoint, value: Amount, From dce5337805ca71ca88df9bd121eb0ef3bdeeac6d Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Wed, 23 Feb 2022 14:38:22 +0700 Subject: [PATCH 049/153] Add comment and reorder RBF validation calls --- mempool/src/pool.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index a204e5f83b..9c73d67306 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -393,6 +393,7 @@ impl MempoolImpl { .collect::>(); for entry in &conflicts { + // Enforce BIP125 Rule #1. entry .is_replaceable(&self.store) .then(|| ()) @@ -405,11 +406,11 @@ impl MempoolImpl { // transactions that would need to be removed (direct conflicts and all descendants), check // that the replacement transaction pays more than its direct conflicts. self.pays_more_than_direct_conflicts(tx, &conflicts)?; + // Enforce BIP125 Rule #2. + self.spends_no_new_unconfirmed_outputs(tx, &conflicts)?; // Enforce BIP125 Rule #5. let conflicts_with_descendants = self.potential_replacements_within_limit(&conflicts)?; - // Enforce BIP125 Rule #2. - self.spends_no_new_unconfirmed_outputs(tx, &conflicts)?; // Enforce BIP125 Rule #3. self.pays_more_than_conflicts_with_descendants(tx, &conflicts_with_descendants)?; } From 0bb258594a3f4b1a0b22a4a847eedb963a0fdbc8 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Wed, 23 Feb 2022 14:42:19 +0700 Subject: [PATCH 050/153] pays_more_than_conflicts_with_descendant returns total fee of conflicts and descendants --- mempool/src/pool.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 9c73d67306..f4a0446fc6 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -421,7 +421,7 @@ impl MempoolImpl { &self, tx: &Transaction, conflicts_with_descendants: &BTreeSet, - ) -> Result<(), TxValidationError> { + ) -> Result { let conflicts_with_descendants = conflicts_with_descendants.iter().map(|conflict_id| { self.store.txs_by_id.get(conflict_id).expect("tx should exist in mempool") }); @@ -434,7 +434,7 @@ impl MempoolImpl { (replacement_fee > total_conflict_fees) .then(|| ()) .ok_or(TxValidationError::TransactionFeeLowerThanConflictsWithDescendants)?; - Ok(()) + Ok(total_conflict_fees) } fn spends_no_new_unconfirmed_outputs( From 3c4641b8dd69a9f92eca0405d63e61065a066b30 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Wed, 23 Feb 2022 15:09:25 +0700 Subject: [PATCH 051/153] Validation: replacement tx pays for its bandwidth --- mempool/src/pool.rs | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index f4a0446fc6..f80d180c4c 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -22,6 +22,8 @@ const MAX_BLOCK_SIZE_BYTES: usize = 1_000_000; const MEMPOOL_MAX_TXS: usize = 1_000_000; const MAX_BIP125_REPLACEMENT_CANDIDATES: usize = 100; +// TODO this should really be taken from some global node settings +const RELAY_FEE_PER_BYTE: usize = 1; impl TryGetFee for MempoolImpl { fn try_get_fee(&self, tx: &Transaction) -> Result { @@ -311,6 +313,10 @@ pub enum TxValidationError { ConflictsFeeOverflow, #[error("TransactionFeeLowerThanConflictsWithDescendants")] TransactionFeeLowerThanConflictsWithDescendants, + #[error("AdditionalFeesUnderflow")] + AdditionalFeesUnderflow, + #[error("InsufficientFeesToRelay")] + InsufficientFeesToRelay, } impl From for MempoolError { @@ -412,11 +418,28 @@ impl MempoolImpl { let conflicts_with_descendants = self.potential_replacements_within_limit(&conflicts)?; // Enforce BIP125 Rule #3. - self.pays_more_than_conflicts_with_descendants(tx, &conflicts_with_descendants)?; + let total_conflict_fees = + self.pays_more_than_conflicts_with_descendants(tx, &conflicts_with_descendants)?; + self.pays_for_bandwidth(tx, total_conflict_fees)?; } Ok(()) } + fn pays_for_bandwidth( + &self, + tx: &Transaction, + total_conflict_fees: Amount, + ) -> Result<(), TxValidationError> { + let additional_fees = (self.try_get_fee(tx)? - total_conflict_fees) + .ok_or(TxValidationError::AdditionalFeesUnderflow)?; + // TODO should we return an error here instead of expect? + let relay_fee = + Amount::from(u128::try_from(tx.encoded_size() * RELAY_FEE_PER_BYTE).expect("Overflow")); + (additional_fees >= relay_fee) + .then(|| ()) + .ok_or(TxValidationError::InsufficientFeesToRelay) + } + fn pays_more_than_conflicts_with_descendants( &self, tx: &Transaction, From 78fd4f51b171bade5e321becd666e4fd17a466e0 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Wed, 23 Feb 2022 15:25:19 +0700 Subject: [PATCH 052/153] Accomodate for relay fee checks in older tests --- mempool/src/pool.rs | 47 ++++++++++++++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index f80d180c4c..d136739643 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -433,8 +433,9 @@ impl MempoolImpl { let additional_fees = (self.try_get_fee(tx)? - total_conflict_fees) .ok_or(TxValidationError::AdditionalFeesUnderflow)?; // TODO should we return an error here instead of expect? - let relay_fee = - Amount::from(u128::try_from(tx.encoded_size() * RELAY_FEE_PER_BYTE).expect("Overflow")); + let relay_fee = Amount::from_atoms( + u128::try_from(tx.encoded_size() * RELAY_FEE_PER_BYTE).expect("Overflow"), + ); (additional_fees >= relay_fee) .then(|| ()) .ok_or(TxValidationError::InsufficientFeesToRelay) @@ -1185,7 +1186,17 @@ mod tests { #[test] fn tx_replace() -> anyhow::Result<()> { - assert!(test_replace_tx(Amount::from_atoms(10), Amount::from_atoms(15)).is_ok()); + let relay_fee = + u128::try_from(TX_GENERATOR_SINGLE_INPUT_SINGLE_OUTPUT_SIZE * RELAY_FEE_PER_BYTE) + .expect("relay fee overflow"); + let replacement_fee = Amount::from_atoms(relay_fee + 100); + assert!(test_replace_tx(Amount::from_atoms(100), replacement_fee).is_ok()); + assert!(matches!( + test_replace_tx(Amount::from_atoms(100), Amount::from_atoms(relay_fee + 99)), + Err(MempoolError::TxValidationError( + TxValidationError::InsufficientFeesToRelay + )) + )); assert!(matches!( test_replace_tx(Amount::from_atoms(10), Amount::from_atoms(10)), Err(MempoolError::TxValidationError( @@ -1230,17 +1241,21 @@ mod tests { )?; mempool.add_transaction(child_tx)?; - let replacement_tx = tx_spend_input( - &mempool, - child_tx_input, - Amount::from_atoms(15), - flags, - locktime, - )?; + let relay_fee = + u128::try_from(TX_SPEND_INPUT_SIZE * RELAY_FEE_PER_BYTE).expect("relay fee overflow"); + let replacement_fee = Amount::from_atoms(relay_fee + 15); + let replacement_tx = + tx_spend_input(&mempool, child_tx_input, replacement_fee, flags, locktime)?; mempool.add_transaction(replacement_tx)?; Ok(()) } + // To test our validation of BIP125 Rule#4 (replacement transaction pays for its own bandwidth), we need to know the necessary relay fee before creating the transaction. The relay fee depends on the size of the transaction. The usual way to get the size of a transaction is to call `tx.encoded_size` but we cannot do this until we have created the transaction itself. To get around this scycle, we have precomputed the size of all transaction created by `tx_spend_input`. This value will be the same for all transactions created by this function. + const TX_SPEND_INPUT_SIZE: usize = 82; + // Similarly, we have precomputed the value of any single-input, single-output transaction + // created using `TxGenerator`. + const TX_GENERATOR_SINGLE_INPUT_SINGLE_OUTPUT_SIZE: usize = 74; + fn tx_spend_input( mempool: &MempoolImpl, input: TxInput, @@ -1540,7 +1555,7 @@ mod tests { let no_rbf = 0; // Create transaction that we will attempt to replace - let original_fee = Amount::from(10); + let original_fee = Amount::from_atoms(100); let replaced_tx = tx_spend_input(&mempool, input.clone(), original_fee, rbf, locktime)?; let replaced_id = replaced_tx.get_id(); mempool.add_transaction(replaced_tx)?; @@ -1548,7 +1563,7 @@ mod tests { // Create some children for this transaction let descendant_outpoint_source_id = OutPointSourceId::Transaction(replaced_id); - let descendant1_fee = Amount::from(10); + let descendant1_fee = Amount::from_atoms(100); let descendant1 = tx_spend_input( &mempool, TxInput::new( @@ -1562,7 +1577,7 @@ mod tests { )?; mempool.add_transaction(descendant1)?; - let descendant2_fee = Amount::from(10); + let descendant2_fee = Amount::from_atoms(100); let descendant2 = tx_spend_input( &mempool, TxInput::new(descendant_outpoint_source_id, 1, DUMMY_WITNESS_MSG.to_vec()), @@ -1575,7 +1590,7 @@ mod tests { //Create a new incoming transaction that conflicts with `replaced_tx` because it spends //`input`. It will be rejected because its fee exactly equals (so is not greater than) the //sum of the fees of the conflict together with its descendants - let insufficient_rbf_fee = Amount::from(30); + let insufficient_rbf_fee = Amount::from_atoms(300); let incoming_tx = tx_spend_input( &mempool, input.clone(), @@ -1591,7 +1606,9 @@ mod tests { )) )); - let sufficient_rbf_fee = Amount::from(31); + let relay_fee = + u128::try_from(TX_SPEND_INPUT_SIZE * RELAY_FEE_PER_BYTE).expect("relay fee overflow"); + let sufficient_rbf_fee = Amount::from_atoms(300 + relay_fee); let incoming_tx = tx_spend_input(&mempool, input, sufficient_rbf_fee, no_rbf, locktime)?; assert!(mempool.add_transaction(incoming_tx).is_ok()); From 5c207915128b957caabc5da94ed29be07a1f807e Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Wed, 23 Feb 2022 16:44:29 +0700 Subject: [PATCH 053/153] Refactor TxGenerator: builder pattern for num_inputs, num_outputs --- mempool/src/pool.rs | 90 +++++++++++++++++---------------------------- 1 file changed, 33 insertions(+), 57 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index d136739643..dfcea51a91 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -764,18 +764,9 @@ mod tests { } impl TxGenerator { - fn new( - mempool: &MempoolImpl, - num_inputs: usize, - num_outputs: usize, - ) -> Self { + fn new(mempool: &MempoolImpl) -> Self { let unconfirmed_outputs = BTreeSet::new(); - Self::create_tx_generator( - &mempool.chain_state, - &unconfirmed_outputs, - num_inputs, - num_outputs, - ) + Self::create_tx_generator(&mempool.chain_state, &unconfirmed_outputs) } fn with_fee(mut self, fee: Amount) -> Self { @@ -783,25 +774,24 @@ mod tests { self } - fn new_with_unconfirmed( - mempool: &MempoolImpl, - num_inputs: usize, - num_outputs: usize, - ) -> Self { + fn with_num_inputs(mut self, num_inputs: usize) -> Self { + self.num_inputs = num_inputs; + self + } + + fn with_num_outputs(mut self, num_outputs: usize) -> Self { + self.num_outputs = num_outputs; + self + } + + fn new_with_unconfirmed(mempool: &MempoolImpl) -> Self { let unconfirmed_outputs = mempool.available_outpoints(); - Self::create_tx_generator( - &mempool.chain_state, - &unconfirmed_outputs, - num_inputs, - num_outputs, - ) + Self::create_tx_generator(&mempool.chain_state, &unconfirmed_outputs) } fn create_tx_generator( chain_state: &ChainStateMock, unconfirmed_outputs: &BTreeSet, - num_inputs: usize, - num_outputs: usize, ) -> Self { let coin_pool = chain_state .unspent_outpoints() @@ -812,8 +802,8 @@ mod tests { Self { coin_pool, - num_inputs, - num_outputs, + num_inputs: 1, + num_outputs: 1, tx_fee: Amount::from_atoms(0), } } @@ -971,10 +961,8 @@ mod tests { #[test] fn txs_sorted() -> anyhow::Result<()> { let chain_state = ChainStateMock::new(); - let num_inputs = 1; - let num_outputs = 1; let mut mempool = MempoolImpl::create(chain_state); - let mut tx_generator = TxGenerator::new(&mempool, num_inputs, num_outputs); + let mut tx_generator = TxGenerator::new(&mempool); let target_txs = 100; for _ in 0..target_txs { @@ -1000,9 +988,8 @@ mod tests { #[test] fn tx_no_inputs() -> anyhow::Result<()> { let mut mempool = setup(); - let num_inputs = 0; - let num_outputs = 1; - let tx = TxGenerator::new(&mempool, num_inputs, num_outputs) + let tx = TxGenerator::new(&mempool) + .with_num_inputs(0) .generate_tx() .expect("generate_tx failed"); assert!(matches!( @@ -1019,9 +1006,8 @@ mod tests { #[test] fn tx_no_outputs() -> anyhow::Result<()> { let mut mempool = setup(); - let num_inputs = 1; - let num_outputs = 0; - let tx = TxGenerator::new(&mempool, num_inputs, num_outputs) + let tx = TxGenerator::new(&mempool) + .with_num_outputs(0) .generate_tx() .expect("generate_tx failed"); assert!(matches!( @@ -1151,9 +1137,8 @@ mod tests { #[test] fn tx_too_big() -> anyhow::Result<()> { let mut mempool = setup(); - let num_inputs = 1; - let num_outputs = 400_000; - let tx = TxGenerator::new(&mempool, num_inputs, num_outputs) + let tx = TxGenerator::new(&mempool) + .with_num_outputs(400_000) .generate_tx() .expect("generate_tx failed"); assert!(matches!( @@ -1167,15 +1152,13 @@ mod tests { fn test_replace_tx(original_fee: Amount, replacement_fee: Amount) -> Result<(), MempoolError> { let mut mempool = setup(); - let num_inputs = 1; - let num_outputs = 1; - let tx = TxGenerator::new(&mempool, num_inputs, num_outputs) + let tx = TxGenerator::new(&mempool) .with_fee(original_fee) .generate_replaceable_tx() .expect("generate_replaceable_tx"); mempool.add_transaction(tx)?; - let tx = TxGenerator::new(&mempool, num_inputs, num_outputs) + let tx = TxGenerator::new(&mempool) .with_fee(replacement_fee) .generate_tx() .expect("generate_tx_failed"); @@ -1215,9 +1198,7 @@ mod tests { #[test] fn tx_replace_child() -> anyhow::Result<()> { let mut mempool = setup(); - let num_inputs = 1; - let num_outputs = 1; - let tx = TxGenerator::new_with_unconfirmed(&mempool, num_inputs, num_outputs) + let tx = TxGenerator::new_with_unconfirmed(&mempool) .generate_replaceable_tx() .expect("generate_replaceable_tx"); mempool.add_transaction(tx.clone())?; @@ -1302,9 +1283,8 @@ mod tests { #[test] fn one_ancestor_signal_is_enough() -> anyhow::Result<()> { let mut mempool = setup(); - let num_inputs = 1; - let num_outputs = 2; - let tx = TxGenerator::new_with_unconfirmed(&mempool, num_inputs, num_outputs) + let tx = TxGenerator::new_with_unconfirmed(&mempool) + .with_num_outputs(2) .generate_tx() .expect("generate_replaceable_tx"); @@ -1447,9 +1427,8 @@ mod tests { mempool: &mut MempoolImpl, num_potential_replacements: usize, ) -> anyhow::Result<()> { - let num_inputs = 1; - let num_outputs = num_potential_replacements - 1; - let tx = TxGenerator::new(mempool, num_inputs, num_outputs) + let tx = TxGenerator::new(mempool) + .with_num_outputs(num_potential_replacements - 1) .generate_replaceable_tx() .expect("generate_tx failed"); let input = tx.get_inputs().first().expect("one input").to_owned(); @@ -1501,9 +1480,8 @@ mod tests { #[test] fn spends_new_unconfirmed() -> anyhow::Result<()> { let mut mempool = setup(); - let num_inputs = 1; - let num_outputs = 2; - let tx = TxGenerator::new(&mempool, num_inputs, num_outputs) + let tx = TxGenerator::new(&mempool) + .with_num_outputs(2) .generate_replaceable_tx() .expect("generate_replaceable_tx"); let outpoint_source_id = OutPointSourceId::Transaction(tx.get_id()); @@ -1539,9 +1517,7 @@ mod tests { #[test] fn pays_more_than_conflicts_with_descendants() -> anyhow::Result<()> { let mut mempool = setup(); - let num_inputs = 1; - let num_outputs = 1; - let tx = TxGenerator::new(&mempool, num_inputs, num_outputs) + let tx = TxGenerator::new(&mempool) .generate_replaceable_tx() .expect("generate_replaceable_tx"); let tx_id = tx.get_id(); From 194f4eb3975727cb20cd2083c152411652954432 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Wed, 23 Feb 2022 17:00:20 +0700 Subject: [PATCH 054/153] TxGenerator: builder pattern for replaceability flag --- mempool/src/pool.rs | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index dfcea51a91..a7b6fc2239 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -761,6 +761,7 @@ mod tests { num_inputs: usize, num_outputs: usize, tx_fee: Amount, + replaceable: bool, } impl TxGenerator { @@ -784,6 +785,11 @@ mod tests { self } + fn replaceable(mut self) -> Self { + self.replaceable = true; + self + } + fn new_with_unconfirmed(mempool: &MempoolImpl) -> Self { let unconfirmed_outputs = mempool.available_outpoints(); Self::create_tx_generator(&mempool.chain_state, &unconfirmed_outputs) @@ -805,6 +811,7 @@ mod tests { num_inputs: 1, num_outputs: 1, tx_fee: Amount::from_atoms(0), + replaceable: false, } } @@ -812,7 +819,7 @@ mod tests { let valued_inputs = self.generate_tx_inputs(); let outputs = self.generate_tx_outputs(&valued_inputs)?; let locktime = 0; - let flags = 0; + let flags = if self.replaceable { 1 } else { 0 }; let (inputs, _): (Vec, Vec) = valued_inputs.into_iter().unzip(); let spent_outpoints = inputs.iter().map(|input| input.outpoint()).collect::>(); @@ -830,17 +837,6 @@ mod tests { Ok(tx) } - fn generate_replaceable_tx(mut self) -> anyhow::Result { - let valued_inputs = self.generate_tx_inputs(); - let outputs = self.generate_tx_outputs(&valued_inputs)?; - let locktime = 0; - let flags = 1; - let (inputs, _values): (Vec, Vec) = valued_inputs.into_iter().unzip(); - let tx = Transaction::new(flags, inputs, outputs, locktime)?; - assert!(tx.is_replaceable()); - Ok(tx) - } - fn generate_tx_inputs(&mut self) -> Vec<(TxInput, Amount)> { std::iter::repeat(()) .take(self.num_inputs) @@ -1154,7 +1150,8 @@ mod tests { let mut mempool = setup(); let tx = TxGenerator::new(&mempool) .with_fee(original_fee) - .generate_replaceable_tx() + .replaceable() + .generate_tx() .expect("generate_replaceable_tx"); mempool.add_transaction(tx)?; @@ -1199,7 +1196,8 @@ mod tests { fn tx_replace_child() -> anyhow::Result<()> { let mut mempool = setup(); let tx = TxGenerator::new_with_unconfirmed(&mempool) - .generate_replaceable_tx() + .replaceable() + .generate_tx() .expect("generate_replaceable_tx"); mempool.add_transaction(tx.clone())?; @@ -1429,7 +1427,8 @@ mod tests { ) -> anyhow::Result<()> { let tx = TxGenerator::new(mempool) .with_num_outputs(num_potential_replacements - 1) - .generate_replaceable_tx() + .replaceable() + .generate_tx() .expect("generate_tx failed"); let input = tx.get_inputs().first().expect("one input").to_owned(); let outputs = tx.get_outputs().clone(); @@ -1482,7 +1481,8 @@ mod tests { let mut mempool = setup(); let tx = TxGenerator::new(&mempool) .with_num_outputs(2) - .generate_replaceable_tx() + .replaceable() + .generate_tx() .expect("generate_replaceable_tx"); let outpoint_source_id = OutPointSourceId::Transaction(tx.get_id()); mempool.add_transaction(tx)?; @@ -1518,7 +1518,8 @@ mod tests { fn pays_more_than_conflicts_with_descendants() -> anyhow::Result<()> { let mut mempool = setup(); let tx = TxGenerator::new(&mempool) - .generate_replaceable_tx() + .replaceable() + .generate_tx() .expect("generate_replaceable_tx"); let tx_id = tx.get_id(); mempool.add_transaction(tx)?; From cfe46ac8c13eaced4c02a7679012cd2f2aeba0e8 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Wed, 23 Feb 2022 17:02:07 +0700 Subject: [PATCH 055/153] Remove outdated comments --- mempool/src/pool.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index a7b6fc2239..9e7d39004e 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -951,9 +951,6 @@ mod tests { Ok(()) } - // The "fees" now a are calculated as sum of the outputs - // This test creates transactions with a single input and a single output to check that the - // mempool sorts txs by fee #[test] fn txs_sorted() -> anyhow::Result<()> { let chain_state = ChainStateMock::new(); From 342f07394164b20ccfc24d375cfee3fbef7edae4 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Wed, 23 Feb 2022 17:16:53 +0700 Subject: [PATCH 056/153] wip delete TxGenerator::new and rename TxGenerator::new_with_unconfirmed to TxGenerator::new (this makes the tx_replace test fail because it is badly written) --- mempool/src/pool.rs | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 9e7d39004e..c19d0ef263 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -765,11 +765,6 @@ mod tests { } impl TxGenerator { - fn new(mempool: &MempoolImpl) -> Self { - let unconfirmed_outputs = BTreeSet::new(); - Self::create_tx_generator(&mempool.chain_state, &unconfirmed_outputs) - } - fn with_fee(mut self, fee: Amount) -> Self { self.tx_fee = fee; self @@ -790,7 +785,7 @@ mod tests { self } - fn new_with_unconfirmed(mempool: &MempoolImpl) -> Self { + fn new(mempool: &MempoolImpl) -> Self { let unconfirmed_outputs = mempool.available_outpoints(); Self::create_tx_generator(&mempool.chain_state, &unconfirmed_outputs) } @@ -1192,7 +1187,7 @@ mod tests { #[test] fn tx_replace_child() -> anyhow::Result<()> { let mut mempool = setup(); - let tx = TxGenerator::new_with_unconfirmed(&mempool) + let tx = TxGenerator::new(&mempool) .replaceable() .generate_tx() .expect("generate_replaceable_tx"); @@ -1278,7 +1273,7 @@ mod tests { #[test] fn one_ancestor_signal_is_enough() -> anyhow::Result<()> { let mut mempool = setup(); - let tx = TxGenerator::new_with_unconfirmed(&mempool) + let tx = TxGenerator::new(&mempool) .with_num_outputs(2) .generate_tx() .expect("generate_replaceable_tx"); From d09cf9da43e42f6f0c888a5244aba58e85daf08c Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Wed, 23 Feb 2022 17:29:33 +0700 Subject: [PATCH 057/153] Make test_replace_tx use tx_spend_input instead of TxGenerator --- mempool/src/pool.rs | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index c19d0ef263..797712a994 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -765,11 +765,6 @@ mod tests { } impl TxGenerator { - fn with_fee(mut self, fee: Amount) -> Self { - self.tx_fee = fee; - self - } - fn with_num_inputs(mut self, num_inputs: usize) -> Self { self.num_inputs = num_inputs; self @@ -1140,27 +1135,37 @@ mod tests { fn test_replace_tx(original_fee: Amount, replacement_fee: Amount) -> Result<(), MempoolError> { let mut mempool = setup(); - let tx = TxGenerator::new(&mempool) - .with_fee(original_fee) - .replaceable() - .generate_tx() - .expect("generate_replaceable_tx"); - mempool.add_transaction(tx)?; + let outpoint = mempool + .available_outpoints() + .iter() + .next() + .expect("there should be an outpoint since setup creates the genesis transaction") + .outpoint + .to_owned(); - let tx = TxGenerator::new(&mempool) - .with_fee(replacement_fee) - .generate_tx() - .expect("generate_tx_failed"); + let outpoint_source_id = OutPointSourceId::from( + outpoint.get_tx_id().get_tx_id().expect("Not Coinbase").to_owned(), + ); + + let input = TxInput::new(outpoint_source_id, 0, DUMMY_WITNESS_MSG.to_vec()); + let flags = 1; + let locktime = 0; + let tx = tx_spend_input(&mempool, input.clone(), original_fee, flags, locktime) + .expect("should be able to spend here"); + mempool.add_transaction(tx)?; + let flags = 0; + let tx = tx_spend_input(&mempool, input, replacement_fee, flags, locktime) + .expect("should be able to spend here"); mempool.add_transaction(tx)?; + Ok(()) } #[test] fn tx_replace() -> anyhow::Result<()> { let relay_fee = - u128::try_from(TX_GENERATOR_SINGLE_INPUT_SINGLE_OUTPUT_SIZE * RELAY_FEE_PER_BYTE) - .expect("relay fee overflow"); + u128::try_from(TX_SPEND_INPUT_SIZE * RELAY_FEE_PER_BYTE).expect("relay fee overflow"); let replacement_fee = Amount::from_atoms(relay_fee + 100); assert!(test_replace_tx(Amount::from_atoms(100), replacement_fee).is_ok()); assert!(matches!( @@ -1223,9 +1228,6 @@ mod tests { // To test our validation of BIP125 Rule#4 (replacement transaction pays for its own bandwidth), we need to know the necessary relay fee before creating the transaction. The relay fee depends on the size of the transaction. The usual way to get the size of a transaction is to call `tx.encoded_size` but we cannot do this until we have created the transaction itself. To get around this scycle, we have precomputed the size of all transaction created by `tx_spend_input`. This value will be the same for all transactions created by this function. const TX_SPEND_INPUT_SIZE: usize = 82; - // Similarly, we have precomputed the value of any single-input, single-output transaction - // created using `TxGenerator`. - const TX_GENERATOR_SINGLE_INPUT_SINGLE_OUTPUT_SIZE: usize = 74; fn tx_spend_input( mempool: &MempoolImpl, From 592801c1cc9e9fb061f43ef54ff7a10ad1cebd19 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Thu, 24 Feb 2022 12:54:21 +0700 Subject: [PATCH 058/153] Rename MempoolError to simply Error within the context of the crate --- mempool/src/lib.rs | 2 + mempool/src/pool.rs | 109 +++++++++++++++++++++++++------------------- 2 files changed, 63 insertions(+), 48 deletions(-) diff --git a/mempool/src/lib.rs b/mempool/src/lib.rs index 167730954a..db81f664cf 100644 --- a/mempool/src/lib.rs +++ b/mempool/src/lib.rs @@ -1,3 +1,5 @@ #![deny(clippy::clone_on_ref_ptr)] pub mod pool; + +pub use pool::Error as MempoolError; diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 797712a994..8e91392011 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -54,11 +54,11 @@ impl TryGetFee for MempoolImpl { pub trait Mempool { fn create(chain_state: C) -> Self; - fn add_transaction(&mut self, tx: Transaction) -> Result<(), MempoolError>; + fn add_transaction(&mut self, tx: Transaction) -> Result<(), Error>; fn get_all(&self) -> Vec<&Transaction>; fn contains_transaction(&self, tx: &Id) -> bool; fn drop_transaction(&mut self, tx: &Id); - fn new_tip_set(&mut self) -> Result<(), MempoolError>; + fn new_tip_set(&mut self) -> Result<(), Error>; } pub trait ChainState: Debug { @@ -145,7 +145,7 @@ impl TxMempoolEntry { if visited.contains(child) { continue; } else { - visited.insert(child.to_owned()); + visited.insert(*child); store .get_entry(child) .expect("entry") @@ -199,10 +199,10 @@ impl MempoolStore { &self, outpoint: &OutPoint, ) -> Result { - let tx_id = outpoint.tx_id().get_tx_id().expect("Not coinbase").clone(); + let tx_id = *outpoint.tx_id().get_tx_id().expect("Not coinbase"); let err = || TxValidationError::OutPointNotFound { outpoint: outpoint.clone(), - tx_id: tx_id.clone(), + tx_id, }; self.txs_by_id .get(&tx_id.get()) @@ -241,7 +241,7 @@ impl MempoolStore { } } - fn add_tx(&mut self, entry: TxMempoolEntry) -> Result<(), MempoolError> { + fn add_tx(&mut self, entry: TxMempoolEntry) -> Result<(), Error> { self.append_to_parents(&entry); self.update_ancestor_count(&entry); self.mark_outpoints_as_spent(&entry); @@ -270,7 +270,7 @@ impl MempoolStore { } #[derive(Debug, Error)] -pub enum MempoolError { +pub enum Error { #[error("Mempool is full")] MempoolFull, #[error(transparent)] @@ -319,9 +319,9 @@ pub enum TxValidationError { InsufficientFeesToRelay, } -impl From for MempoolError { +impl From for Error { fn from(e: TxValidationError) -> Self { - MempoolError::TxValidationError(e) + Error::TxValidationError(e) } } @@ -468,16 +468,16 @@ impl MempoolImpl { ) -> Result<(), TxValidationError> { let outpoints_spent_by_conflicts = conflicts .iter() - .flat_map(|conflict| conflict.tx.get_inputs().iter().map(|input| input.get_outpoint())) + .flat_map(|conflict| conflict.tx.inputs().iter().map(|input| input.outpoint())) .collect::>(); - tx.get_inputs() + tx.inputs() .iter() .find(|input| { // input spends an unconfirmed output input.spends_unconfirmed(self) && // this unconfirmed output is not spent by one of the conflicts - !outpoints_spent_by_conflicts.contains(&input.get_outpoint()) + !outpoints_spent_by_conflicts.contains(&input.outpoint()) }) .map_or(Ok(()), |_| { Err(TxValidationError::SpendsNewUnconfirmedOutput) @@ -530,9 +530,7 @@ trait SpendsUnconfirmed { impl SpendsUnconfirmed for TxInput { fn spends_unconfirmed(&self, mempool: &MempoolImpl) -> bool { - mempool.contains_transaction( - self.get_outpoint().get_tx_id().get_tx_id().expect("Not coinbase"), - ) + mempool.contains_transaction(self.outpoint().tx_id().get_tx_id().expect("Not coinbase")) } } @@ -544,17 +542,17 @@ impl Mempool for MempoolImpl { } } - fn new_tip_set(&mut self) -> Result<(), MempoolError> { + fn new_tip_set(&mut self) -> Result<(), Error> { unimplemented!() } // - fn add_transaction(&mut self, tx: Transaction) -> Result<(), MempoolError> { + fn add_transaction(&mut self, tx: Transaction) -> Result<(), Error> { // TODO (1). First, we need to decide on criteria for the Mempool to be considered full. Maybe number // of transactions is not a good enough indicator. Consider checking mempool size as well // TODO (2) What to do when the mempool is full. Instead of rejecting Do incoming transaction we probably want to evict a low-score transaction if self.store.txs_by_fee.len() >= MEMPOOL_MAX_TXS { - return Err(MempoolError::MempoolFull); + return Err(Error::MempoolFull); } self.validate_transaction(&tx)?; let entry = self.create_entry(tx)?; @@ -977,7 +975,7 @@ mod tests { .expect("generate_tx failed"); assert!(matches!( mempool.add_transaction(tx), - Err(MempoolError::TxValidationError(TxValidationError::NoInputs)) + Err(Error::TxValidationError(TxValidationError::NoInputs)) )); Ok(()) } @@ -995,9 +993,7 @@ mod tests { .expect("generate_tx failed"); assert!(matches!( mempool.add_transaction(tx), - Err(MempoolError::TxValidationError( - TxValidationError::NoOutputs - )) + Err(Error::TxValidationError(TxValidationError::NoOutputs)) )); Ok(()) } @@ -1035,9 +1031,7 @@ mod tests { assert!(matches!( mempool.add_transaction(tx), - Err(MempoolError::TxValidationError( - TxValidationError::DuplicateInputs - )) + Err(Error::TxValidationError(TxValidationError::DuplicateInputs)) )); Ok(()) } @@ -1067,7 +1061,7 @@ mod tests { mempool.add_transaction(tx.clone())?; assert!(matches!( mempool.add_transaction(tx), - Err(MempoolError::TxValidationError( + Err(Error::TxValidationError( TxValidationError::TransactionAlreadyInMempool )) )); @@ -1109,7 +1103,7 @@ mod tests { assert!(matches!( mempool.add_transaction(tx), - Err(MempoolError::TxValidationError( + Err(Error::TxValidationError( TxValidationError::OutPointNotFound { .. } )) )); @@ -1126,14 +1120,14 @@ mod tests { .expect("generate_tx failed"); assert!(matches!( mempool.add_transaction(tx), - Err(MempoolError::TxValidationError( + Err(Error::TxValidationError( TxValidationError::ExceedsMaxBlockSize )) )); Ok(()) } - fn test_replace_tx(original_fee: Amount, replacement_fee: Amount) -> Result<(), MempoolError> { + fn test_replace_tx(original_fee: Amount, replacement_fee: Amount) -> Result<(), Error> { let mut mempool = setup(); let outpoint = mempool .available_outpoints() @@ -1141,13 +1135,16 @@ mod tests { .next() .expect("there should be an outpoint since setup creates the genesis transaction") .outpoint - .to_owned(); + .clone(); - let outpoint_source_id = OutPointSourceId::from( - outpoint.get_tx_id().get_tx_id().expect("Not Coinbase").to_owned(), - ); + let outpoint_source_id = + OutPointSourceId::from(*outpoint.tx_id().get_tx_id().expect("Not Coinbase")); - let input = TxInput::new(outpoint_source_id, 0, DUMMY_WITNESS_MSG.to_vec()); + let input = TxInput::new( + outpoint_source_id, + 0, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ); let flags = 1; let locktime = 0; let tx = tx_spend_input(&mempool, input.clone(), original_fee, flags, locktime) @@ -1170,19 +1167,19 @@ mod tests { assert!(test_replace_tx(Amount::from_atoms(100), replacement_fee).is_ok()); assert!(matches!( test_replace_tx(Amount::from_atoms(100), Amount::from_atoms(relay_fee + 99)), - Err(MempoolError::TxValidationError( + Err(Error::TxValidationError( TxValidationError::InsufficientFeesToRelay )) )); assert!(matches!( test_replace_tx(Amount::from_atoms(10), Amount::from_atoms(10)), - Err(MempoolError::TxValidationError( + Err(Error::TxValidationError( TxValidationError::ReplacementFeeLowerThanOriginal { .. } )) )); assert!(matches!( test_replace_tx(Amount::from_atoms(10), Amount::from_atoms(5)), - Err(MempoolError::TxValidationError( + Err(Error::TxValidationError( TxValidationError::ReplacementFeeLowerThanOriginal { .. } )) )); @@ -1424,8 +1421,8 @@ mod tests { .replaceable() .generate_tx() .expect("generate_tx failed"); - let input = tx.get_inputs().first().expect("one input").to_owned(); - let outputs = tx.get_outputs().clone(); + let input = tx.inputs().first().expect("one input").clone(); + let outputs = tx.outputs().clone(); let tx_id = tx.get_id(); mempool.add_transaction(tx)?; @@ -1436,7 +1433,7 @@ mod tests { let input = TxInput::new( outpoint_source_id.clone(), index.try_into().unwrap(), - DUMMY_WITNESS_MSG.to_vec(), + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), ); let fee = Amount::from_atoms(0); let tx = tx_spend_input(mempool, input, fee, flags, locktime)?; @@ -1458,7 +1455,7 @@ mod tests { .expect("failed to downcast"); assert!(matches!( err, - MempoolError::TxValidationError(TxValidationError::TooManyPotentialReplacements) + Error::TxValidationError(TxValidationError::TooManyPotentialReplacements) )); Ok(()) } @@ -1481,8 +1478,16 @@ mod tests { let outpoint_source_id = OutPointSourceId::Transaction(tx.get_id()); mempool.add_transaction(tx)?; - let input1 = TxInput::new(outpoint_source_id.clone(), 0, DUMMY_WITNESS_MSG.to_vec()); - let input2 = TxInput::new(outpoint_source_id, 1, DUMMY_WITNESS_MSG.to_vec()); + let input1 = TxInput::new( + outpoint_source_id.clone(), + 0, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ); + let input2 = TxInput::new( + outpoint_source_id, + 1, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ); let locktime = 0; let flags = 0; @@ -1501,7 +1506,7 @@ mod tests { let res = mempool.add_transaction(incoming_tx); assert!(matches!( res, - Err(MempoolError::TxValidationError( + Err(Error::TxValidationError( TxValidationError::SpendsNewUnconfirmedOutput )) )); @@ -1519,7 +1524,11 @@ mod tests { mempool.add_transaction(tx)?; let outpoint_source_id = OutPointSourceId::Transaction(tx_id); - let input = TxInput::new(outpoint_source_id, 0, DUMMY_WITNESS_MSG.to_vec()); + let input = TxInput::new( + outpoint_source_id, + 0, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ); let locktime = 0; let rbf = 1; @@ -1540,7 +1549,7 @@ mod tests { TxInput::new( descendant_outpoint_source_id.clone(), 0, - DUMMY_WITNESS_MSG.to_vec(), + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), ), descendant1_fee, no_rbf, @@ -1551,7 +1560,11 @@ mod tests { let descendant2_fee = Amount::from_atoms(100); let descendant2 = tx_spend_input( &mempool, - TxInput::new(descendant_outpoint_source_id, 1, DUMMY_WITNESS_MSG.to_vec()), + TxInput::new( + descendant_outpoint_source_id, + 1, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ), descendant2_fee, no_rbf, locktime, @@ -1572,7 +1585,7 @@ mod tests { assert!(matches!( mempool.add_transaction(incoming_tx), - Err(MempoolError::TxValidationError( + Err(Error::TxValidationError( TxValidationError::TransactionFeeLowerThanConflictsWithDescendants )) )); From 80d370acb6c4485ea034c3dc012b8e46fc6b8306 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Thu, 24 Feb 2022 15:30:50 +0700 Subject: [PATCH 059/153] Bug Fix: move rbf checks out of conflict loop --- mempool/src/pool.rs | 56 +++++++++++++++++++++++++-------------------- 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 8e91392011..69af242259 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -384,13 +384,6 @@ impl MempoolImpl { return Err(TxValidationError::TransactionAlreadyInMempool); } - self.rbf_checks(tx)?; - self.verify_inputs_available(tx)?; - - Ok(()) - } - - fn rbf_checks(&self, tx: &Transaction) -> Result<(), TxValidationError> { let conflicts = tx .inputs() .iter() @@ -398,30 +391,42 @@ impl MempoolImpl { .map(|id_conflict| self.store.get_entry(&id_conflict).expect("entry for id")) .collect::>(); - for entry in &conflicts { + if !conflicts.is_empty() { + self.rbf_checks(tx, &conflicts)?; + } + + self.verify_inputs_available(tx)?; + + Ok(()) + } + + fn rbf_checks( + &self, + tx: &Transaction, + conflicts: &[&TxMempoolEntry], + ) -> Result<(), TxValidationError> { + for entry in conflicts { // Enforce BIP125 Rule #1. entry .is_replaceable(&self.store) .then(|| ()) .ok_or(TxValidationError::ConflictWithIrreplaceableTransaction)?; - - // It's possible that the replacement pays more fees than its direct conflicts but not more - // than all conflicts (i.e. the direct conflicts have high-fee descendants). However, if the - // replacement doesn't pay more fees than its direct conflicts, then we can be sure it's not - // more economically rational to mine. Before we go digging through the mempool for all - // transactions that would need to be removed (direct conflicts and all descendants), check - // that the replacement transaction pays more than its direct conflicts. - self.pays_more_than_direct_conflicts(tx, &conflicts)?; - // Enforce BIP125 Rule #2. - self.spends_no_new_unconfirmed_outputs(tx, &conflicts)?; - // Enforce BIP125 Rule #5. - let conflicts_with_descendants = - self.potential_replacements_within_limit(&conflicts)?; - // Enforce BIP125 Rule #3. - let total_conflict_fees = - self.pays_more_than_conflicts_with_descendants(tx, &conflicts_with_descendants)?; - self.pays_for_bandwidth(tx, total_conflict_fees)?; } + // It's possible that the replacement pays more fees than its direct conflicts but not more + // than all conflicts (i.e. the direct conflicts have high-fee descendants). However, if the + // replacement doesn't pay more fees than its direct conflicts, then we can be sure it's not + // more economically rational to mine. Before we go digging through the mempool for all + // transactions that would need to be removed (direct conflicts and all descendants), check + // that the replacement transaction pays more than its direct conflicts. + self.pays_more_than_direct_conflicts(tx, conflicts)?; + // Enforce BIP125 Rule #2. + self.spends_no_new_unconfirmed_outputs(tx, conflicts)?; + // Enforce BIP125 Rule #5. + let conflicts_with_descendants = self.potential_replacements_within_limit(conflicts)?; + // Enforce BIP125 Rule #3. + let total_conflict_fees = + self.pays_more_than_conflicts_with_descendants(tx, &conflicts_with_descendants)?; + self.pays_for_bandwidth(tx, total_conflict_fees)?; Ok(()) } @@ -454,6 +459,7 @@ impl MempoolImpl { .map(|conflict| conflict.fee) .sum::>() .ok_or(TxValidationError::ConflictsFeeOverflow)?; + let replacement_fee = self.try_get_fee(tx)?; (replacement_fee > total_conflict_fees) .then(|| ()) From 93dbbe1f3c620f45ea5e0e024a8b49160a70aa9c Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Thu, 24 Feb 2022 15:57:06 +0700 Subject: [PATCH 060/153] Refactor rbf_checks --- mempool/src/pool.rs | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 69af242259..ab2bc07659 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -384,6 +384,14 @@ impl MempoolImpl { return Err(TxValidationError::TransactionAlreadyInMempool); } + self.rbf_checks(tx)?; + + self.verify_inputs_available(tx)?; + + Ok(()) + } + + fn rbf_checks(&self, tx: &Transaction) -> Result<(), TxValidationError> { let conflicts = tx .inputs() .iter() @@ -391,16 +399,14 @@ impl MempoolImpl { .map(|id_conflict| self.store.get_entry(&id_conflict).expect("entry for id")) .collect::>(); - if !conflicts.is_empty() { - self.rbf_checks(tx, &conflicts)?; + if conflicts.is_empty() { + Ok(()) + } else { + self.do_rbf_checks(tx, &conflicts) } - - self.verify_inputs_available(tx)?; - - Ok(()) } - fn rbf_checks( + fn do_rbf_checks( &self, tx: &Transaction, conflicts: &[&TxMempoolEntry], From b6f19b9fe74de76cee898857c6ec6cfb18764f6d Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Thu, 24 Feb 2022 15:58:11 +0700 Subject: [PATCH 061/153] Mempool: Add comment --- mempool/src/pool.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index ab2bc07659..93e419fe29 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -432,6 +432,7 @@ impl MempoolImpl { // Enforce BIP125 Rule #3. let total_conflict_fees = self.pays_more_than_conflicts_with_descendants(tx, &conflicts_with_descendants)?; + // Enforce BIP125 Rule #4. self.pays_for_bandwidth(tx, total_conflict_fees)?; Ok(()) } From 423be7491d5398bbd842575deaf6d266dc37464b Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Thu, 24 Feb 2022 16:50:49 +0700 Subject: [PATCH 062/153] Add function get_relay_fee --- mempool/src/pool.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 93e419fe29..49682e29fa 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -52,6 +52,11 @@ impl TryGetFee for MempoolImpl { } } +fn get_relay_fee(tx: &Transaction) -> Amount { + // TODO we should never reach the expect, but should this be an error anyway? + Amount::from_atoms(u128::try_from(tx.encoded_size() * RELAY_FEE_PER_BYTE).expect("Overflow")) +} + pub trait Mempool { fn create(chain_state: C) -> Self; fn add_transaction(&mut self, tx: Transaction) -> Result<(), Error>; @@ -445,9 +450,7 @@ impl MempoolImpl { let additional_fees = (self.try_get_fee(tx)? - total_conflict_fees) .ok_or(TxValidationError::AdditionalFeesUnderflow)?; // TODO should we return an error here instead of expect? - let relay_fee = Amount::from_atoms( - u128::try_from(tx.encoded_size() * RELAY_FEE_PER_BYTE).expect("Overflow"), - ); + let relay_fee = get_relay_fee(tx); (additional_fees >= relay_fee) .then(|| ()) .ok_or(TxValidationError::InsufficientFeesToRelay) From 2b5bcf6e8dc27c8c5fea5ed74b2e168028e2b763 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Thu, 24 Feb 2022 16:52:45 +0700 Subject: [PATCH 063/153] Validation: minimum_relay_fees --- mempool/src/pool.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 49682e29fa..2af181d63c 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -393,9 +393,17 @@ impl MempoolImpl { self.verify_inputs_available(tx)?; + self.pays_minimum_relay_fees(tx)?; + Ok(()) } + fn pays_minimum_relay_fees(&self, tx: &Transaction) -> Result<(), TxValidationError> { + (self.try_get_fee(tx)? >= get_relay_fee(tx)) + .then(|| ()) + .ok_or(TxValidationError::InsufficientFeesToRelay) + } + fn rbf_checks(&self, tx: &Transaction) -> Result<(), TxValidationError> { let conflicts = tx .inputs() From a6008e4791502e608ea253821f5d8fba372846e0 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Thu, 24 Feb 2022 16:56:14 +0700 Subject: [PATCH 064/153] TESTS: accomodate for minimum relay fee check --- mempool/src/pool.rs | 157 +++++++++++++++++++++++++++++++++----------- 1 file changed, 119 insertions(+), 38 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 2af181d63c..740c49dd1c 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -142,12 +142,20 @@ impl TxMempoolEntry { fn unconfirmed_descendants(&self, store: &MempoolStore) -> BTreeSet { let mut visited = BTreeSet::new(); self.unconfirmed_descendants_inner(&mut visited, store); + eprintln!( + "found {} unconfirmed descendants:\n {:?}", + visited.len(), + visited + ); visited } fn unconfirmed_descendants_inner(&self, visited: &mut BTreeSet, store: &MempoolStore) { + eprintln!("unconfirmed_descendants_inner"); for child in self.children.iter() { + eprintln!("iterating: child: {:?}", child); if visited.contains(child) { + eprintln!("child already visited"); continue; } else { visited.insert(*child); @@ -252,7 +260,9 @@ impl MempoolStore { self.mark_outpoints_as_spent(&entry); self.txs_by_fee.entry(entry.fee).or_default().insert(entry.tx_id()); + let tx_id = entry.tx_id(); self.txs_by_id.insert(entry.tx_id(), entry); + assert!(self.txs_by_id.get(&tx_id).is_some()); Ok(()) } @@ -303,7 +313,7 @@ pub enum TxValidationError { ConflictWithIrreplaceableTransaction, #[error("TransactionFeeOverflow")] TransactionFeeOverflow, - #[error("ReplacementFeeLowerThanOriginal")] + #[error("ReplacementFeeLowerThanOriginal: The replacement transaction has fee {replacement_fee:?}, the original transaction has fee {original_fee:?}")] ReplacementFeeLowerThanOriginal { replacement_tx: H256, replacement_fee: Amount, @@ -322,6 +332,8 @@ pub enum TxValidationError { AdditionalFeesUnderflow, #[error("InsufficientFeesToRelay")] InsufficientFeesToRelay, + #[error("InsufficientFeesToRelayRBF")] + InsufficientFeesToRelayRBF, } impl From for Error { @@ -459,9 +471,13 @@ impl MempoolImpl { .ok_or(TxValidationError::AdditionalFeesUnderflow)?; // TODO should we return an error here instead of expect? let relay_fee = get_relay_fee(tx); + eprintln!( + "relay_fee: {:?}, additional_fees {:?}, total_conflict_fees {:?}, replacement_fee: {:?}", + relay_fee, additional_fees, total_conflict_fees, self.try_get_fee(tx)? + ); (additional_fees >= relay_fee) .then(|| ()) - .ok_or(TxValidationError::InsufficientFeesToRelay) + .ok_or(TxValidationError::InsufficientFeesToRelayRBF) } fn pays_more_than_conflicts_with_descendants( @@ -628,6 +644,40 @@ mod tests { outpoint: OutPoint, value: Amount, } + fn dummy_input() -> TxInput { + let outpoint_source_id = OutPointSourceId::Transaction(Id::new(H256::zero())); + let output_index = 0; + let witness = DUMMY_WITNESS_MSG.to_vec(); + TxInput::new( + outpoint_source_id, + output_index, + InputWitness::NoSignature(Some(witness)), + ) + } + + fn dummy_output() -> TxOutput { + let value = Amount::from_atoms(0); + let purpose = OutputPurpose::Transfer(Destination::AnyoneCanSpend); + TxOutput::new(value, purpose) + } + + fn estimate_tx_size(num_inputs: usize, num_outputs: usize) -> usize { + let inputs = (0..num_inputs).into_iter().map(|_| dummy_input()).collect(); + let outputs = (0..num_outputs).into_iter().map(|_| dummy_output()).collect(); + let flags = 0; + let locktime = 0; + let size = Transaction::new(flags, inputs, outputs, locktime).unwrap().encoded_size(); + // Take twice the encoded size of the dummy tx.Real Txs are larger than these dummy ones, + // but taking 2 times the size seems to work + 2 * size + } + + #[test] + fn dummy_size() { + eprintln!("1, 1: {}", estimate_tx_size(1, 1)); + eprintln!("1, 2: {}", estimate_tx_size(1, 2)); + eprintln!("1, 400: {}", estimate_tx_size(1, 400)); + } fn valued_outpoint( tx_id: &Id, @@ -802,6 +852,14 @@ mod tests { self } + // TODO not sure if we need this + /* + fn with_fee(mut self, tx_fee: Amount) -> Self { + self.tx_fee = tx_fee; + self + } + */ + fn new(mempool: &MempoolImpl) -> Self { let unconfirmed_outputs = mempool.available_outpoints(); Self::create_tx_generator(&mempool.chain_state, &unconfirmed_outputs) @@ -828,6 +886,10 @@ mod tests { } fn generate_tx(&mut self) -> anyhow::Result { + self.tx_fee = Amount::from_atoms(get_relay_fee_from_tx_size(estimate_tx_size( + self.num_inputs, + self.num_outputs, + ))); let valued_inputs = self.generate_tx_inputs(); let outputs = self.generate_tx_outputs(&valued_inputs)?; let locktime = 0; @@ -872,12 +934,16 @@ mod tests { let sum_of_inputs = values.into_iter().sum::>().expect("Overflow in sum of input values"); - let total_to_spend = (sum_of_inputs - self.tx_fee).expect("underflow"); + let total_to_spend = (sum_of_inputs - self.tx_fee) + .expect("generate_tx_outputs: underflow computing total_to_spend"); let mut left_to_spend = total_to_spend; let mut outputs = Vec::new(); let max_output_value = Amount::from_atoms(1_000); + // We want every output to be spendable in a single-input, single-output transaction + // So it has to larger in value than the relay fee for such a transaction + let min_output_value = Amount::from_atoms(100); for _ in 0..self.num_outputs - 1 { let max_output_value = std::cmp::min( (left_to_spend / 2).expect("division failed"), @@ -886,7 +952,7 @@ mod tests { if max_output_value == Amount::from_atoms(0) { return Err(anyhow::Error::msg("No more funds to spend")); } - let value = Amount::random(Amount::from_atoms(1)..=max_output_value); + let value = Amount::random(min_output_value..=max_output_value); outputs.push(TxOutput::new( value, OutputPurpose::Transfer(Destination::AnyoneCanSpend), @@ -928,6 +994,10 @@ mod tests { } } + fn get_relay_fee_from_tx_size(tx_size: usize) -> u128 { + u128::try_from(tx_size * RELAY_FEE_PER_BYTE).expect("relay fee overflow") + } + #[test] fn add_single_tx() -> anyhow::Result<()> { let mut mempool = MempoolImpl::create(ChainStateMock::new()); @@ -948,7 +1018,8 @@ mod tests { 0, InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), ); - let tx = tx_spend_input(&mempool, input, None, flags, locktime)?; + let relay_fee = Amount::from_atoms(get_relay_fee_from_tx_size(TX_SPEND_INPUT_SIZE)); + let tx = tx_spend_input(&mempool, input, relay_fee, flags, locktime)?; let tx_clone = tx.clone(); let tx_id = tx.get_id(); @@ -1185,24 +1256,26 @@ mod tests { #[test] fn tx_replace() -> anyhow::Result<()> { - let relay_fee = - u128::try_from(TX_SPEND_INPUT_SIZE * RELAY_FEE_PER_BYTE).expect("relay fee overflow"); + let relay_fee = get_relay_fee_from_tx_size(TX_SPEND_INPUT_SIZE); let replacement_fee = Amount::from_atoms(relay_fee + 100); - assert!(test_replace_tx(Amount::from_atoms(100), replacement_fee).is_ok()); + test_replace_tx(Amount::from_atoms(100), replacement_fee)?; + let res = test_replace_tx(Amount::from_atoms(100), Amount::from_atoms(relay_fee + 99)); assert!(matches!( - test_replace_tx(Amount::from_atoms(100), Amount::from_atoms(relay_fee + 99)), + res, Err(Error::TxValidationError( - TxValidationError::InsufficientFeesToRelay + TxValidationError::InsufficientFeesToRelayRBF )) )); + let res = test_replace_tx(Amount::from_atoms(100), Amount::from_atoms(100)); assert!(matches!( - test_replace_tx(Amount::from_atoms(10), Amount::from_atoms(10)), + res, Err(Error::TxValidationError( TxValidationError::ReplacementFeeLowerThanOriginal { .. } )) )); + let res = test_replace_tx(Amount::from_atoms(100), Amount::from_atoms(90)); assert!(matches!( - test_replace_tx(Amount::from_atoms(10), Amount::from_atoms(5)), + res, Err(Error::TxValidationError( TxValidationError::ReplacementFeeLowerThanOriginal { .. } )) @@ -1232,15 +1305,14 @@ mod tests { let child_tx = tx_spend_input( &mempool, child_tx_input.clone(), - Amount::from_atoms(10), + Amount::from_atoms(100), flags, locktime, )?; mempool.add_transaction(child_tx)?; - let relay_fee = - u128::try_from(TX_SPEND_INPUT_SIZE * RELAY_FEE_PER_BYTE).expect("relay fee overflow"); - let replacement_fee = Amount::from_atoms(relay_fee + 15); + let relay_fee = get_relay_fee_from_tx_size(TX_SPEND_INPUT_SIZE); + let replacement_fee = Amount::from_atoms(relay_fee + 100); let replacement_tx = tx_spend_input(&mempool, child_tx_input, replacement_fee, flags, locktime)?; mempool.add_transaction(replacement_tx)?; @@ -1257,17 +1329,20 @@ mod tests { flags: u32, locktime: u32, ) -> anyhow::Result { + let fee = fee.into().map_or_else( + || Amount::from_atoms(get_relay_fee_from_tx_size(estimate_tx_size(1, 2))), + std::convert::identity, + ); tx_spend_several_inputs(mempool, &[input], fee, flags, locktime) } fn tx_spend_several_inputs( mempool: &MempoolImpl, inputs: &[TxInput], - fee: impl Into>, + fee: Amount, flags: u32, locktime: u32, ) -> anyhow::Result { - let fee = fee.into().map_or(Amount::from_atoms(0), std::convert::identity); let input_value = inputs .iter() .map(|input| mempool.get_input_value(input)) @@ -1276,10 +1351,12 @@ mod tests { .sum::>() .expect("tx_spend_input: overflow"); - let available_for_spending = (input_value - fee).expect("underflow"); + let available_for_spending = (input_value - fee) + .expect("tx_spend_several_inputs: underflow computing available_for_spending"); let spent = (available_for_spending / 2).expect("division error"); - let change = (available_for_spending - spent).expect("underflow"); + let change = (available_for_spending - spent) + .expect("tx_spend_several_inputs: underflow_computing change"); Transaction::new( flags, @@ -1296,6 +1373,8 @@ mod tests { #[test] fn one_ancestor_signal_is_enough() -> anyhow::Result<()> { let mut mempool = setup(); + // TODO add a function which calculates the tx size based on number of outputs and inputs, + // so that we can calculate the minimum relay fee let tx = TxGenerator::new(&mempool) .with_num_outputs(2) .generate_tx() @@ -1347,7 +1426,8 @@ mod tests { InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), ); - let original_fee = Amount::from_atoms(10); + // TODO compute minimum necessary relay fee instead of just overestimating it + let original_fee = Amount::from_atoms(200); let dummy_output = TxOutput::new( original_fee, OutputPurpose::Transfer(Destination::AnyoneCanSpend), @@ -1453,20 +1533,21 @@ mod tests { let flags = 0; let locktime = 0; let outpoint_source_id = OutPointSourceId::Transaction(tx_id); + let fee = get_relay_fee_from_tx_size(TX_SPEND_INPUT_SIZE); for (index, _) in outputs.iter().enumerate() { let input = TxInput::new( outpoint_source_id.clone(), index.try_into().unwrap(), InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), ); - let fee = Amount::from_atoms(0); - let tx = tx_spend_input(mempool, input, fee, flags, locktime)?; + let tx = tx_spend_input(mempool, input, Amount::from_atoms(fee), flags, locktime)?; mempool.add_transaction(tx)?; } - let replacement_tx = - tx_spend_input(mempool, input, Amount::from_atoms(100), flags, locktime)?; - mempool.add_transaction(replacement_tx).map_err(anyhow::Error::from) + let replacement_fee = Amount::from_atoms(1000) * fee; + let replacement_tx = tx_spend_input(mempool, input, replacement_fee, flags, locktime)?; + mempool.add_transaction(replacement_tx).map_err(anyhow::Error::from)?; + Ok(()) } #[test] @@ -1515,10 +1596,11 @@ mod tests { let locktime = 0; let flags = 0; - let original_fee = Amount::from_atoms(0); + let original_fee = Amount::from_atoms(100); let replaced_tx = tx_spend_input(&mempool, input1.clone(), original_fee, flags, locktime)?; mempool.add_transaction(replaced_tx)?; - let replacement_fee = Amount::from_atoms(10); + let relay_fee = get_relay_fee_from_tx_size(TX_SPEND_INPUT_SIZE); + let replacement_fee = Amount::from_atoms(100 + relay_fee); let incoming_tx = tx_spend_several_inputs( &mempool, &[input1, input2], @@ -1540,10 +1622,7 @@ mod tests { #[test] fn pays_more_than_conflicts_with_descendants() -> anyhow::Result<()> { let mut mempool = setup(); - let tx = TxGenerator::new(&mempool) - .replaceable() - .generate_tx() - .expect("generate_replaceable_tx"); + let tx = TxGenerator::new(&mempool).generate_tx().expect("generate_replaceable_tx"); let tx_id = tx.get_id(); mempool.add_transaction(tx)?; @@ -1561,6 +1640,7 @@ mod tests { // Create transaction that we will attempt to replace let original_fee = Amount::from_atoms(100); let replaced_tx = tx_spend_input(&mempool, input.clone(), original_fee, rbf, locktime)?; + let replaced_tx_fee = mempool.try_get_fee(&replaced_tx)?; let replaced_id = replaced_tx.get_id(); mempool.add_transaction(replaced_tx)?; @@ -1598,7 +1678,10 @@ mod tests { //Create a new incoming transaction that conflicts with `replaced_tx` because it spends //`input`. It will be rejected because its fee exactly equals (so is not greater than) the //sum of the fees of the conflict together with its descendants - let insufficient_rbf_fee = Amount::from_atoms(300); + let insufficient_rbf_fee = [replaced_tx_fee, descendant1_fee, descendant2_fee] + .into_iter() + .sum::>() + .unwrap(); let incoming_tx = tx_spend_input( &mempool, input.clone(), @@ -1614,12 +1697,10 @@ mod tests { )) )); - let relay_fee = - u128::try_from(TX_SPEND_INPUT_SIZE * RELAY_FEE_PER_BYTE).expect("relay fee overflow"); - let sufficient_rbf_fee = Amount::from_atoms(300 + relay_fee); + let relay_fee = get_relay_fee_from_tx_size(TX_SPEND_INPUT_SIZE); + let sufficient_rbf_fee = insufficient_rbf_fee + Amount::from_atoms(relay_fee); let incoming_tx = tx_spend_input(&mempool, input, sufficient_rbf_fee, no_rbf, locktime)?; - assert!(mempool.add_transaction(incoming_tx).is_ok()); - + mempool.add_transaction(incoming_tx)?; Ok(()) } } From 7280e16a7fb41fba52e8f25bafc489219940893a Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Fri, 25 Feb 2022 16:37:59 +0700 Subject: [PATCH 065/153] TESTS: Fix txs_sorted (compute fees properly) --- mempool/src/pool.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 740c49dd1c..1fd6bfb2c9 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -1039,7 +1039,7 @@ mod tests { let chain_state = ChainStateMock::new(); let mut mempool = MempoolImpl::create(chain_state); let mut tx_generator = TxGenerator::new(&mempool); - let target_txs = 100; + let target_txs = 10; for _ in 0..target_txs { match tx_generator.generate_tx() { @@ -1053,8 +1053,8 @@ mod tests { let fees = mempool .get_all() .iter() - .map(|tx| tx.outputs().first().expect("TX should have exactly one output").value()) - .collect::>(); + .map(|tx| mempool.try_get_fee(tx)) + .collect::, _>>()?; let mut fees_sorted = fees.clone(); fees_sorted.sort_by(|a, b| b.cmp(a)); assert_eq!(fees, fees_sorted); From 6d356da720f42f5f2457841c5f1bb394973f5870 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Fri, 25 Feb 2022 15:51:03 +0700 Subject: [PATCH 066/153] TxMempoolEntry: new types for Ancestors and Descendants --- mempool/src/pool.rs | 51 ++++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 1fd6bfb2c9..bd622a8a5f 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -75,6 +75,12 @@ trait TryGetFee { fn try_get_fee(&self, tx: &Transaction) -> Result; } +#[derive(Debug)] +struct Ancestors(BTreeSet); + +#[derive(Debug)] +struct Descendants(BTreeSet); + #[derive(Debug, PartialEq, Eq, Clone)] struct TxMempoolEntry { tx: Transaction, @@ -115,22 +121,23 @@ impl TxMempoolEntry { self.tx.is_replaceable() || self .unconfirmed_ancestors(store) + .0 .iter() .any(|ancestor| store.get_entry(ancestor).expect("entry").tx.is_replaceable()) } - fn unconfirmed_ancestors(&self, store: &MempoolStore) -> BTreeSet { - let mut visited = BTreeSet::new(); + fn unconfirmed_ancestors(&self, store: &MempoolStore) -> Ancestors { + let mut visited = Ancestors(BTreeSet::new()); self.unconfirmed_ancestors_inner(&mut visited, store); visited } - fn unconfirmed_ancestors_inner(&self, visited: &mut BTreeSet, store: &MempoolStore) { + fn unconfirmed_ancestors_inner(&self, visited: &mut Ancestors, store: &MempoolStore) { for parent in self.parents.iter() { - if visited.contains(parent) { + if visited.0.contains(parent) { continue; } else { - visited.insert(*parent); + visited.0.insert(*parent); store .get_entry(parent) .expect("entry") @@ -139,26 +146,18 @@ impl TxMempoolEntry { } } - fn unconfirmed_descendants(&self, store: &MempoolStore) -> BTreeSet { - let mut visited = BTreeSet::new(); + fn unconfirmed_descendants(&self, store: &MempoolStore) -> Descendants { + let mut visited = Descendants(BTreeSet::new()); self.unconfirmed_descendants_inner(&mut visited, store); - eprintln!( - "found {} unconfirmed descendants:\n {:?}", - visited.len(), - visited - ); visited } - fn unconfirmed_descendants_inner(&self, visited: &mut BTreeSet, store: &MempoolStore) { - eprintln!("unconfirmed_descendants_inner"); + fn unconfirmed_descendants_inner(&self, visited: &mut Descendants, store: &MempoolStore) { for child in self.children.iter() { - eprintln!("iterating: child: {:?}", child); - if visited.contains(child) { - eprintln!("child already visited"); + if visited.0.contains(child) { continue; } else { - visited.insert(*child); + visited.0.insert(*child); store .get_entry(child) .expect("entry") @@ -241,7 +240,7 @@ impl MempoolStore { } fn update_ancestor_count(&mut self, entry: &TxMempoolEntry) { - for ancestor in entry.unconfirmed_ancestors(self) { + for ancestor in entry.unconfirmed_ancestors(self).0 { let ancestor = self.txs_by_id.get_mut(&ancestor).expect("ancestor"); ancestor.count_with_descendants += 1; } @@ -556,7 +555,7 @@ impl MempoolImpl { } let replacements_with_descendants = conflicts .iter() - .flat_map(|conflict| conflict.unconfirmed_descendants(&self.store)) + .flat_map(|conflict| conflict.unconfirmed_descendants(&self.store).0) .chain(conflicts.iter().map(|conflict| conflict.tx_id())) .collect(); @@ -1499,12 +1498,12 @@ mod tests { let entry4 = mempool.store.get_entry(ids.get(3).expect("index")).expect("entry"); let entry5 = mempool.store.get_entry(ids.get(4).expect("index")).expect("entry"); let entry6 = mempool.store.get_entry(ids.get(5).expect("index")).expect("entry"); - assert_eq!(entry1.unconfirmed_ancestors(&mempool.store).len(), 0); - assert_eq!(entry2.unconfirmed_ancestors(&mempool.store).len(), 0); - assert_eq!(entry3.unconfirmed_ancestors(&mempool.store).len(), 2); - assert_eq!(entry4.unconfirmed_ancestors(&mempool.store).len(), 3); - assert_eq!(entry5.unconfirmed_ancestors(&mempool.store).len(), 3); - assert_eq!(entry6.unconfirmed_ancestors(&mempool.store).len(), 5); + assert_eq!(entry1.unconfirmed_ancestors(&mempool.store).0.len(), 0); + assert_eq!(entry2.unconfirmed_ancestors(&mempool.store).0.len(), 0); + assert_eq!(entry3.unconfirmed_ancestors(&mempool.store).0.len(), 2); + assert_eq!(entry4.unconfirmed_ancestors(&mempool.store).0.len(), 3); + assert_eq!(entry5.unconfirmed_ancestors(&mempool.store).0.len(), 3); + assert_eq!(entry6.unconfirmed_ancestors(&mempool.store).0.len(), 5); assert_eq!(entry1.count_with_descendants(), 5); assert_eq!(entry2.count_with_descendants(), 5); From 9984a495b9102672ad289cd67f4fa6c338265db4 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Tue, 1 Mar 2022 14:18:14 +0700 Subject: [PATCH 067/153] More precise errors for tx fee computation --- mempool/src/pool.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index bd622a8a5f..2ba7934820 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -41,14 +41,14 @@ impl TryGetFee for MempoolImpl { .iter() .cloned() .sum::>() - .ok_or(TxValidationError::TransactionFeeOverflow)?; + .ok_or(TxValidationError::InputValuesOverflow)?; let sum_outputs = tx .outputs() .iter() .map(|output| output.value()) .sum::>() - .ok_or(TxValidationError::TransactionFeeOverflow)?; - (sum_inputs - sum_outputs).ok_or(TxValidationError::TransactionFeeOverflow) + .ok_or(TxValidationError::OutputValuesOverflow)?; + (sum_inputs - sum_outputs).ok_or(TxValidationError::InputsBelowOutputs) } } @@ -310,8 +310,12 @@ pub enum TxValidationError { TransactionAlreadyInMempool, #[error("ConflictWithIrreplaceableTransaction")] ConflictWithIrreplaceableTransaction, - #[error("TransactionFeeOverflow")] - TransactionFeeOverflow, + #[error("InputValuesOverflow")] + InputValuesOverflow, + #[error("OutputValuesOverflow")] + OutputValuesOverflow, + #[error("InputsBelowOutputs")] + InputsBelowOutputs, #[error("ReplacementFeeLowerThanOriginal: The replacement transaction has fee {replacement_fee:?}, the original transaction has fee {original_fee:?}")] ReplacementFeeLowerThanOriginal { replacement_tx: H256, From 94f3795c7c17b3a2ba8b1a02c638ef12b4d3aa79 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Mon, 7 Mar 2022 17:56:32 +0700 Subject: [PATCH 068/153] Add pow function to amount --- common/src/primitives/amount.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/common/src/primitives/amount.rs b/common/src/primitives/amount.rs index 4b9bc4e370..05a30f2905 100644 --- a/common/src/primitives/amount.rs +++ b/common/src/primitives/amount.rs @@ -134,6 +134,13 @@ impl Amount { pub fn random(range: std::ops::RangeInclusive) -> Amount { Amount::from_atoms(rand::thread_rng().gen_range(range.start().val..=range.end().val)) } + + pub fn pow(self, exponent: usize) -> Option { + (0..exponent).into_iter().try_fold(Amount::from(1), |mut partial_result, _| { + partial_result = (partial_result * self)?; + Some(partial_result) + }) + } } impl From for u128 { @@ -802,4 +809,10 @@ mod tests { assert_eq!(Amount { val: 12345678901200 }.into_fixedpoint_str(1), "1234567890120"); assert_eq!(Amount { val: 123456789012300 }.into_fixedpoint_str(1), "12345678901230"); } + + #[test] + fn pow() { + let x = Amount { val: 2 }; + assert_eq!(x.pow(4), Some(Amount { val: 16 })); + } } From 9fecf339dd25ea583108eb9d79e3f9220c269c8a Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Mon, 7 Mar 2022 15:12:19 +0700 Subject: [PATCH 069/153] Add finalize_tx and move entry creation into it --- mempool/src/pool.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 2ba7934820..08a1a40111 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -565,6 +565,15 @@ impl MempoolImpl { Ok(replacements_with_descendants) } + + fn finalize_tx(&mut self, tx: Transaction) -> Result<(), Error> { + let entry = self.create_entry(tx)?; + self.store.add_tx(entry)?; + // TODO evict conflicts + // add the tx + // limit mempool size + Ok(()) + } } trait SpendsUnconfirmed { @@ -598,8 +607,7 @@ impl Mempool for MempoolImpl { return Err(Error::MempoolFull); } self.validate_transaction(&tx)?; - let entry = self.create_entry(tx)?; - self.store.add_tx(entry)?; + self.finalize_tx(tx)?; Ok(()) } From dee73dc614cd443b82f84ea83d5b5efe433391bf Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Mon, 7 Mar 2022 15:27:50 +0700 Subject: [PATCH 070/153] Return conflicts from validation --- common/src/primitives/amount.rs | 10 ++++++---- mempool/Cargo.toml | 1 + mempool/src/pool.rs | 28 ++++++++++++++++------------ 3 files changed, 23 insertions(+), 16 deletions(-) diff --git a/common/src/primitives/amount.rs b/common/src/primitives/amount.rs index 05a30f2905..5f081cce7c 100644 --- a/common/src/primitives/amount.rs +++ b/common/src/primitives/amount.rs @@ -136,10 +136,12 @@ impl Amount { } pub fn pow(self, exponent: usize) -> Option { - (0..exponent).into_iter().try_fold(Amount::from(1), |mut partial_result, _| { - partial_result = (partial_result * self)?; - Some(partial_result) - }) + (0..exponent) + .into_iter() + .try_fold(Amount::from_atoms(1), |mut partial_result, _| { + partial_result = (partial_result * self.into())?; + Some(partial_result) + }) } } diff --git a/mempool/Cargo.toml b/mempool/Cargo.toml index e510a9c86c..8288effd07 100644 --- a/mempool/Cargo.toml +++ b/mempool/Cargo.toml @@ -10,6 +10,7 @@ license = "MIT" parity-scale-codec = {version = "2.3.1", features = ["derive", "chain-error"]} serialization = { path = '../serialization' } common = { path = '../common' } +utils = { path = '../utils' } anyhow = "1.0" thiserror = "1.0" diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 08a1a40111..8d75d383f6 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -15,6 +15,7 @@ use common::primitives::amount::Amount; use common::primitives::Id; use common::primitives::Idable; use common::primitives::H256; +use utils::newtype; // TODO this willbe defined elsewhere (some of limits.rs file) const MAX_BLOCK_SIZE_BYTES: usize = 1_000_000; @@ -75,11 +76,9 @@ trait TryGetFee { fn try_get_fee(&self, tx: &Transaction) -> Result; } -#[derive(Debug)] -struct Ancestors(BTreeSet); - -#[derive(Debug)] -struct Descendants(BTreeSet); +newtype!(struct Ancestors(BTreeSet)); +newtype!(struct Descendants(BTreeSet)); +newtype!(struct Conflicts(BTreeSet)); #[derive(Debug, PartialEq, Eq, Clone)] struct TxMempoolEntry { @@ -378,7 +377,7 @@ impl MempoolImpl { Ok(TxMempoolEntry::new(tx, fee, parents)) } - fn validate_transaction(&self, tx: &Transaction) -> Result<(), TxValidationError> { + fn validate_transaction(&self, tx: &Transaction) -> Result { if tx.inputs().is_empty() { return Err(TxValidationError::NoInputs); } @@ -404,13 +403,13 @@ impl MempoolImpl { return Err(TxValidationError::TransactionAlreadyInMempool); } - self.rbf_checks(tx)?; + let conflicts = self.rbf_checks(tx)?; self.verify_inputs_available(tx)?; self.pays_minimum_relay_fees(tx)?; - Ok(()) + Ok(conflicts) } fn pays_minimum_relay_fees(&self, tx: &Transaction) -> Result<(), TxValidationError> { @@ -419,7 +418,7 @@ impl MempoolImpl { .ok_or(TxValidationError::InsufficientFeesToRelay) } - fn rbf_checks(&self, tx: &Transaction) -> Result<(), TxValidationError> { + fn rbf_checks(&self, tx: &Transaction) -> Result { let conflicts = tx .inputs() .iter() @@ -428,7 +427,7 @@ impl MempoolImpl { .collect::>(); if conflicts.is_empty() { - Ok(()) + Ok(Conflicts(BTreeSet::new())) } else { self.do_rbf_checks(tx, &conflicts) } @@ -438,7 +437,7 @@ impl MempoolImpl { &self, tx: &Transaction, conflicts: &[&TxMempoolEntry], - ) -> Result<(), TxValidationError> { + ) -> Result { for entry in conflicts { // Enforce BIP125 Rule #1. entry @@ -462,7 +461,7 @@ impl MempoolImpl { self.pays_more_than_conflicts_with_descendants(tx, &conflicts_with_descendants)?; // Enforce BIP125 Rule #4. self.pays_for_bandwidth(tx, total_conflict_fees)?; - Ok(()) + Ok(Conflicts::from(conflicts_with_descendants)) } fn pays_for_bandwidth( @@ -574,6 +573,11 @@ impl MempoolImpl { // limit mempool size Ok(()) } + + #[allow(unused)] + fn limit_mempool_size(&mut self) -> Result<(), Error> { + Ok(()) + } } trait SpendsUnconfirmed { From 9c8de4a44edcfc33efcc0e8dd9245ff8c5a85c7e Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Mon, 7 Mar 2022 15:32:09 +0700 Subject: [PATCH 071/153] Drop conflicts when adding validated tx to mempool --- mempool/src/pool.rs | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 8d75d383f6..71d74c1ec0 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -277,6 +277,13 @@ impl MempoolStore { } } + // TODO test that conflicts are dropped + fn drop_conflicts(&mut self, conflicts: Conflicts) { + for conflict in conflicts.0 { + self.drop_tx(&Id::new(&conflict)) + } + } + fn find_conflicting_tx(&self, outpoint: &OutPoint) -> Option { self.spender_txs.get(outpoint).cloned() } @@ -565,11 +572,10 @@ impl MempoolImpl { Ok(replacements_with_descendants) } - fn finalize_tx(&mut self, tx: Transaction) -> Result<(), Error> { + fn finalize_tx(&mut self, tx: Transaction, conflicts: Conflicts) -> Result<(), Error> { + self.store.drop_conflicts(conflicts); let entry = self.create_entry(tx)?; self.store.add_tx(entry)?; - // TODO evict conflicts - // add the tx // limit mempool size Ok(()) } @@ -610,8 +616,8 @@ impl Mempool for MempoolImpl { if self.store.txs_by_fee.len() >= MEMPOOL_MAX_TXS { return Err(Error::MempoolFull); } - self.validate_transaction(&tx)?; - self.finalize_tx(tx)?; + let conflicts = self.validate_transaction(&tx)?; + self.finalize_tx(tx, conflicts)?; Ok(()) } From c999b3c886e769b3e477d31050718c598f452827 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Tue, 8 Mar 2022 10:18:51 +0700 Subject: [PATCH 072/153] TESTS: verify conflicts are removed --- mempool/src/pool.rs | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 71d74c1ec0..23a18d93a2 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -277,7 +277,6 @@ impl MempoolStore { } } - // TODO test that conflicts are dropped fn drop_conflicts(&mut self, conflicts: Conflicts) { for conflict in conflicts.0 { self.drop_tx(&Id::new(&conflict)) @@ -1263,14 +1262,16 @@ mod tests { ); let flags = 1; let locktime = 0; - let tx = tx_spend_input(&mempool, input.clone(), original_fee, flags, locktime) + let original = tx_spend_input(&mempool, input.clone(), original_fee, flags, locktime) .expect("should be able to spend here"); - mempool.add_transaction(tx)?; + let original_id = original.get_id(); + mempool.add_transaction(original)?; let flags = 0; - let tx = tx_spend_input(&mempool, input, replacement_fee, flags, locktime) + let replacement = tx_spend_input(&mempool, input, replacement_fee, flags, locktime) .expect("should be able to spend here"); - mempool.add_transaction(tx)?; + mempool.add_transaction(replacement)?; + assert!(!mempool.contains_transaction(&original_id)); Ok(()) } @@ -1460,6 +1461,7 @@ mod tests { flags_irreplaceable, locktime, )?; + let replaced_tx_id = replaced_tx.get_id(); mempool.add_transaction(replaced_tx)?; @@ -1471,6 +1473,7 @@ mod tests { )?; mempool.add_transaction(replacing_tx)?; + assert!(!mempool.contains_transaction(&replaced_tx_id)); Ok(()) } @@ -1564,10 +1567,17 @@ mod tests { let tx = tx_spend_input(mempool, input, Amount::from_atoms(fee), flags, locktime)?; mempool.add_transaction(tx)?; } + let mempool_size_before_replacement = mempool.store.txs_by_id.len(); let replacement_fee = Amount::from_atoms(1000) * fee; let replacement_tx = tx_spend_input(mempool, input, replacement_fee, flags, locktime)?; mempool.add_transaction(replacement_tx).map_err(anyhow::Error::from)?; + let mempool_size_after_replacement = mempool.store.txs_by_id.len(); + + assert_eq!( + mempool_size_after_replacement, + mempool_size_before_replacement - num_potential_replacements + 1 + ); Ok(()) } @@ -1666,7 +1676,7 @@ mod tests { mempool.add_transaction(replaced_tx)?; // Create some children for this transaction - let descendant_outpoint_source_id = OutPointSourceId::Transaction(replaced_id); + let descendant_outpoint_source_id = OutPointSourceId::Transaction(replaced_id.clone()); let descendant1_fee = Amount::from_atoms(100); let descendant1 = tx_spend_input( @@ -1680,6 +1690,7 @@ mod tests { no_rbf, locktime, )?; + let descendant1_id = descendant1.get_id(); mempool.add_transaction(descendant1)?; let descendant2_fee = Amount::from_atoms(100); @@ -1694,6 +1705,7 @@ mod tests { no_rbf, locktime, )?; + let descendant2_id = descendant2.get_id(); mempool.add_transaction(descendant2)?; //Create a new incoming transaction that conflicts with `replaced_tx` because it spends @@ -1722,6 +1734,10 @@ mod tests { let sufficient_rbf_fee = insufficient_rbf_fee + Amount::from_atoms(relay_fee); let incoming_tx = tx_spend_input(&mempool, input, sufficient_rbf_fee, no_rbf, locktime)?; mempool.add_transaction(incoming_tx)?; + + assert!(!mempool.contains_transaction(&replaced_id)); + assert!(!mempool.contains_transaction(&descendant1_id)); + assert!(!mempool.contains_transaction(&descendant2_id)); Ok(()) } } From 70a9da98728880a09a85bd01c93ce9ed63ed3cff Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Tue, 15 Mar 2022 12:04:00 +0700 Subject: [PATCH 073/153] Fix bug in confirmed_outpoints --- mempool/src/pool.rs | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 23a18d93a2..e8ea4b0419 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -819,12 +819,22 @@ mod tests { } fn confirmed_outpoints(&self) -> BTreeSet { - self.txs - .values() - .flat_map(|tx| { - std::iter::repeat(tx.get_id()) - .zip(tx.outputs().iter().enumerate()) - .map(move |(tx_id, (i, output))| valued_outpoint(&tx_id, i as u32, output)) + self.outpoints + .iter() + .map(|outpoint| { + let tx_id = outpoint + .tx_id() + .get_tx_id() + .cloned() + .expect("Outpoints in these tests are created from TXs"); + let index = outpoint.output_index(); + let tx = self.txs.get(&tx_id.get()).expect("Inconsistent Chain State"); + let output = tx + .outputs() + .get(index as usize) + .expect("Inconsistent Chain State: output not found"); + + valued_outpoint(&tx_id, index, output) }) .collect() } From bd2145fe7d83f714e47cd63c911ad846b8fce774 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Tue, 15 Mar 2022 11:37:38 +0700 Subject: [PATCH 074/153] replace expect with error when when fee higher than inputs --- common/src/primitives/amount.rs | 2 +- mempool/src/pool.rs | 14 ++++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/common/src/primitives/amount.rs b/common/src/primitives/amount.rs index 5f081cce7c..627b1cf1ea 100644 --- a/common/src/primitives/amount.rs +++ b/common/src/primitives/amount.rs @@ -139,7 +139,7 @@ impl Amount { (0..exponent) .into_iter() .try_fold(Amount::from_atoms(1), |mut partial_result, _| { - partial_result = (partial_result * self.into())?; + partial_result = (partial_result * self.val)?; Some(partial_result) }) } diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index e8ea4b0419..062ccf8a42 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -964,8 +964,9 @@ mod tests { let sum_of_inputs = values.into_iter().sum::>().expect("Overflow in sum of input values"); - let total_to_spend = (sum_of_inputs - self.tx_fee) - .expect("generate_tx_outputs: underflow computing total_to_spend"); + let total_to_spend = (sum_of_inputs - self.tx_fee).ok_or_else(||anyhow::anyhow!( + "generate_tx_outputs: underflow computing total_to_spend - sum_of_inputs = {:?}, fee = {:?}", sum_of_inputs, self.tx_fee + ))?; let mut left_to_spend = total_to_spend; let mut outputs = Vec::new(); @@ -1383,8 +1384,13 @@ mod tests { .sum::>() .expect("tx_spend_input: overflow"); - let available_for_spending = (input_value - fee) - .expect("tx_spend_several_inputs: underflow computing available_for_spending"); + let available_for_spending = (input_value - fee).ok_or_else(|| { + anyhow::anyhow!( + "tx_spend_several_inputs: input_value ({:?}) lower than fee ({:?})", + input_value, + fee + ) + })?; let spent = (available_for_spending / 2).expect("division error"); let change = (available_for_spending - spent) From 49404a48f2e3bfdd6aa8e8d7bed1272d9fa8adb9 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Wed, 16 Mar 2022 16:34:31 +0700 Subject: [PATCH 075/153] Replace expect calls with errors --- mempool/src/pool.rs | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 062ccf8a42..165f61d7e0 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -279,7 +279,7 @@ impl MempoolStore { fn drop_conflicts(&mut self, conflicts: Conflicts) { for conflict in conflicts.0 { - self.drop_tx(&Id::new(&conflict)) + self.drop_tx(&Id::new(conflict)) } } @@ -655,6 +655,7 @@ mod tests { use common::chain::transaction::{Destination, TxInput, TxOutput}; use common::chain::OutPointSourceId; use common::chain::OutputPurpose; + use core::panic; use rand::Rng; const DUMMY_WITNESS_MSG: &[u8] = b"dummy_witness_msg"; @@ -1382,19 +1383,21 @@ mod tests { .collect::, _>>()? .into_iter() .sum::>() - .expect("tx_spend_input: overflow"); + .ok_or_else(|| anyhow::anyhow!("tx_spend_input: overflow"))?; let available_for_spending = (input_value - fee).ok_or_else(|| { - anyhow::anyhow!( + let msg = format!( "tx_spend_several_inputs: input_value ({:?}) lower than fee ({:?})", - input_value, - fee - ) + input_value, fee + ); + anyhow::Error::msg(msg) })?; let spent = (available_for_spending / 2).expect("division error"); - let change = (available_for_spending - spent) - .expect("tx_spend_several_inputs: underflow_computing change"); + let change = (available_for_spending - spent).ok_or_else(|| { + let msg = String::from("Error computing change"); + anyhow::Error::msg(msg) + })?; Transaction::new( flags, From 86c65734862f42f9cb2f6759166bdf0f62939f65 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Tue, 15 Mar 2022 16:12:55 +0700 Subject: [PATCH 076/153] Increase min, max output values in unit tests to make more deterministic --- mempool/src/pool.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 165f61d7e0..8635cbf52b 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -972,10 +972,10 @@ mod tests { let mut left_to_spend = total_to_spend; let mut outputs = Vec::new(); - let max_output_value = Amount::from_atoms(1_000); + let max_output_value = Amount::from_atoms(10_000); // We want every output to be spendable in a single-input, single-output transaction // So it has to larger in value than the relay fee for such a transaction - let min_output_value = Amount::from_atoms(100); + let min_output_value = Amount::from_atoms(1000); for _ in 0..self.num_outputs - 1 { let max_output_value = std::cmp::min( (left_to_spend / 2).expect("division failed"), From 17eb18596cb4373eb7bb06edc07eba042d44d157 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Tue, 15 Mar 2022 12:04:57 +0700 Subject: [PATCH 077/153] Rename ChainStateMock's members --- mempool/src/pool.rs | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 8635cbf52b..7db97b5b5e 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -769,8 +769,8 @@ mod tests { #[derive(Debug, Clone)] pub(crate) struct ChainStateMock { - txs: HashMap, - outpoints: BTreeSet, + confirmed_txs: HashMap, + available_outpoints: BTreeSet, } impl ChainStateMock { @@ -784,13 +784,13 @@ mod tests { .map(|(index, _)| OutPoint::new(outpoint_source_id.clone(), index as u32)) .collect(); Self { - txs: std::iter::once((genesis_tx.get_id().get(), genesis_tx)).collect(), - outpoints, + confirmed_txs: std::iter::once((genesis_tx.get_id().get(), genesis_tx)).collect(), + available_outpoints: outpoints, } } fn unspent_outpoints(&self) -> BTreeSet { - self.outpoints + self.available_outpoints .iter() .map(|outpoint| { let value = @@ -804,11 +804,11 @@ mod tests { } fn confirmed_txs(&self) -> &HashMap { - &self.txs + &self.confirmed_txs } fn get_outpoint_value(&self, outpoint: &OutPoint) -> Result { - self.txs + self.confirmed_txs .get(&outpoint.tx_id().get_tx_id().expect("Not Coinbase").get()) .ok_or_else(|| anyhow::anyhow!("tx for outpoint sought in chain state, not found")) .and_then(|tx| { @@ -820,7 +820,7 @@ mod tests { } fn confirmed_outpoints(&self) -> BTreeSet { - self.outpoints + self.available_outpoints .iter() .map(|outpoint| { let tx_id = outpoint @@ -829,7 +829,8 @@ mod tests { .cloned() .expect("Outpoints in these tests are created from TXs"); let index = outpoint.output_index(); - let tx = self.txs.get(&tx_id.get()).expect("Inconsistent Chain State"); + let tx = + self.confirmed_txs.get(&tx_id.get()).expect("Inconsistent Chain State"); let output = tx .outputs() .get(index as usize) @@ -843,11 +844,11 @@ mod tests { impl ChainState for ChainStateMock { fn contains_outpoint(&self, outpoint: &OutPoint) -> bool { - self.outpoints.iter().any(|value| *value == *outpoint) + self.available_outpoints.iter().any(|value| *value == *outpoint) } fn get_outpoint_value(&self, outpoint: &OutPoint) -> Result { - self.txs + self.confirmed_txs .get(&outpoint.tx_id().get_tx_id().expect("Not coinbase").get()) .ok_or_else(|| anyhow::anyhow!("tx for outpoint sought in chain state, not found")) .and_then(|tx| { From 190fcdbf9e36b80d8e933178ad378f31969b5022 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Mon, 7 Mar 2022 12:00:02 +0700 Subject: [PATCH 078/153] Implement Rolling Fee Rate and eviction of expired entries --- Cargo.lock | 3 + common/src/primitives/amount.rs | 18 +- mempool/Cargo.toml | 2 + mempool/src/error.rs | 77 +++ mempool/src/feerate.rs | 56 ++ mempool/src/lib.rs | 4 +- mempool/src/pool.rs | 960 +++++++++++++++++++++++++------- 7 files changed, 904 insertions(+), 216 deletions(-) create mode 100644 mempool/src/error.rs create mode 100644 mempool/src/feerate.rs diff --git a/Cargo.lock b/Cargo.lock index f2e8963616..eedda1f76f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2192,10 +2192,13 @@ version = "0.1.0" dependencies = [ "anyhow", "common", + "lazy_static", + "mockall", "parity-scale-codec 2.3.1", "rand 0.8.5", "serialization", "thiserror", + "utils", ] [[package]] diff --git a/common/src/primitives/amount.rs b/common/src/primitives/amount.rs index 627b1cf1ea..cdc1e7d129 100644 --- a/common/src/primitives/amount.rs +++ b/common/src/primitives/amount.rs @@ -135,13 +135,11 @@ impl Amount { Amount::from_atoms(rand::thread_rng().gen_range(range.start().val..=range.end().val)) } - pub fn pow(self, exponent: usize) -> Option { - (0..exponent) - .into_iter() - .try_fold(Amount::from_atoms(1), |mut partial_result, _| { - partial_result = (partial_result * self.val)?; - Some(partial_result) - }) + // TODO this looks risky, consult Ben/Sam + pub fn div_by_float(self, divisor: f64) -> Self { + Self { + val: (self.val as f64 / divisor) as u128, + } } } @@ -811,10 +809,4 @@ mod tests { assert_eq!(Amount { val: 12345678901200 }.into_fixedpoint_str(1), "1234567890120"); assert_eq!(Amount { val: 123456789012300 }.into_fixedpoint_str(1), "12345678901230"); } - - #[test] - fn pow() { - let x = Amount { val: 2 }; - assert_eq!(x.pow(4), Some(Amount { val: 16 })); - } } diff --git a/mempool/Cargo.toml b/mempool/Cargo.toml index 8288effd07..41ef0d3225 100644 --- a/mempool/Cargo.toml +++ b/mempool/Cargo.toml @@ -13,6 +13,8 @@ common = { path = '../common' } utils = { path = '../utils' } anyhow = "1.0" thiserror = "1.0" +lazy_static = "1.4" +mockall = "0.11.0" [dev-dependencies] rand = "0.8.4" diff --git a/mempool/src/error.rs b/mempool/src/error.rs new file mode 100644 index 0000000000..59cca905a7 --- /dev/null +++ b/mempool/src/error.rs @@ -0,0 +1,77 @@ +use thiserror::Error; + +use common::chain::transaction::Transaction; +use common::chain::OutPoint; +use common::primitives::amount::Amount; +use common::primitives::Id; +use common::primitives::H256; + +#[derive(Debug, Error)] +pub enum Error { + #[error("Mempool is full")] + MempoolFull, + #[error(transparent)] + TxValidationError(TxValidationError), +} + +#[derive(Debug, Error)] +pub enum TxValidationError { + #[error("No Inputs")] + NoInputs, + #[error("No Ouputs")] + NoOutputs, + #[error("DuplicateInputs")] + DuplicateInputs, + #[error("LooseCoinbase")] + LooseCoinbase, + #[error("OutPointNotFound {outpoint:?}")] + OutPointNotFound { + outpoint: OutPoint, + tx_id: Id, + }, + #[error("ExceedsMaxBlockSize")] + ExceedsMaxBlockSize, + #[error("TransactionAlreadyInMempool")] + TransactionAlreadyInMempool, + #[error("ConflictWithIrreplaceableTransaction")] + ConflictWithIrreplaceableTransaction, + #[error("InputValuesOverflow")] + InputValuesOverflow, + #[error("OutputValuesOverflow")] + OutputValuesOverflow, + #[error("InputsBelowOutputs")] + InputsBelowOutputs, + #[error("ReplacementFeeLowerThanOriginal: The replacement transaction has fee {replacement_fee:?}, the original transaction has fee {original_fee:?}")] + ReplacementFeeLowerThanOriginal { + replacement_tx: H256, + replacement_fee: Amount, + original_tx: H256, + original_fee: Amount, + }, + #[error("TooManyPotentialReplacements")] + TooManyPotentialReplacements, + #[error("SpendsNewUnconfirmedInput")] + SpendsNewUnconfirmedOutput, + #[error("ConflictsFeeOverflow")] + ConflictsFeeOverflow, + #[error("TransactionFeeLowerThanConflictsWithDescendants")] + TransactionFeeLowerThanConflictsWithDescendants, + #[error("AdditionalFeesUnderflow")] + AdditionalFeesUnderflow, + #[error("InsufficientFeesToRelay")] + InsufficientFeesToRelay { tx_fee: Amount, relay_fee: Amount }, + #[error("InsufficientFeesToRelayRBF")] + InsufficientFeesToRelayRBF, + #[error("RollingFeeThresholdNotMet")] + RollingFeeThresholdNotMet { minimum_fee: Amount, tx_fee: Amount }, + #[error("FeeRate error")] + FeeRateError, + #[error("Internal Error")] + InternalError, +} + +impl From for Error { + fn from(e: TxValidationError) -> Self { + Error::TxValidationError(e) + } +} diff --git a/mempool/src/feerate.rs b/mempool/src/feerate.rs new file mode 100644 index 0000000000..6767565a42 --- /dev/null +++ b/mempool/src/feerate.rs @@ -0,0 +1,56 @@ +use common::primitives::amount::Amount; + +use crate::error::TxValidationError; + +lazy_static::lazy_static! { + pub(crate) static ref INCREMENTAL_RELAY_FEE_RATE: FeeRate = FeeRate::new(Amount::from_atoms(1000)); +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) struct FeeRate { + tokens_per_byte: Amount, +} + +impl FeeRate { + pub(crate) fn new(tokens_per_byte: Amount) -> Self { + Self { tokens_per_byte } + } + + pub(crate) fn of_tx(fee: Amount, tx_size: usize) -> Result { + let tokens_per_byte = Self::div_up(fee, tx_size)?; + Ok(Self { tokens_per_byte }) + } + + pub(crate) fn compute_fee(&self, size: usize) -> Result { + (self.tokens_per_byte * size as u128).ok_or(TxValidationError::FeeRateError) + } + + pub(crate) fn tokens_per_byte(&self) -> Amount { + self.tokens_per_byte + } + + fn div_up(fee: Amount, tx_size: usize) -> Result { + let tx_size = tx_size as u128; + (((fee + Amount::from_atoms(tx_size)).ok_or(TxValidationError::FeeRateError)? + - Amount::from_atoms(1)) + .ok_or(TxValidationError::FeeRateError)? + / tx_size) + .ok_or(TxValidationError::FeeRateError) + } +} + +impl std::ops::Add for FeeRate { + type Output = Option; + fn add(self, other: Self) -> Self::Output { + let tokens_per_byte = self.tokens_per_byte + other.tokens_per_byte; + tokens_per_byte.map(|tokens_per_byte| FeeRate { tokens_per_byte }) + } +} + +impl std::ops::Div for FeeRate { + type Output = Option; + fn div(self, other: Self) -> Self::Output { + let tokens_per_byte = self.tokens_per_byte / other.tokens_per_byte.into(); + tokens_per_byte.map(|tokens_per_byte| FeeRate { tokens_per_byte }) + } +} diff --git a/mempool/src/lib.rs b/mempool/src/lib.rs index db81f664cf..05a34d101c 100644 --- a/mempool/src/lib.rs +++ b/mempool/src/lib.rs @@ -1,5 +1,7 @@ #![deny(clippy::clone_on_ref_ptr)] +pub mod error; +mod feerate; pub mod pool; -pub use pool::Error as MempoolError; +pub use error::Error as MempoolError; diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 7db97b5b5e..36852ed80f 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -1,12 +1,13 @@ -use std::cmp::Ord; +use std::cell::Cell; use std::cmp::Ordering; use std::collections::BTreeMap; use std::collections::BTreeSet; use std::collections::HashMap; use std::fmt::Debug; +use std::time::Duration; +use mockall::*; use serialization::Encode; -use thiserror::Error; use common::chain::transaction::Transaction; use common::chain::transaction::TxInput; @@ -17,6 +18,12 @@ use common::primitives::Idable; use common::primitives::H256; use utils::newtype; +use crate::error::Error; +use crate::error::TxValidationError; +use crate::feerate::FeeRate; +use crate::feerate::INCREMENTAL_RELAY_FEE_RATE; + +const ROLLING_FEE_BASE_HALFLIFE: Time = Duration::new(60 * 60 * 12, 1); // TODO this willbe defined elsewhere (some of limits.rs file) const MAX_BLOCK_SIZE_BYTES: usize = 1_000_000; @@ -26,7 +33,30 @@ const MAX_BIP125_REPLACEMENT_CANDIDATES: usize = 100; // TODO this should really be taken from some global node settings const RELAY_FEE_PER_BYTE: usize = 1; -impl TryGetFee for MempoolImpl { +const MAX_MEMPOOL_SIZE_BYTES: usize = 300_000_000; + +const DEFAULT_MEMPOOL_EXPIRY: Duration = Duration::new(336 * 60 * 60, 0); + +const ROLLING_FEE_DECAY_INTERVAL: Time = Duration::new(10, 0); + +pub(crate) type MemoryUsage = usize; + +#[automock] +pub trait GetMemoryUsage: 'static { + fn get_memory_usage(&self) -> MemoryUsage; +} + +pub(crate) type Time = Duration; +pub trait GetTime: Clone + 'static { + fn get_time(&self) -> Time; +} + +impl TryGetFee for MempoolImpl +where + C: ChainState, + T: GetTime, + M: GetMemoryUsage, +{ fn try_get_fee(&self, tx: &Transaction) -> Result { let inputs = tx .inputs() @@ -58,16 +88,16 @@ fn get_relay_fee(tx: &Transaction) -> Amount { Amount::from_atoms(u128::try_from(tx.encoded_size() * RELAY_FEE_PER_BYTE).expect("Overflow")) } -pub trait Mempool { - fn create(chain_state: C) -> Self; +pub trait Mempool { + fn create(chain_state: C, clock: T, memory_usage_estimator: M) -> Self; fn add_transaction(&mut self, tx: Transaction) -> Result<(), Error>; fn get_all(&self) -> Vec<&Transaction>; fn contains_transaction(&self, tx: &Id) -> bool; fn drop_transaction(&mut self, tx: &Id); - fn new_tip_set(&mut self) -> Result<(), Error>; + fn new_tip_set(&mut self, chain_state: C); } -pub trait ChainState: Debug { +pub trait ChainState: Debug + 'static { fn contains_outpoint(&self, outpoint: &OutPoint) -> bool; fn get_outpoint_value(&self, outpoint: &OutPoint) -> Result; } @@ -87,16 +117,23 @@ struct TxMempoolEntry { parents: BTreeSet, children: BTreeSet, count_with_descendants: usize, + creation_time: Time, } impl TxMempoolEntry { - fn new(tx: Transaction, fee: Amount, parents: BTreeSet) -> TxMempoolEntry { + fn new( + tx: Transaction, + fee: Amount, + parents: BTreeSet, + creation_time: Time, + ) -> TxMempoolEntry { Self { tx, fee, parents, children: BTreeSet::default(), count_with_descendants: 1, + creation_time, } } @@ -112,10 +149,18 @@ impl TxMempoolEntry { self.parents.iter() } + fn unconfirmed_children(&self) -> impl Iterator { + self.children.iter() + } + fn get_children_mut(&mut self) -> &mut BTreeSet { &mut self.children } + fn get_parents_mut(&mut self) -> &mut BTreeSet { + &mut self.parents + } + fn is_replaceable(&self, store: &MempoolStore) -> bool { self.tx.is_replaceable() || self @@ -178,16 +223,64 @@ impl Ord for TxMempoolEntry { } } +#[derive(Clone, Copy, Debug)] +struct RollingFeeRate { + block_since_last_rolling_fee_bump: bool, + rolling_minimum_fee_rate: FeeRate, + last_rolling_fee_update: Time, +} + +impl RollingFeeRate { + fn decay_fee(mut self, halflife: Time, current_time: Time) -> Result { + println!( + "decay_fee: old fee rate: {:?}", + self.rolling_minimum_fee_rate + ); + println!("current_time {:?}", current_time); + println!("last_rolling_fee_update {:?}", self.last_rolling_fee_update); + println!("halflife {:?}", halflife); + let divisor = 2f64.powf( + (current_time.as_secs() - self.last_rolling_fee_update.as_secs()) as f64 + / (halflife.as_secs() as f64), + ); + self.rolling_minimum_fee_rate = + FeeRate::new(self.rolling_minimum_fee_rate.tokens_per_byte().div_by_float(divisor)); + + println!( + "decay_fee: new fee rate: {:?}", + self.rolling_minimum_fee_rate + ); + self.last_rolling_fee_update = current_time; + Ok(self) + } +} + +impl RollingFeeRate { + pub(crate) fn new(creation_time: Time) -> Self { + Self { + block_since_last_rolling_fee_bump: false, + rolling_minimum_fee_rate: FeeRate::new(Amount::from_atoms(0)), + last_rolling_fee_update: creation_time, + } + } +} + #[derive(Debug)] -pub struct MempoolImpl { +pub struct MempoolImpl { store: MempoolStore, + rolling_fee_rate: Cell, + max_size: usize, + max_tx_age: Duration, chain_state: C, + clock: T, + memory_usage_estimator: M, } #[derive(Debug)] struct MempoolStore { txs_by_id: HashMap, txs_by_fee: BTreeMap>, + txs_by_creation_time: BTreeMap>, spender_txs: BTreeMap, } @@ -196,10 +289,16 @@ impl MempoolStore { Self { txs_by_fee: BTreeMap::new(), txs_by_id: HashMap::new(), + txs_by_creation_time: BTreeMap::new(), spender_txs: BTreeMap::new(), } } + fn is_empty(&self) -> bool { + self.txs_by_id.is_empty() + // TODO maybe add some asserts here + } + // Checks whether the outpoint is to be created by an unconfirmed tx fn contains_outpoint(&self, outpoint: &OutPoint) -> bool { matches!(self.txs_by_id.get(&outpoint.tx_id().get_tx_id().expect("Not Coinbase").get()), @@ -232,12 +331,32 @@ impl MempoolStore { for parent in entry.unconfirmed_parents() { self.txs_by_id .get_mut(parent) - .expect("be there") + .expect("append_to_parents") .get_children_mut() .insert(entry.tx_id()); } } + fn remove_from_parents(&mut self, entry: &TxMempoolEntry) { + for parent in entry.unconfirmed_parents() { + self.txs_by_id + .get_mut(parent) + .expect("remove_from_parents") + .get_children_mut() + .remove(&entry.tx_id()); + } + } + + fn remove_from_children(&mut self, entry: &TxMempoolEntry) { + for child in entry.unconfirmed_children() { + self.txs_by_id + .get_mut(child) + .expect("remove_from_children") + .get_parents_mut() + .remove(&entry.tx_id()); + } + } + fn update_ancestor_count(&mut self, entry: &TxMempoolEntry) { for ancestor in entry.unconfirmed_ancestors(self).0 { let ancestor = self.txs_by_id.get_mut(&ancestor).expect("ancestor"); @@ -252,34 +371,68 @@ impl MempoolStore { } } - fn add_tx(&mut self, entry: TxMempoolEntry) -> Result<(), Error> { + fn add_tx(&mut self, entry: TxMempoolEntry) { self.append_to_parents(&entry); self.update_ancestor_count(&entry); self.mark_outpoints_as_spent(&entry); self.txs_by_fee.entry(entry.fee).or_default().insert(entry.tx_id()); + self.txs_by_creation_time + .entry(entry.creation_time) + .or_default() + .insert(entry.tx_id()); let tx_id = entry.tx_id(); self.txs_by_id.insert(entry.tx_id(), entry); assert!(self.txs_by_id.get(&tx_id).is_some()); - - Ok(()) } - fn drop_tx(&mut self, tx_id: &Id) { + fn remove_tx(&mut self, tx_id: &Id) { + println!("remove_tx: {}", tx_id.get()); if let Some(entry) = self.txs_by_id.remove(&tx_id.get()) { - self.txs_by_fee.entry(entry.fee).and_modify(|entries| { - entries.remove(&tx_id.get()).then(|| ()).expect("Inconsistent mempool store") - }); - self.spender_txs.retain(|_, id| *id != tx_id.get()) + self.update_for_drop(&entry); + self.drop_tx(&entry); } else { assert!(!self.txs_by_fee.values().flatten().any(|id| *id == tx_id.get())); assert!(!self.spender_txs.iter().any(|(_, id)| *id == tx_id.get())); } } + fn update_for_drop(&mut self, entry: &TxMempoolEntry) { + self.remove_from_parents(entry); + self.remove_from_children(entry); + } + + fn drop_tx(&mut self, entry: &TxMempoolEntry) { + self.txs_by_fee.entry(entry.fee).and_modify(|entries| { + entries.remove(&entry.tx_id()).then(|| ()).expect("Inconsistent mempool store") + }); + self.txs_by_creation_time.entry(entry.creation_time).and_modify(|entries| { + entries.remove(&entry.tx_id()).then(|| ()).expect("Inconsistent mempool store") + }); + self.spender_txs.retain(|_, id| *id != entry.tx_id()) + } + fn drop_conflicts(&mut self, conflicts: Conflicts) { for conflict in conflicts.0 { - self.drop_tx(&Id::new(conflict)) + self.remove_tx(&Id::new(conflict)) + } + } + + fn drop_tx_and_descendants(&mut self, tx_id: Id) { + if let Some(entry) = self.txs_by_id.get(&tx_id.get()).cloned() { + let descendants = entry.unconfirmed_descendants(self); + println!( + "about to drop {} which has {} descendants", + tx_id.get(), + descendants.len() + ); + self.remove_tx(&entry.tx.get_id()); + for descendant_id in descendants.0 { + // It may be that this descendant has several ancestors and has already been removed + if let Some(descendant) = self.txs_by_id.get(&descendant_id).cloned() { + self.remove_tx(&descendant.tx.get_id()) + } + } } } @@ -288,69 +441,80 @@ impl MempoolStore { } } -#[derive(Debug, Error)] -pub enum Error { - #[error("Mempool is full")] - MempoolFull, - #[error(transparent)] - TxValidationError(TxValidationError), -} +impl MempoolImpl +where + C: ChainState, + T: GetTime, + M: GetMemoryUsage, +{ + fn rolling_fee_halflife(&self) -> Time { + let mem_usage = self.get_memory_usage(); + if mem_usage < self.max_size / 4 { + ROLLING_FEE_BASE_HALFLIFE / 4 + } else if mem_usage < self.max_size / 2 { + ROLLING_FEE_BASE_HALFLIFE / 2 + } else { + ROLLING_FEE_BASE_HALFLIFE + } + } -#[derive(Debug, Error)] -pub enum TxValidationError { - #[error("No Inputs")] - NoInputs, - #[error("No Ouputs")] - NoOutputs, - #[error("DuplicateInputs")] - DuplicateInputs, - #[error("OutPointNotFound {outpoint:?}")] - OutPointNotFound { - outpoint: OutPoint, - tx_id: Id, - }, - #[error("ExceedsMaxBlockSize")] - ExceedsMaxBlockSize, - #[error("TransactionAlreadyInMempool")] - TransactionAlreadyInMempool, - #[error("ConflictWithIrreplaceableTransaction")] - ConflictWithIrreplaceableTransaction, - #[error("InputValuesOverflow")] - InputValuesOverflow, - #[error("OutputValuesOverflow")] - OutputValuesOverflow, - #[error("InputsBelowOutputs")] - InputsBelowOutputs, - #[error("ReplacementFeeLowerThanOriginal: The replacement transaction has fee {replacement_fee:?}, the original transaction has fee {original_fee:?}")] - ReplacementFeeLowerThanOriginal { - replacement_tx: H256, - replacement_fee: Amount, - original_tx: H256, - original_fee: Amount, - }, - #[error("TooManyPotentialReplacements")] - TooManyPotentialReplacements, - #[error("SpendsNewUnconfirmedInput")] - SpendsNewUnconfirmedOutput, - #[error("ConflictsFeeOverflow")] - ConflictsFeeOverflow, - #[error("TransactionFeeLowerThanConflictsWithDescendants")] - TransactionFeeLowerThanConflictsWithDescendants, - #[error("AdditionalFeesUnderflow")] - AdditionalFeesUnderflow, - #[error("InsufficientFeesToRelay")] - InsufficientFeesToRelay, - #[error("InsufficientFeesToRelayRBF")] - InsufficientFeesToRelayRBF, -} + fn get_memory_usage(&self) -> usize { + self.memory_usage_estimator.get_memory_usage() + } -impl From for Error { - fn from(e: TxValidationError) -> Self { - Error::TxValidationError(e) + pub(crate) fn update_min_fee_rate(&self, rate: FeeRate) { + let mut rolling_fee_rate = self.rolling_fee_rate.get(); + rolling_fee_rate.rolling_minimum_fee_rate = rate; + rolling_fee_rate.block_since_last_rolling_fee_bump = false; + self.rolling_fee_rate.set(rolling_fee_rate) + } + + pub(crate) fn get_update_min_fee_rate(&self) -> Result { + let rolling_fee_rate = self.rolling_fee_rate.get(); + if !rolling_fee_rate.block_since_last_rolling_fee_bump + || rolling_fee_rate.rolling_minimum_fee_rate == FeeRate::new(Amount::from_atoms(0)) + { + return Ok(rolling_fee_rate.rolling_minimum_fee_rate); + } else if self.clock.get_time() + > rolling_fee_rate.last_rolling_fee_update + ROLLING_FEE_DECAY_INTERVAL + { + // Decay the rolling fee + self.decay_rolling_fee_rate()?; + println!( + "rolling fee rate after decay_rolling_fee_rate {:?}", + self.rolling_fee_rate + ); + + if self.rolling_fee_rate.get().rolling_minimum_fee_rate + < (*INCREMENTAL_RELAY_FEE_RATE / FeeRate::new(Amount::from_atoms(2))) + .expect("not division by zero") + { + println!("rolling fee rate {:?} less than half of the incremental fee rate, dropping the fee", self.rolling_fee_rate.get().rolling_minimum_fee_rate); + self.drop_rolling_fee(); + return Ok(self.rolling_fee_rate.get().rolling_minimum_fee_rate); + } + } + + Ok(std::cmp::max( + self.rolling_fee_rate.get().rolling_minimum_fee_rate, + *INCREMENTAL_RELAY_FEE_RATE, + )) + } + + fn drop_rolling_fee(&self) { + let mut rolling_fee_rate = self.rolling_fee_rate.get(); + rolling_fee_rate.rolling_minimum_fee_rate = FeeRate::new(Amount::from_atoms(0)); + self.rolling_fee_rate.set(rolling_fee_rate) + } + + fn decay_rolling_fee_rate(&self) -> Result<(), TxValidationError> { + let halflife = self.rolling_fee_halflife(); + let time = self.clock.get_time(); + self.rolling_fee_rate + .set(self.rolling_fee_rate.get().decay_fee(halflife, time)?); + Ok(()) } -} -impl MempoolImpl { fn verify_inputs_available(&self, tx: &Transaction) -> Result<(), TxValidationError> { tx.inputs() .iter() @@ -380,7 +544,19 @@ impl MempoolImpl { .collect::>(); let fee = self.try_get_fee(&tx)?; - Ok(TxMempoolEntry::new(tx, fee, parents)) + let time = self.clock.get_time(); + Ok(TxMempoolEntry::new(tx, fee, parents, time)) + } + + fn get_update_minimum_mempool_fee( + &self, + tx: &Transaction, + ) -> Result { + let minimum_fee_rate = self.get_update_min_fee_rate()?; + println!("minimum fee rate {:?}", minimum_fee_rate); + println!("tx_size: {:?}", tx.encoded_size()); + + minimum_fee_rate.compute_fee(tx.encoded_size()) } fn validate_transaction(&self, tx: &Transaction) -> Result { @@ -415,13 +591,29 @@ impl MempoolImpl { self.pays_minimum_relay_fees(tx)?; + self.pays_minimum_mempool_fee(tx)?; + Ok(conflicts) } + fn pays_minimum_mempool_fee(&self, tx: &Transaction) -> Result<(), TxValidationError> { + let tx_fee = self.try_get_fee(tx)?; + let minimum_fee = self.get_update_minimum_mempool_fee(tx)?; + (tx_fee >= minimum_fee) + .then(|| ()) + .ok_or(TxValidationError::RollingFeeThresholdNotMet { + minimum_fee, + tx_fee, + }) + } + fn pays_minimum_relay_fees(&self, tx: &Transaction) -> Result<(), TxValidationError> { - (self.try_get_fee(tx)? >= get_relay_fee(tx)) + let tx_fee = self.try_get_fee(tx)?; + let relay_fee = get_relay_fee(tx); + eprintln!("tx_fee {:?}, relay_fee {:?}", tx_fee, relay_fee); + (tx_fee >= relay_fee) .then(|| ()) - .ok_or(TxValidationError::InsufficientFeesToRelay) + .ok_or(TxValidationError::InsufficientFeesToRelay { tx_fee, relay_fee }) } fn rbf_checks(&self, tx: &Transaction) -> Result { @@ -477,12 +669,7 @@ impl MempoolImpl { ) -> Result<(), TxValidationError> { let additional_fees = (self.try_get_fee(tx)? - total_conflict_fees) .ok_or(TxValidationError::AdditionalFeesUnderflow)?; - // TODO should we return an error here instead of expect? let relay_fee = get_relay_fee(tx); - eprintln!( - "relay_fee: {:?}, additional_fees {:?}, total_conflict_fees {:?}, replacement_fee: {:?}", - relay_fee, additional_fees, total_conflict_fees, self.try_get_fee(tx)? - ); (additional_fees >= relay_fee) .then(|| ()) .ok_or(TxValidationError::InsufficientFeesToRelayRBF) @@ -574,37 +761,142 @@ impl MempoolImpl { fn finalize_tx(&mut self, tx: Transaction, conflicts: Conflicts) -> Result<(), Error> { self.store.drop_conflicts(conflicts); let entry = self.create_entry(tx)?; - self.store.add_tx(entry)?; - // limit mempool size - Ok(()) + let id = entry.tx.get_id().get(); + self.store.add_tx(entry); + self.limit_mempool_size()?; + self.store.txs_by_id.contains_key(&id).then(|| ()).ok_or(Error::MempoolFull) } - #[allow(unused)] fn limit_mempool_size(&mut self) -> Result<(), Error> { + self.remove_expired_transactions(); + let removed_fees = self.trim()?; + if !removed_fees.is_empty() { + let new_minimum_fee_rate = + (*removed_fees.iter().max().expect("removed_fees should not be empty") + + *INCREMENTAL_RELAY_FEE_RATE) + .ok_or(TxValidationError::FeeRateError)?; + if new_minimum_fee_rate > self.rolling_fee_rate.get().rolling_minimum_fee_rate { + self.update_min_fee_rate(new_minimum_fee_rate) + } + } + Ok(()) } + + fn remove_expired_transactions(&mut self) { + let expired: Vec<_> = self + .store + .txs_by_creation_time + .values() + .flatten() + .map(|entry_id| self.store.txs_by_id.get(entry_id).expect("entry should exist")) + .filter(|entry| { + let now = self.clock.get_time(); + if now - entry.creation_time > self.max_tx_age { + println!( + "will evict {} which was created at {:?} and it is now {:?}", + entry.tx_id(), + entry.creation_time, + now + ); + true + } else { + false + } + }) + .cloned() + .collect(); + + for tx_id in expired.iter().map(|entry| entry.tx.get_id()) { + self.store.drop_tx_and_descendants(tx_id) + } + } + + fn trim(&mut self) -> Result, Error> { + let mut removed_fees = Vec::new(); + while !self.store.is_empty() && self.get_memory_usage() > self.max_size { + // TODO sort by descendant score, not by fee + let removed_id = + self.store.txs_by_fee.values().flatten().next().expect("pool not empty"); + let removed = + self.store.txs_by_id.get(removed_id).expect("tx with id should exist").clone(); + + println!( + "removed tx pays a fee of {:?} and has size {}", + removed.fee, + removed.tx.encoded_size() + ); + removed_fees.push(FeeRate::of_tx(removed.fee, removed.tx.encoded_size())?); + self.store.drop_tx_and_descendants(removed.tx.get_id()); + } + Ok(removed_fees) + } } -trait SpendsUnconfirmed { - fn spends_unconfirmed(&self, mempool: &MempoolImpl) -> bool; +trait SpendsUnconfirmed +where + C: ChainState, + T: GetTime, + M: GetMemoryUsage, +{ + fn spends_unconfirmed(&self, mempool: &MempoolImpl) -> bool; } -impl SpendsUnconfirmed for TxInput { - fn spends_unconfirmed(&self, mempool: &MempoolImpl) -> bool { +impl SpendsUnconfirmed for TxInput +where + C: ChainState, + T: GetTime, + M: GetMemoryUsage, +{ + fn spends_unconfirmed(&self, mempool: &MempoolImpl) -> bool { mempool.contains_transaction(self.outpoint().tx_id().get_tx_id().expect("Not coinbase")) } } -impl Mempool for MempoolImpl { - fn create(chain_state: C) -> Self { +#[derive(Clone)] +struct SystemClock; +impl GetTime for SystemClock { + fn get_time(&self) -> Duration { + common::primitives::time::get() + } +} + +#[derive(Clone)] +struct SystemUsageEstimator; +impl GetMemoryUsage for SystemUsageEstimator { + fn get_memory_usage(&self) -> MemoryUsage { + //TODO implement real usage estimation here + 0 + } +} + +impl Mempool for MempoolImpl +where + C: ChainState, + T: GetTime, + M: GetMemoryUsage, +{ + fn create(chain_state: C, clock: T, memory_usage_estimator: M) -> Self { Self { store: MempoolStore::new(), chain_state, + max_size: MAX_MEMPOOL_SIZE_BYTES, + max_tx_age: DEFAULT_MEMPOOL_EXPIRY, + rolling_fee_rate: Cell::new(RollingFeeRate::new(clock.get_time())), + clock, + memory_usage_estimator, } } - fn new_tip_set(&mut self) -> Result<(), Error> { - unimplemented!() + fn new_tip_set(&mut self, chain_state: C) { + self.chain_state = chain_state; + self.rolling_fee_rate.set({ + let mut rolling_fee_rate = self.rolling_fee_rate.get(); + // TODO Not sure we should set the flag to true when a block is disconnected/during a + // reorg + rolling_fee_rate.block_since_last_rolling_fee_bump = true; + rolling_fee_rate + }) } // @@ -635,7 +927,7 @@ impl Mempool for MempoolImpl { // TODO Consider returning an error fn drop_transaction(&mut self, tx_id: &Id) { - self.store.drop_tx(tx_id); + self.store.remove_tx(tx_id); } } @@ -656,7 +948,8 @@ mod tests { use common::chain::OutPointSourceId; use common::chain::OutputPurpose; use core::panic; - use rand::Rng; + use std::sync::atomic::{AtomicU64, Ordering}; + use std::sync::Arc; const DUMMY_WITNESS_MSG: &[u8] = b"dummy_witness_msg"; @@ -748,23 +1041,57 @@ mod tests { } } - impl MempoolImpl { - fn available_outpoints(&self) -> BTreeSet { - self.store + impl MempoolImpl + where + T: GetTime, + M: GetMemoryUsage, + { + fn available_outpoints(&self, allow_double_spend: bool) -> BTreeSet { + let mut available = self + .store .unconfirmed_outpoints() .into_iter() .chain(self.chain_state.confirmed_outpoints()) - .collect() + .collect::>(); + if !allow_double_spend { + available.retain(|valued_outpoint| { + !self.store.spender_txs.contains_key(&valued_outpoint.outpoint) + }); + } + available } fn get_input_value(&self, input: &TxInput) -> anyhow::Result { - self.available_outpoints() + let allow_double_spend = true; + self.available_outpoints(allow_double_spend) .iter() .find_map(|valued_outpoint| { (valued_outpoint.outpoint == *input.outpoint()).then(|| valued_outpoint.value) }) .ok_or_else(|| anyhow::anyhow!("No such unconfirmed output")) } + + fn get_minimum_rolling_fee(&self) -> FeeRate { + self.rolling_fee_rate.get().rolling_minimum_fee_rate + } + + fn process_block(&mut self, tx_id: &Id) -> anyhow::Result<()> { + let mut chain_state = self.chain_state.clone(); + chain_state.add_confirmed_tx( + self.store + .txs_by_id + .get(&tx_id.get()) + .cloned() + .ok_or_else(|| { + anyhow::anyhow!("process_block: tx {} not found in mempool", tx_id.get()) + })? + .tx, + ); + println!("Setting tip to {:?}", chain_state); + self.new_tip_set(chain_state); + self.drop_transaction(tx_id); + Ok(()) + } } #[derive(Debug, Clone)] @@ -789,36 +1116,10 @@ mod tests { } } - fn unspent_outpoints(&self) -> BTreeSet { - self.available_outpoints - .iter() - .map(|outpoint| { - let value = - self.get_outpoint_value(outpoint).expect("Inconsistent Chain State"); - ValuedOutPoint { - outpoint: outpoint.clone(), - value, - } - }) - .collect() - } - fn confirmed_txs(&self) -> &HashMap { &self.confirmed_txs } - fn get_outpoint_value(&self, outpoint: &OutPoint) -> Result { - self.confirmed_txs - .get(&outpoint.tx_id().get_tx_id().expect("Not Coinbase").get()) - .ok_or_else(|| anyhow::anyhow!("tx for outpoint sought in chain state, not found")) - .and_then(|tx| { - tx.outputs() - .get(outpoint.output_index() as usize) - .ok_or_else(|| anyhow::anyhow!("outpoint index out of bounds")) - .map(|output| output.value()) - }) - } - fn confirmed_outpoints(&self) -> BTreeSet { self.available_outpoints .iter() @@ -840,6 +1141,20 @@ mod tests { }) .collect() } + + fn add_confirmed_tx(&mut self, tx: Transaction) { + let outpoints_spent: BTreeSet<_> = + tx.inputs().iter().map(|input| input.outpoint()).collect(); + let outpoints_created: BTreeSet<_> = tx + .outputs() + .iter() + .enumerate() + .map(|(i, _)| OutPoint::new(OutPointSourceId::Transaction(tx.get_id()), i as u32)) + .collect(); + self.available_outpoints.extend(outpoints_created); + self.available_outpoints.retain(|outpoint| !outpoints_spent.contains(outpoint)); + self.confirmed_txs.insert(tx.get_id().get(), tx); + } } impl ChainState for ChainStateMock { @@ -860,12 +1175,20 @@ mod tests { } } + /* FIXME The second call in the following flow sometimes returns TransactionAlreadyInMempool + let tx1 = TxGenerator::new(&mempool).generate_tx()?; + mempool.add_transaction(tx1)?; + + let tx2 = TxGenerator::new(&mempool).generate_tx()?; + mempool.add_transaction(tx2)?; + */ struct TxGenerator { coin_pool: BTreeSet, num_inputs: usize, num_outputs: usize, tx_fee: Amount, replaceable: bool, + allow_double_spend: bool, } impl TxGenerator { @@ -884,40 +1207,27 @@ mod tests { self } - // TODO not sure if we need this - /* - fn with_fee(mut self, tx_fee: Amount) -> Self { - self.tx_fee = tx_fee; + fn with_fee(mut self, fee: Amount) -> Self { + self.tx_fee = fee; self } - */ - - fn new(mempool: &MempoolImpl) -> Self { - let unconfirmed_outputs = mempool.available_outpoints(); - Self::create_tx_generator(&mempool.chain_state, &unconfirmed_outputs) - } - - fn create_tx_generator( - chain_state: &ChainStateMock, - unconfirmed_outputs: &BTreeSet, - ) -> Self { - let coin_pool = chain_state - .unspent_outpoints() - .iter() - .chain(unconfirmed_outputs) - .cloned() - .collect(); + fn new() -> Self { Self { - coin_pool, + coin_pool: BTreeSet::new(), num_inputs: 1, num_outputs: 1, tx_fee: Amount::from_atoms(0), replaceable: false, + allow_double_spend: false, } } - fn generate_tx(&mut self) -> anyhow::Result { + fn generate_tx( + &mut self, + mempool: &MempoolImpl, + ) -> anyhow::Result { + self.coin_pool = mempool.available_outpoints(self.allow_double_spend); self.tx_fee = Amount::from_atoms(get_relay_fee_from_tx_size(estimate_tx_size( self.num_inputs, self.num_outputs, @@ -1016,12 +1326,7 @@ mod tests { let num_outpoints = self.coin_pool.len(); (num_outpoints > 0) .then(|| { - let index = rand::thread_rng().gen_range(0..num_outpoints); - self.coin_pool - .iter() - .nth(index) - .cloned() - .expect("Outpoint set should not be empty") + self.coin_pool.iter().next().cloned().expect("Outpoint set should not be empty") }) .ok_or_else(|| anyhow::anyhow!("no outpoints left")) } @@ -1033,7 +1338,7 @@ mod tests { #[test] fn add_single_tx() -> anyhow::Result<()> { - let mut mempool = MempoolImpl::create(ChainStateMock::new()); + let mut mempool = setup(); let genesis_tx = mempool .chain_state @@ -1069,13 +1374,12 @@ mod tests { #[test] fn txs_sorted() -> anyhow::Result<()> { - let chain_state = ChainStateMock::new(); - let mut mempool = MempoolImpl::create(chain_state); - let mut tx_generator = TxGenerator::new(&mempool); + let mut mempool = setup(); + let mut tx_generator = TxGenerator::new(); let target_txs = 10; for _ in 0..target_txs { - match tx_generator.generate_tx() { + match tx_generator.generate_tx(&mempool) { Ok(tx) => { mempool.add_transaction(tx.clone())?; } @@ -1097,9 +1401,9 @@ mod tests { #[test] fn tx_no_inputs() -> anyhow::Result<()> { let mut mempool = setup(); - let tx = TxGenerator::new(&mempool) + let tx = TxGenerator::new() .with_num_inputs(0) - .generate_tx() + .generate_tx(&mempool) .expect("generate_tx failed"); assert!(matches!( mempool.add_transaction(tx), @@ -1108,16 +1412,20 @@ mod tests { Ok(()) } - fn setup() -> MempoolImpl { - MempoolImpl::create(ChainStateMock::new()) + fn setup() -> MempoolImpl { + MempoolImpl::create( + ChainStateMock::new(), + SystemClock {}, + SystemUsageEstimator {}, + ) } #[test] fn tx_no_outputs() -> anyhow::Result<()> { let mut mempool = setup(); - let tx = TxGenerator::new(&mempool) + let tx = TxGenerator::new() .with_num_outputs(0) - .generate_tx() + .generate_tx(&mempool) .expect("generate_tx failed"); assert!(matches!( mempool.add_transaction(tx), @@ -1128,7 +1436,7 @@ mod tests { #[test] fn tx_duplicate_inputs() -> anyhow::Result<()> { - let mut mempool = MempoolImpl::create(ChainStateMock::new()); + let mut mempool = setup(); let genesis_tx = mempool .chain_state @@ -1166,7 +1474,7 @@ mod tests { #[test] fn tx_already_in_mempool() -> anyhow::Result<()> { - let mut mempool = MempoolImpl::create(ChainStateMock::new()); + let mut mempool = setup(); let genesis_tx = mempool .chain_state @@ -1198,7 +1506,7 @@ mod tests { #[test] fn outpoint_not_found() -> anyhow::Result<()> { - let mut mempool = MempoolImpl::create(ChainStateMock::new()); + let mut mempool = setup(); let genesis_tx = mempool .chain_state @@ -1242,9 +1550,9 @@ mod tests { #[test] fn tx_too_big() -> anyhow::Result<()> { let mut mempool = setup(); - let tx = TxGenerator::new(&mempool) + let tx = TxGenerator::new() .with_num_outputs(400_000) - .generate_tx() + .generate_tx(&mempool) .expect("generate_tx failed"); assert!(matches!( mempool.add_transaction(tx), @@ -1258,7 +1566,7 @@ mod tests { fn test_replace_tx(original_fee: Amount, replacement_fee: Amount) -> Result<(), Error> { let mut mempool = setup(); let outpoint = mempool - .available_outpoints() + .available_outpoints(true) .iter() .next() .expect("there should be an outpoint since setup creates the genesis transaction") @@ -1321,9 +1629,9 @@ mod tests { #[test] fn tx_replace_child() -> anyhow::Result<()> { let mut mempool = setup(); - let tx = TxGenerator::new(&mempool) + let tx = TxGenerator::new() .replaceable() - .generate_tx() + .generate_tx(&mempool) .expect("generate_replaceable_tx"); mempool.add_transaction(tx.clone())?; @@ -1354,11 +1662,11 @@ mod tests { Ok(()) } - // To test our validation of BIP125 Rule#4 (replacement transaction pays for its own bandwidth), we need to know the necessary relay fee before creating the transaction. The relay fee depends on the size of the transaction. The usual way to get the size of a transaction is to call `tx.encoded_size` but we cannot do this until we have created the transaction itself. To get around this scycle, we have precomputed the size of all transaction created by `tx_spend_input`. This value will be the same for all transactions created by this function. - const TX_SPEND_INPUT_SIZE: usize = 82; + // To test our validation of BIP125 Rule#4 (replacement transaction pays for its own bandwidth), we need to know the necessary relay fee before creating the transaction. The relay fee depends on the size of the transaction. The usual way to get the size of a transaction is to call `tx.encoded_size` but we cannot do this until we have created the transaction itself. To get around this cycle, we have precomputed the size of all transaction created by `tx_spend_input`. This value will be the same for all transactions created by this function. + const TX_SPEND_INPUT_SIZE: usize = 84; - fn tx_spend_input( - mempool: &MempoolImpl, + fn tx_spend_input( + mempool: &MempoolImpl, input: TxInput, fee: impl Into>, flags: u32, @@ -1371,8 +1679,8 @@ mod tests { tx_spend_several_inputs(mempool, &[input], fee, flags, locktime) } - fn tx_spend_several_inputs( - mempool: &MempoolImpl, + fn tx_spend_several_inputs( + mempool: &MempoolImpl, inputs: &[TxInput], fee: Amount, flags: u32, @@ -1417,9 +1725,9 @@ mod tests { let mut mempool = setup(); // TODO add a function which calculates the tx size based on number of outputs and inputs, // so that we can calculate the minimum relay fee - let tx = TxGenerator::new(&mempool) + let tx = TxGenerator::new() .with_num_outputs(2) - .generate_tx() + .generate_tx(&mempool) .expect("generate_replaceable_tx"); mempool.add_transaction(tx.clone())?; @@ -1500,6 +1808,7 @@ mod tests { #[test] fn tx_mempool_entry() -> anyhow::Result<()> { + use common::primitives::time; let mut mempool = setup(); // Input different flag values just to make the hashes of these dummy transactions // different @@ -1511,30 +1820,36 @@ mod tests { // Generation 1 let tx1_parents = BTreeSet::default(); - let entry1 = TxMempoolEntry::new(txs.get(0).unwrap().clone(), fee, tx1_parents); + let entry1 = + TxMempoolEntry::new(txs.get(0).unwrap().clone(), fee, tx1_parents, time::get()); let tx2_parents = BTreeSet::default(); - let entry2 = TxMempoolEntry::new(txs.get(1).unwrap().clone(), fee, tx2_parents); + let entry2 = + TxMempoolEntry::new(txs.get(1).unwrap().clone(), fee, tx2_parents, time::get()); // Generation 2 let tx3_parents = vec![entry1.tx_id(), entry2.tx_id()].into_iter().collect(); - let entry3 = TxMempoolEntry::new(txs.get(2).unwrap().clone(), fee, tx3_parents); + let entry3 = + TxMempoolEntry::new(txs.get(2).unwrap().clone(), fee, tx3_parents, time::get()); // Generation 3 let tx4_parents = vec![entry3.tx_id()].into_iter().collect(); let tx5_parents = vec![entry3.tx_id()].into_iter().collect(); - let entry4 = TxMempoolEntry::new(txs.get(3).unwrap().clone(), fee, tx4_parents); - let entry5 = TxMempoolEntry::new(txs.get(4).unwrap().clone(), fee, tx5_parents); + let entry4 = + TxMempoolEntry::new(txs.get(3).unwrap().clone(), fee, tx4_parents, time::get()); + let entry5 = + TxMempoolEntry::new(txs.get(4).unwrap().clone(), fee, tx5_parents, time::get()); // Generation 4 let tx6_parents = vec![entry3.tx_id(), entry4.tx_id(), entry5.tx_id()].into_iter().collect(); - let entry6 = TxMempoolEntry::new(txs.get(5).unwrap().clone(), fee, tx6_parents); + let entry6 = + TxMempoolEntry::new(txs.get(5).unwrap().clone(), fee, tx6_parents, time::get()); let entries = vec![entry1, entry2, entry3, entry4, entry5, entry6]; let ids = entries.clone().into_iter().map(|entry| entry.tx_id()).collect::>(); for entry in entries.into_iter() { - mempool.store.add_tx(entry)?; + mempool.store.add_tx(entry); } let entry1 = mempool.store.get_entry(ids.get(0).expect("index")).expect("entry"); @@ -1560,14 +1875,14 @@ mod tests { Ok(()) } - fn test_bip125_max_replacements( - mempool: &mut MempoolImpl, + fn test_bip125_max_replacements( + mempool: &mut MempoolImpl, num_potential_replacements: usize, ) -> anyhow::Result<()> { - let tx = TxGenerator::new(mempool) + let tx = TxGenerator::new() .with_num_outputs(num_potential_replacements - 1) .replaceable() - .generate_tx() + .generate_tx(mempool) .expect("generate_tx failed"); let input = tx.inputs().first().expect("one input").clone(); let outputs = tx.outputs().clone(); @@ -1626,10 +1941,10 @@ mod tests { #[test] fn spends_new_unconfirmed() -> anyhow::Result<()> { let mut mempool = setup(); - let tx = TxGenerator::new(&mempool) + let tx = TxGenerator::new() .with_num_outputs(2) .replaceable() - .generate_tx() + .generate_tx(&mempool) .expect("generate_replaceable_tx"); let outpoint_source_id = OutPointSourceId::Transaction(tx.get_id()); mempool.add_transaction(tx)?; @@ -1673,7 +1988,7 @@ mod tests { #[test] fn pays_more_than_conflicts_with_descendants() -> anyhow::Result<()> { let mut mempool = setup(); - let tx = TxGenerator::new(&mempool).generate_tx().expect("generate_replaceable_tx"); + let tx = TxGenerator::new().generate_tx(&mempool).expect("generate_replaceable_tx"); let tx_id = tx.get_id(); mempool.add_transaction(tx)?; @@ -1696,7 +2011,7 @@ mod tests { mempool.add_transaction(replaced_tx)?; // Create some children for this transaction - let descendant_outpoint_source_id = OutPointSourceId::Transaction(replaced_id.clone()); + let descendant_outpoint_source_id = OutPointSourceId::Transaction(replaced_id); let descendant1_fee = Amount::from_atoms(100); let descendant1 = tx_spend_input( @@ -1760,4 +2075,245 @@ mod tests { assert!(!mempool.contains_transaction(&descendant2_id)); Ok(()) } + + #[derive(Clone)] + struct MockClock { + time: Arc, + } + + impl MockClock { + fn new() -> Self { + Self { + time: Arc::new(AtomicU64::new(0)), + } + } + + fn set(&self, time: Time) { + self.time.store(time.as_secs(), Ordering::SeqCst) + } + + fn increment(&self, inc: Time) { + self.time.store( + self.time.load(Ordering::SeqCst) + inc.as_secs(), + Ordering::SeqCst, + ) + } + } + + impl GetTime for MockClock { + fn get_time(&self) -> Time { + Duration::new(self.time.load(Ordering::SeqCst), 0) + } + } + + #[test] + fn only_expired_entries_removed() -> anyhow::Result<()> { + let mock_clock = MockClock::new(); + + let mut mempool = MempoolImpl::create( + ChainStateMock::new(), + mock_clock.clone(), + SystemUsageEstimator {}, + ); + + let num_inputs = 1; + let num_outputs = 2; + let big_fee = get_relay_fee_from_tx_size(estimate_tx_size(num_inputs, num_outputs)) + 100; + let parent = TxGenerator::new() + .with_num_inputs(num_inputs) + .with_num_outputs(num_outputs) + .with_fee(Amount::from_atoms(big_fee)) + .generate_tx(&mempool)?; + let parent_id = parent.get_id(); + mempool.add_transaction(parent)?; + + let flags = 0; + let locktime = 0; + let outpoint_source_id = OutPointSourceId::Transaction(parent_id); + let child_0 = tx_spend_input( + &mempool, + TxInput::new( + outpoint_source_id.clone(), + 0, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ), + None, + flags, + locktime, + )?; + + let child_1 = tx_spend_input( + &mempool, + TxInput::new( + outpoint_source_id, + 1, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ), + None, + flags, + locktime, + )?; + let child_1_id = child_1.get_id(); + + let expired_tx_id = child_0.get_id(); + mempool.add_transaction(child_0)?; + + // Simulate the parent being added to a block + // We have to do this because if we leave this parent in the mempool then it will be + // expired, and so removed along with both its children, and thus the addition of child_1 to + // the mempool will fail + mempool.process_block(&parent_id)?; + mock_clock.set(DEFAULT_MEMPOOL_EXPIRY + Duration::new(1, 0)); + + mempool.add_transaction(child_1)?; + assert!(!mempool.contains_transaction(&expired_tx_id)); + assert!(mempool.contains_transaction(&child_1_id)); + Ok(()) + } + + #[test] + fn rolling_fee() -> anyhow::Result<()> { + let mock_clock = MockClock::new(); + let mut mock_usage = MockGetMemoryUsage::new(); + // Add parent + // Add first child + mock_usage.expect_get_memory_usage().times(2).return_const(0usize); + // Add second child, triggering the trimming process + mock_usage + .expect_get_memory_usage() + .times(1) + .return_const(MAX_MEMPOOL_SIZE_BYTES + 1); + // After removing one entry, cause the code to exit the loop by showing a small usage + mock_usage.expect_get_memory_usage().return_const(0usize); + + let chain_state = ChainStateMock::new(); + let mut mempool = MempoolImpl::create(chain_state, mock_clock.clone(), mock_usage); + + let num_inputs = 1; + let num_outputs = 3; + + // Use a higher than default fee because we don't want this transction to be evicted during + // the trimming process + let parent = TxGenerator::new() + .with_num_inputs(num_inputs) + .with_num_outputs(num_outputs) + .generate_tx(&mempool)?; + let parent_id = parent.get_id(); + mempool.add_transaction(parent)?; + + let flags = 0; + let locktime = 0; + let outpoint_source_id = OutPointSourceId::Transaction(parent_id); + + // child_0 has the lower fee so it will be evicted when memory usage is too high + let child_0 = tx_spend_input( + &mempool, + TxInput::new( + outpoint_source_id.clone(), + 0, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ), + None, + flags, + locktime, + )?; + let child_0_id = child_0.get_id(); + + let big_fee = Amount::from_atoms( + get_relay_fee_from_tx_size(estimate_tx_size(num_inputs, num_outputs)) + 100, + ); + let child_1 = tx_spend_input( + &mempool, + TxInput::new( + outpoint_source_id.clone(), + 1, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ), + big_fee, + flags, + locktime, + )?; + let child_1_id = child_1.get_id(); + mempool.add_transaction(child_0.clone())?; + println!("added child_0"); + mempool.add_transaction(child_1)?; + println!("added child_1"); + + assert_eq!(mempool.store.txs_by_id.len(), 2); + assert!(mempool.contains_transaction(&child_1_id)); + assert!(!mempool.contains_transaction(&child_0_id)); + let rolling_fee = mempool.get_minimum_rolling_fee(); + assert_eq!( + rolling_fee, + (FeeRate::of_tx(mempool.try_get_fee(&child_0)?, child_0.encoded_size())? + + *INCREMENTAL_RELAY_FEE_RATE) + .ok_or(TxValidationError::FeeRateError)? + ); + + // Now that the minimum rolling fee has been bumped up, a low-fee tx will not pass + // validation + let child_2 = tx_spend_input( + &mempool, + TxInput::new( + outpoint_source_id.clone(), + 2, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ), + None, + flags, + locktime, + )?; + assert!(matches!( + mempool.add_transaction(child_2), + Err(Error::TxValidationError( + TxValidationError::RollingFeeThresholdNotMet { .. } + )) + )); + + // We provide a sufficient fee for the tx to pass the minimum rolling fee requirement + let child_2_high_fee = tx_spend_input( + &mempool, + TxInput::new( + outpoint_source_id, + 2, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ), + mempool.get_minimum_rolling_fee().compute_fee(estimate_tx_size(1, 1))?, + flags, + locktime, + )?; + mempool.add_transaction(child_2_high_fee)?; + + // We simulate a block being accepted so the rolling fee will begin to decay + mempool.process_block(&parent_id)?; + + // Because the rolling fee is only updated when we attempt to add a tx to the mempool + // we need to submit a "dummy" tx to trigger these updates. + + // Since memory usage is now zero, it is less than 1/4 of the max size + // and ROLLING_FEE_BASE_HALFLIFE / 4 is the time it will take for the fee to halve + let halflife = ROLLING_FEE_BASE_HALFLIFE / 4; + mock_clock.increment(halflife); + let dummy_tx = TxGenerator::new().generate_tx(&mempool)?; + assert!(matches!( + mempool.add_transaction(dummy_tx.clone()), + Err(Error::TxValidationError( + TxValidationError::RollingFeeThresholdNotMet { .. } + )) + )); + assert_eq!( + mempool.get_minimum_rolling_fee(), + (rolling_fee / FeeRate::new(Amount::from_atoms(2))).unwrap() + ); + + mock_clock.increment(halflife); + // Fee will have dropped under INCREMENTAL_RELAY_FEE_RATE / 2 by now, so it will be set to + // zero and our tx will be submitted successfully + mempool.add_transaction(dummy_tx)?; + assert_eq!( + mempool.get_minimum_rolling_fee(), + FeeRate::new(Amount::from_atoms(0)) + ); + Ok(()) + } } From 51abf64b26b96c2ee0c045e83d6382bede3e4044 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Thu, 17 Mar 2022 13:57:59 +0700 Subject: [PATCH 079/153] rename random_unspent_outpoint -> get_unspent_outpoint --- mempool/src/pool.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 36852ed80f..e5560f720e 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -1311,7 +1311,7 @@ mod tests { } fn generate_input(&self) -> anyhow::Result<(TxInput, Amount)> { - let ValuedOutPoint { outpoint, value } = self.random_unspent_outpoint()?; + let ValuedOutPoint { outpoint, value } = self.get_unspent_outpoint()?; Ok(( TxInput::new( outpoint.tx_id(), @@ -1322,7 +1322,7 @@ mod tests { )) } - fn random_unspent_outpoint(&self) -> anyhow::Result { + fn get_unspent_outpoint(&self) -> anyhow::Result { let num_outpoints = self.coin_pool.len(); (num_outpoints > 0) .then(|| { From ef14b0e005afa6beaaf2bfeae42f19ec315ce582 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Thu, 17 Mar 2022 14:34:45 +0700 Subject: [PATCH 080/153] Make InsufficientFeesToRelay error more informative --- mempool/src/pool.rs | 49 +++++++++++++++++++++------------------------ 1 file changed, 23 insertions(+), 26 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index e5560f720e..63aec777ab 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -610,7 +610,7 @@ where fn pays_minimum_relay_fees(&self, tx: &Transaction) -> Result<(), TxValidationError> { let tx_fee = self.try_get_fee(tx)?; let relay_fee = get_relay_fee(tx); - eprintln!("tx_fee {:?}, relay_fee {:?}", tx_fee, relay_fee); + eprintln!("tx_fee: {:?}, relay_fee: {:?}", tx_fee, relay_fee); (tx_fee >= relay_fee) .then(|| ()) .ok_or(TxValidationError::InsufficientFeesToRelay { tx_fee, relay_fee }) @@ -1232,7 +1232,7 @@ mod tests { self.num_inputs, self.num_outputs, ))); - let valued_inputs = self.generate_tx_inputs(); + let valued_inputs = self.generate_tx_inputs()?; let outputs = self.generate_tx_outputs(&valued_inputs)?; let locktime = 0; let flags = if self.replaceable { 1 } else { 0 }; @@ -1253,11 +1253,22 @@ mod tests { Ok(tx) } - fn generate_tx_inputs(&mut self) -> Vec<(TxInput, Amount)> { - std::iter::repeat(()) - .take(self.num_inputs) - .filter_map(|_| self.generate_input().ok()) - .collect() + fn generate_tx_inputs(&mut self) -> anyhow::Result> { + Ok(self + .get_unspent_outpoints(self.num_inputs)? + .iter() + .map(|valued_outpoint| { + let ValuedOutPoint { outpoint, value } = valued_outpoint; + ( + TxInput::new( + outpoint.tx_id(), + outpoint.output_index(), + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ), + *value, + ) + }) + .collect()) } fn generate_tx_outputs( @@ -1283,7 +1294,7 @@ mod tests { let mut left_to_spend = total_to_spend; let mut outputs = Vec::new(); - let max_output_value = Amount::from_atoms(10_000); + let max_output_value = Amount::from_atoms(100_000); // We want every output to be spendable in a single-input, single-output transaction // So it has to larger in value than the relay fee for such a transaction let min_output_value = Amount::from_atoms(1000); @@ -1310,24 +1321,10 @@ mod tests { Ok(outputs) } - fn generate_input(&self) -> anyhow::Result<(TxInput, Amount)> { - let ValuedOutPoint { outpoint, value } = self.get_unspent_outpoint()?; - Ok(( - TxInput::new( - outpoint.tx_id(), - outpoint.output_index(), - InputWitness::NoSignature(None), - ), - value, - )) - } - - fn get_unspent_outpoint(&self) -> anyhow::Result { - let num_outpoints = self.coin_pool.len(); - (num_outpoints > 0) - .then(|| { - self.coin_pool.iter().next().cloned().expect("Outpoint set should not be empty") - }) + fn get_unspent_outpoints(&self, num_outputs: usize) -> anyhow::Result> { + let num_available_outpoints = self.coin_pool.len(); + (num_available_outpoints >= num_outputs) + .then(|| self.coin_pool.iter().take(num_outputs).cloned().collect()) .ok_or_else(|| anyhow::anyhow!("no outpoints left")) } } From a716afefb02ce274ad2495c943c07d6c5897c482 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Thu, 17 Mar 2022 14:56:54 +0700 Subject: [PATCH 081/153] TESTS: TxGenerator fee handling --- mempool/src/pool.rs | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 63aec777ab..f7461ecf28 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -1186,7 +1186,7 @@ mod tests { coin_pool: BTreeSet, num_inputs: usize, num_outputs: usize, - tx_fee: Amount, + tx_fee: Option, replaceable: bool, allow_double_spend: bool, } @@ -1208,7 +1208,7 @@ mod tests { } fn with_fee(mut self, fee: Amount) -> Self { - self.tx_fee = fee; + self.tx_fee = Some(fee); self } @@ -1217,7 +1217,7 @@ mod tests { coin_pool: BTreeSet::new(), num_inputs: 1, num_outputs: 1, - tx_fee: Amount::from_atoms(0), + tx_fee: None, replaceable: false, allow_double_spend: false, } @@ -1228,12 +1228,16 @@ mod tests { mempool: &MempoolImpl, ) -> anyhow::Result { self.coin_pool = mempool.available_outpoints(self.allow_double_spend); - self.tx_fee = Amount::from_atoms(get_relay_fee_from_tx_size(estimate_tx_size( - self.num_inputs, - self.num_outputs, - ))); + let fee = if let Some(tx_fee) = self.tx_fee { + tx_fee + } else { + Amount::from_atoms(get_relay_fee_from_tx_size(estimate_tx_size( + self.num_inputs, + self.num_outputs, + ))) + }; let valued_inputs = self.generate_tx_inputs()?; - let outputs = self.generate_tx_outputs(&valued_inputs)?; + let outputs = self.generate_tx_outputs(&valued_inputs, fee)?; let locktime = 0; let flags = if self.replaceable { 1 } else { 0 }; let (inputs, _): (Vec, Vec) = valued_inputs.into_iter().unzip(); @@ -1274,6 +1278,7 @@ mod tests { fn generate_tx_outputs( &self, inputs: &[(TxInput, Amount)], + tx_fee: Amount, ) -> anyhow::Result> { if self.num_outputs == 0 { return Ok(vec![]); @@ -1287,8 +1292,8 @@ mod tests { let sum_of_inputs = values.into_iter().sum::>().expect("Overflow in sum of input values"); - let total_to_spend = (sum_of_inputs - self.tx_fee).ok_or_else(||anyhow::anyhow!( - "generate_tx_outputs: underflow computing total_to_spend - sum_of_inputs = {:?}, fee = {:?}", sum_of_inputs, self.tx_fee + let total_to_spend = (sum_of_inputs - tx_fee).ok_or_else(||anyhow::anyhow!( + "generate_tx_outputs: underflow computing total_to_spend - sum_of_inputs = {:?}, fee = {:?}", sum_of_inputs, tx_fee ))?; let mut left_to_spend = total_to_spend; From c660762e1013cf0d2a898fdadf0e022e82b7298f Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Thu, 17 Mar 2022 15:50:59 +0700 Subject: [PATCH 082/153] TESTS: Add manual implementation of Ord for ValuedOutPoint so coin_pool is sorted in descending fee order --- mempool/src/pool.rs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index f7461ecf28..1e8efdbd01 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -953,11 +953,24 @@ mod tests { const DUMMY_WITNESS_MSG: &[u8] = b"dummy_witness_msg"; - #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)] + #[derive(Debug, PartialEq, Eq, Clone)] struct ValuedOutPoint { outpoint: OutPoint, value: Amount, } + + impl std::cmp::PartialOrd for ValuedOutPoint { + fn partial_cmp(&self, other: &Self) -> Option { + other.value.partial_cmp(&self.value) + } + } + + impl std::cmp::Ord for ValuedOutPoint { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + other.value.cmp(&self.value) + } + } + fn dummy_input() -> TxInput { let outpoint_source_id = OutPointSourceId::Transaction(Id::new(H256::zero())); let output_index = 0; From 1c1cdab692823596cd4ea93873d9212d5c97435b Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Thu, 17 Mar 2022 16:47:04 +0700 Subject: [PATCH 083/153] TESTS: infrastructure improvements --- mempool/Cargo.toml | 3 --- mempool/src/pool.rs | 64 +++++++++++++++++++++++++++++++-------------- 2 files changed, 44 insertions(+), 23 deletions(-) diff --git a/mempool/Cargo.toml b/mempool/Cargo.toml index 41ef0d3225..8eb1dc8321 100644 --- a/mempool/Cargo.toml +++ b/mempool/Cargo.toml @@ -15,6 +15,3 @@ anyhow = "1.0" thiserror = "1.0" lazy_static = "1.4" mockall = "0.11.0" - -[dev-dependencies] -rand = "0.8.4" diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 1e8efdbd01..9e8da094d4 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -995,8 +995,14 @@ mod tests { let locktime = 0; let size = Transaction::new(flags, inputs, outputs, locktime).unwrap().encoded_size(); // Take twice the encoded size of the dummy tx.Real Txs are larger than these dummy ones, - // but taking 2 times the size seems to work - 2 * size + // but taking 3 times the size seems to ensure our txs won't fail the minimum relay fee + // validation (see the function `pays_minimum_relay_fees`) + let result = 3 * size; + println!( + "estimated size for tx with {} inputs and {} outputs: {}", + num_inputs, num_outputs, result + ); + result } #[test] @@ -1249,7 +1255,11 @@ mod tests { self.num_outputs, ))) }; - let valued_inputs = self.generate_tx_inputs()?; + println!( + "Trying to build a tx with {} inputs, {} outputs, and a fee of {:?}", + self.num_inputs, self.num_outputs, fee + ); + let valued_inputs = self.generate_tx_inputs(fee)?; let outputs = self.generate_tx_outputs(&valued_inputs, fee)?; let locktime = 0; let flags = if self.replaceable { 1 } else { 0 }; @@ -1270,9 +1280,9 @@ mod tests { Ok(tx) } - fn generate_tx_inputs(&mut self) -> anyhow::Result> { + fn generate_tx_inputs(&mut self, fee: Amount) -> anyhow::Result> { Ok(self - .get_unspent_outpoints(self.num_inputs)? + .get_unspent_outpoints(self.num_inputs, fee)? .iter() .map(|valued_outpoint| { let ValuedOutPoint { outpoint, value } = valued_outpoint; @@ -1312,19 +1322,8 @@ mod tests { let mut left_to_spend = total_to_spend; let mut outputs = Vec::new(); - let max_output_value = Amount::from_atoms(100_000); - // We want every output to be spendable in a single-input, single-output transaction - // So it has to larger in value than the relay fee for such a transaction - let min_output_value = Amount::from_atoms(1000); for _ in 0..self.num_outputs - 1 { - let max_output_value = std::cmp::min( - (left_to_spend / 2).expect("division failed"), - max_output_value, - ); - if max_output_value == Amount::from_atoms(0) { - return Err(anyhow::Error::msg("No more funds to spend")); - } - let value = Amount::random(min_output_value..=max_output_value); + let value = std::cmp::min(Amount::from_atoms(1000), left_to_spend); outputs.push(TxOutput::new( value, OutputPurpose::Transfer(Destination::AnyoneCanSpend), @@ -1339,11 +1338,35 @@ mod tests { Ok(outputs) } - fn get_unspent_outpoints(&self, num_outputs: usize) -> anyhow::Result> { + fn get_unspent_outpoints( + &self, + num_outputs: usize, + fee: Amount, + ) -> anyhow::Result> { + println!( + "get_unspent_outpoints: num_outputs: {}, fee: {:?}", + num_outputs, fee + ); let num_available_outpoints = self.coin_pool.len(); - (num_available_outpoints >= num_outputs) + let outpoints: Vec<_> = (num_available_outpoints >= num_outputs) .then(|| self.coin_pool.iter().take(num_outputs).cloned().collect()) - .ok_or_else(|| anyhow::anyhow!("no outpoints left")) + .ok_or_else(|| anyhow::anyhow!("no outpoints left"))?; + let sum_of_outputs = outpoints + .iter() + .map(|valued_outpoint| valued_outpoint.value) + .sum::>() + .expect("sum error"); + if fee > sum_of_outputs { + println!( + "get_unspent_outpoints: fee is {:?} but sum_of_outputs is {:?}", + fee, sum_of_outputs + ); + return Err(anyhow::Error::msg( + "get_unspent_outpoints:: not enough for fees", + )); + } + + Ok(outpoints) } } @@ -1418,6 +1441,7 @@ mod tests { let mut mempool = setup(); let tx = TxGenerator::new() .with_num_inputs(0) + .with_fee(Amount::from_atoms(0)) .generate_tx(&mempool) .expect("generate_tx failed"); assert!(matches!( From 7c859426216a15006c2f10bed32e027f5ad5e30f Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Thu, 17 Mar 2022 16:48:57 +0700 Subject: [PATCH 084/153] real_size_tx test --- mempool/src/pool.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 9e8da094d4..7ca1e7ba4d 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -1012,6 +1012,17 @@ mod tests { eprintln!("1, 400: {}", estimate_tx_size(1, 400)); } + #[test] + fn real_size() -> anyhow::Result<()> { + let mempool = setup(); + let tx = TxGenerator::new() + .with_num_inputs(1) + .with_num_outputs(400) + .generate_tx(&mempool)?; + println!("real size of tx {}", tx.encoded_size()); + Ok(()) + } + fn valued_outpoint( tx_id: &Id, outpoint_index: u32, From 772a4e7ae46b5b89c7f8894b2c5c8353f6eee2a8 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Thu, 17 Mar 2022 16:49:27 +0700 Subject: [PATCH 085/153] Add test for different_size_txs --- mempool/src/pool.rs | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 7ca1e7ba4d..70e403bd2c 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -2366,4 +2366,37 @@ mod tests { ); Ok(()) } + + #[test] + fn different_size_txs() -> anyhow::Result<()> { + use std::time::Duration; + use std::time::Instant; + let mut mempool = setup(); + let initial_tx = TxGenerator::new() + .with_num_inputs(1) + .with_num_outputs(10_000) + .generate_tx(&mempool)?; + mempool.add_transaction(initial_tx)?; + + let target_txs = 100; + let mut time_processing_txs = Duration::from_millis(0); + let mut time_creating_txs = Duration::from_millis(0); + for i in 0..target_txs { + let num_inputs = i + 1; + let num_outputs = i + 1; + let before_creating = Instant::now(); + let tx = TxGenerator::new() + .with_num_inputs(num_inputs) + .with_num_outputs(num_outputs) + .generate_tx(&mempool)?; + time_creating_txs += before_creating.elapsed(); + let before_processing = Instant::now(); + mempool.add_transaction(tx)?; + time_processing_txs += before_processing.elapsed() + } + + println!("Total time spent processing: {:?}", time_processing_txs); + println!("Total time spent creating: {:?}", time_creating_txs); + Ok(()) + } } From 970661bfbb067ccc03f91056c3e9d69a5f2908d3 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Fri, 18 Mar 2022 12:29:20 +0700 Subject: [PATCH 086/153] Replace println calls with logging --- mempool/Cargo.toml | 1 + mempool/src/pool.rs | 87 ++++++++++++++++++++++++--------------------- 2 files changed, 48 insertions(+), 40 deletions(-) diff --git a/mempool/Cargo.toml b/mempool/Cargo.toml index 8eb1dc8321..3fb2cd2738 100644 --- a/mempool/Cargo.toml +++ b/mempool/Cargo.toml @@ -11,6 +11,7 @@ parity-scale-codec = {version = "2.3.1", features = ["derive", "chain-error"]} serialization = { path = '../serialization' } common = { path = '../common' } utils = { path = '../utils' } +logging = { path = '../logging' } anyhow = "1.0" thiserror = "1.0" lazy_static = "1.4" diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 70e403bd2c..b56093f7df 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -16,6 +16,9 @@ use common::primitives::amount::Amount; use common::primitives::Id; use common::primitives::Idable; use common::primitives::H256; + +use logging::log; + use utils::newtype; use crate::error::Error; @@ -232,13 +235,14 @@ struct RollingFeeRate { impl RollingFeeRate { fn decay_fee(mut self, halflife: Time, current_time: Time) -> Result { - println!( - "decay_fee: old fee rate: {:?}", - self.rolling_minimum_fee_rate + log::trace!( + "decay_fee: old fee rate: {:?}\nCurrent time: {:?}\nLast Rolling Fee Update: {:?}\nHalflife: {:?}", + self.rolling_minimum_fee_rate, + self.last_rolling_fee_update, + current_time, + halflife, ); - println!("current_time {:?}", current_time); - println!("last_rolling_fee_update {:?}", self.last_rolling_fee_update); - println!("halflife {:?}", halflife); + let divisor = 2f64.powf( (current_time.as_secs() - self.last_rolling_fee_update.as_secs()) as f64 / (halflife.as_secs() as f64), @@ -246,7 +250,7 @@ impl RollingFeeRate { self.rolling_minimum_fee_rate = FeeRate::new(self.rolling_minimum_fee_rate.tokens_per_byte().div_by_float(divisor)); - println!( + log::trace!( "decay_fee: new fee rate: {:?}", self.rolling_minimum_fee_rate ); @@ -387,7 +391,7 @@ impl MempoolStore { } fn remove_tx(&mut self, tx_id: &Id) { - println!("remove_tx: {}", tx_id.get()); + log::info!("remove_tx: {}", tx_id.get()); if let Some(entry) = self.txs_by_id.remove(&tx_id.get()) { self.update_for_drop(&entry); self.drop_tx(&entry); @@ -421,8 +425,8 @@ impl MempoolStore { fn drop_tx_and_descendants(&mut self, tx_id: Id) { if let Some(entry) = self.txs_by_id.get(&tx_id.get()).cloned() { let descendants = entry.unconfirmed_descendants(self); - println!( - "about to drop {} which has {} descendants", + log::trace!( + "Dropping tx {} which has {} descendants", tx_id.get(), descendants.len() ); @@ -480,7 +484,7 @@ where { // Decay the rolling fee self.decay_rolling_fee_rate()?; - println!( + log::debug!( "rolling fee rate after decay_rolling_fee_rate {:?}", self.rolling_fee_rate ); @@ -489,7 +493,7 @@ where < (*INCREMENTAL_RELAY_FEE_RATE / FeeRate::new(Amount::from_atoms(2))) .expect("not division by zero") { - println!("rolling fee rate {:?} less than half of the incremental fee rate, dropping the fee", self.rolling_fee_rate.get().rolling_minimum_fee_rate); + log::trace!("rolling fee rate {:?} less than half of the incremental fee rate, dropping the fee", self.rolling_fee_rate.get().rolling_minimum_fee_rate); self.drop_rolling_fee(); return Ok(self.rolling_fee_rate.get().rolling_minimum_fee_rate); } @@ -553,8 +557,8 @@ where tx: &Transaction, ) -> Result { let minimum_fee_rate = self.get_update_min_fee_rate()?; - println!("minimum fee rate {:?}", minimum_fee_rate); - println!("tx_size: {:?}", tx.encoded_size()); + log::debug!("minimum fee rate {:?}", minimum_fee_rate); + log::debug!("tx_size: {:?}", tx.encoded_size()); minimum_fee_rate.compute_fee(tx.encoded_size()) } @@ -793,8 +797,8 @@ where .filter(|entry| { let now = self.clock.get_time(); if now - entry.creation_time > self.max_tx_age { - println!( - "will evict {} which was created at {:?} and it is now {:?}", + log::trace!( + "Evicting tx {} which was created at {:?}. It is now {:?}", entry.tx_id(), entry.creation_time, now @@ -821,8 +825,9 @@ where let removed = self.store.txs_by_id.get(removed_id).expect("tx with id should exist").clone(); - println!( - "removed tx pays a fee of {:?} and has size {}", + log::debug!( + "Mempool trim: Evicting tx {} which pays a fee of {:?} and has size {}", + removed.tx_id(), removed.fee, removed.tx.encoded_size() ); @@ -998,18 +1003,21 @@ mod tests { // but taking 3 times the size seems to ensure our txs won't fail the minimum relay fee // validation (see the function `pays_minimum_relay_fees`) let result = 3 * size; - println!( + log::debug!( "estimated size for tx with {} inputs and {} outputs: {}", - num_inputs, num_outputs, result + num_inputs, + num_outputs, + result ); result } #[test] fn dummy_size() { - eprintln!("1, 1: {}", estimate_tx_size(1, 1)); - eprintln!("1, 2: {}", estimate_tx_size(1, 2)); - eprintln!("1, 400: {}", estimate_tx_size(1, 400)); + logging::try_init_logging::<&str>(None); + log::debug!("1, 1: {}", estimate_tx_size(1, 1)); + log::debug!("1, 2: {}", estimate_tx_size(1, 2)); + log::debug!("1, 400: {}", estimate_tx_size(1, 400)); } #[test] @@ -1019,7 +1027,7 @@ mod tests { .with_num_inputs(1) .with_num_outputs(400) .generate_tx(&mempool)?; - println!("real size of tx {}", tx.encoded_size()); + log::debug!("real size of tx {}", tx.encoded_size()); Ok(()) } @@ -1117,7 +1125,7 @@ mod tests { })? .tx, ); - println!("Setting tip to {:?}", chain_state); + log::debug!("Setting tip to {:?}", chain_state); self.new_tip_set(chain_state); self.drop_transaction(tx_id); Ok(()) @@ -1266,9 +1274,11 @@ mod tests { self.num_outputs, ))) }; - println!( + log::debug!( "Trying to build a tx with {} inputs, {} outputs, and a fee of {:?}", - self.num_inputs, self.num_outputs, fee + self.num_inputs, + self.num_outputs, + fee ); let valued_inputs = self.generate_tx_inputs(fee)?; let outputs = self.generate_tx_outputs(&valued_inputs, fee)?; @@ -1354,9 +1364,10 @@ mod tests { num_outputs: usize, fee: Amount, ) -> anyhow::Result> { - println!( + log::debug!( "get_unspent_outpoints: num_outputs: {}, fee: {:?}", - num_outputs, fee + num_outputs, + fee ); let num_available_outpoints = self.coin_pool.len(); let outpoints: Vec<_> = (num_available_outpoints >= num_outputs) @@ -1368,16 +1379,12 @@ mod tests { .sum::>() .expect("sum error"); if fee > sum_of_outputs { - println!( - "get_unspent_outpoints: fee is {:?} but sum_of_outputs is {:?}", - fee, sum_of_outputs - ); - return Err(anyhow::Error::msg( - "get_unspent_outpoints:: not enough for fees", - )); + Err(anyhow::Error::msg( + "get_unspent_outpoints:: fee is {:?} but sum of outputs is {:?}", + )) + } else { + Ok(outpoints) } - - Ok(outpoints) } } @@ -2395,8 +2402,8 @@ mod tests { time_processing_txs += before_processing.elapsed() } - println!("Total time spent processing: {:?}", time_processing_txs); - println!("Total time spent creating: {:?}", time_creating_txs); + log::info!("Total time spent processing: {:?}", time_processing_txs); + log::info!("Total time spent creating: {:?}", time_creating_txs); Ok(()) } } From fbc372304297eb74c0f440f58e95c73c649883d0 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Fri, 18 Mar 2022 12:48:59 +0700 Subject: [PATCH 087/153] Delete old TODOs and constant pertaining to mempool fullness --- mempool/src/pool.rs | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index b56093f7df..63acfeb4bd 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -30,9 +30,8 @@ const ROLLING_FEE_BASE_HALFLIFE: Time = Duration::new(60 * 60 * 12, 1); // TODO this willbe defined elsewhere (some of limits.rs file) const MAX_BLOCK_SIZE_BYTES: usize = 1_000_000; -const MEMPOOL_MAX_TXS: usize = 1_000_000; - const MAX_BIP125_REPLACEMENT_CANDIDATES: usize = 100; + // TODO this should really be taken from some global node settings const RELAY_FEE_PER_BYTE: usize = 1; @@ -906,12 +905,6 @@ where // fn add_transaction(&mut self, tx: Transaction) -> Result<(), Error> { - // TODO (1). First, we need to decide on criteria for the Mempool to be considered full. Maybe number - // of transactions is not a good enough indicator. Consider checking mempool size as well - // TODO (2) What to do when the mempool is full. Instead of rejecting Do incoming transaction we probably want to evict a low-score transaction - if self.store.txs_by_fee.len() >= MEMPOOL_MAX_TXS { - return Err(Error::MempoolFull); - } let conflicts = self.validate_transaction(&tx)?; self.finalize_tx(tx, conflicts)?; Ok(()) From 63052f93ea93b03c569d35fbfd760c78494cd733 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Mon, 21 Mar 2022 11:33:11 +0700 Subject: [PATCH 088/153] Use logging in mempool tests --- Cargo.lock | 2 +- mempool/src/pool.rs | 24 ++++++++++++++++++++---- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index eedda1f76f..301ed0571f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2193,9 +2193,9 @@ dependencies = [ "anyhow", "common", "lazy_static", + "logging", "mockall", "parity-scale-codec 2.3.1", - "rand 0.8.5", "serialization", "thiserror", "utils", diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 63acfeb4bd..a09b8f345d 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -1007,7 +1007,7 @@ mod tests { #[test] fn dummy_size() { - logging::try_init_logging::<&str>(None); + logging::init_logging::<&str>(None); log::debug!("1, 1: {}", estimate_tx_size(1, 1)); log::debug!("1, 2: {}", estimate_tx_size(1, 2)); log::debug!("1, 400: {}", estimate_tx_size(1, 400)); @@ -1463,6 +1463,7 @@ mod tests { } fn setup() -> MempoolImpl { + logging::init_logging::<&str>(None); MempoolImpl::create( ChainStateMock::new(), SystemClock {}, @@ -1742,13 +1743,18 @@ mod tests { .collect::, _>>()? .into_iter() .sum::>() - .ok_or_else(|| anyhow::anyhow!("tx_spend_input: overflow"))?; + .ok_or_else(|| { + let msg = String::from("tx_spend_input: overflow"); + log::error!("{}", msg); + anyhow::Error::msg(msg) + })?; let available_for_spending = (input_value - fee).ok_or_else(|| { let msg = format!( "tx_spend_several_inputs: input_value ({:?}) lower than fee ({:?})", input_value, fee ); + log::error!("{}", msg); anyhow::Error::msg(msg) })?; let spent = (available_for_spending / 2).expect("division error"); @@ -2223,6 +2229,7 @@ mod tests { #[test] fn rolling_fee() -> anyhow::Result<()> { + logging::init_logging::<&str>(None); let mock_clock = MockClock::new(); let mut mock_usage = MockGetMemoryUsage::new(); // Add parent @@ -2249,7 +2256,10 @@ mod tests { .with_num_outputs(num_outputs) .generate_tx(&mempool)?; let parent_id = parent.get_id(); + log::debug!("parent_id: {}", parent_id.get()); + log::debug!("before adding parent"); mempool.add_transaction(parent)?; + log::debug!("after adding parent"); let flags = 0; let locktime = 0; @@ -2268,6 +2278,7 @@ mod tests { locktime, )?; let child_0_id = child_0.get_id(); + log::debug!("child_0_id {}", child_0_id.get()); let big_fee = Amount::from_atoms( get_relay_fee_from_tx_size(estimate_tx_size(num_inputs, num_outputs)) + 100, @@ -2284,10 +2295,11 @@ mod tests { locktime, )?; let child_1_id = child_1.get_id(); + log::debug!("child_1_id {}", child_1_id.get()); mempool.add_transaction(child_0.clone())?; - println!("added child_0"); + log::debug!("added child_0"); mempool.add_transaction(child_1)?; - println!("added child_1"); + log::debug!("added child_1"); assert_eq!(mempool.store.txs_by_id.len(), 2); assert!(mempool.contains_transaction(&child_1_id)); @@ -2313,6 +2325,7 @@ mod tests { flags, locktime, )?; + log::debug!("before child2"); assert!(matches!( mempool.add_transaction(child_2), Err(Error::TxValidationError( @@ -2332,6 +2345,7 @@ mod tests { flags, locktime, )?; + log::debug!("before child2_high_fee"); mempool.add_transaction(child_2_high_fee)?; // We simulate a block being accepted so the rolling fee will begin to decay @@ -2345,6 +2359,7 @@ mod tests { let halflife = ROLLING_FEE_BASE_HALFLIFE / 4; mock_clock.increment(halflife); let dummy_tx = TxGenerator::new().generate_tx(&mempool)?; + log::debug!("first attempt to add dummy"); assert!(matches!( mempool.add_transaction(dummy_tx.clone()), Err(Error::TxValidationError( @@ -2359,6 +2374,7 @@ mod tests { mock_clock.increment(halflife); // Fee will have dropped under INCREMENTAL_RELAY_FEE_RATE / 2 by now, so it will be set to // zero and our tx will be submitted successfully + log::debug!("second attempt to add dummy"); mempool.add_transaction(dummy_tx)?; assert_eq!( mempool.get_minimum_rolling_fee(), From 6ce5bdd93d0bf865acae553d39ca7473a73631a5 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Thu, 31 Mar 2022 15:56:05 +0700 Subject: [PATCH 089/153] Improve test function generate_tx_outputs --- mempool/src/pool.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index a09b8f345d..d7daa7ea15 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -1333,11 +1333,13 @@ mod tests { "generate_tx_outputs: underflow computing total_to_spend - sum_of_inputs = {:?}, fee = {:?}", sum_of_inputs, tx_fee ))?; + let value = (sum_of_inputs / u128::try_from(self.num_outputs).expect("conversion")) + .expect("not dividing by zero"); + let mut left_to_spend = total_to_spend; let mut outputs = Vec::new(); for _ in 0..self.num_outputs - 1 { - let value = std::cmp::min(Amount::from_atoms(1000), left_to_spend); outputs.push(TxOutput::new( value, OutputPurpose::Transfer(Destination::AnyoneCanSpend), From 6425ada57fa48f08c18d7a740c8ca1e53bde6afc Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Mon, 4 Apr 2022 10:42:22 +0700 Subject: [PATCH 090/153] make unspend_outpoints a function --- mempool/src/pool.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index d7daa7ea15..ff834ffb97 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -374,6 +374,10 @@ impl MempoolStore { } } + fn unspend_outpoints(&mut self, entry: &TxMempoolEntry) { + self.spender_txs.retain(|_, id| *id != entry.tx_id()) + } + fn add_tx(&mut self, entry: TxMempoolEntry) { self.append_to_parents(&entry); self.update_ancestor_count(&entry); @@ -412,7 +416,7 @@ impl MempoolStore { self.txs_by_creation_time.entry(entry.creation_time).and_modify(|entries| { entries.remove(&entry.tx_id()).then(|| ()).expect("Inconsistent mempool store") }); - self.spender_txs.retain(|_, id| *id != entry.tx_id()) + self.unspend_outpoints(entry) } fn drop_conflicts(&mut self, conflicts: Conflicts) { From 577ce2452fd8e2fa8b67b0f0c9028f33600905c7 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Fri, 1 Apr 2022 12:32:44 +0700 Subject: [PATCH 091/153] derive Debug for Ancestors, Descendants, Conflicts --- mempool/src/pool.rs | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index ff834ffb97..3a96ffc84c 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -108,9 +108,20 @@ trait TryGetFee { fn try_get_fee(&self, tx: &Transaction) -> Result; } -newtype!(struct Ancestors(BTreeSet)); -newtype!(struct Descendants(BTreeSet)); -newtype!(struct Conflicts(BTreeSet)); +newtype!( + #[derive(Debug)] + struct Ancestors(BTreeSet) +); + +newtype!( + #[derive(Debug)] + struct Descendants(BTreeSet) +); + +newtype!( + #[derive(Debug)] + struct Conflicts(BTreeSet) +); #[derive(Debug, PartialEq, Eq, Clone)] struct TxMempoolEntry { From f016eb49f013a11a738a6afaca58e0b01bbbf73e Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Wed, 30 Mar 2022 16:16:56 +0700 Subject: [PATCH 092/153] Add fees_with_descendant field to TxMempoolEntry --- mempool/src/pool.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 3a96ffc84c..a25376cece 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -130,6 +130,7 @@ struct TxMempoolEntry { parents: BTreeSet, children: BTreeSet, count_with_descendants: usize, + fees_with_descendants: Amount, creation_time: Time, } @@ -147,6 +148,7 @@ impl TxMempoolEntry { children: BTreeSet::default(), count_with_descendants: 1, creation_time, + fees_with_descendants: fee, } } From 438eee8c11d50133ed8cc7de4f8057c53b2a0034 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Thu, 31 Mar 2022 13:21:25 +0700 Subject: [PATCH 093/153] Rename txs_by_fee -> txs_by_descendant_score --- mempool/src/pool.rs | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index a25376cece..f511b64350 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -295,7 +295,7 @@ pub struct MempoolImpl { #[derive(Debug)] struct MempoolStore { txs_by_id: HashMap, - txs_by_fee: BTreeMap>, + txs_by_descendant_score: BTreeMap>, txs_by_creation_time: BTreeMap>, spender_txs: BTreeMap, } @@ -303,7 +303,7 @@ struct MempoolStore { impl MempoolStore { fn new() -> Self { Self { - txs_by_fee: BTreeMap::new(), + txs_by_descendant_score: BTreeMap::new(), txs_by_id: HashMap::new(), txs_by_creation_time: BTreeMap::new(), spender_txs: BTreeMap::new(), @@ -396,7 +396,7 @@ impl MempoolStore { self.update_ancestor_count(&entry); self.mark_outpoints_as_spent(&entry); - self.txs_by_fee.entry(entry.fee).or_default().insert(entry.tx_id()); + self.txs_by_descendant_score.entry(entry.fee).or_default().insert(entry.tx_id()); self.txs_by_creation_time .entry(entry.creation_time) .or_default() @@ -412,7 +412,7 @@ impl MempoolStore { self.update_for_drop(&entry); self.drop_tx(&entry); } else { - assert!(!self.txs_by_fee.values().flatten().any(|id| *id == tx_id.get())); + assert!(!self.txs_by_descendant_score.values().flatten().any(|id| *id == tx_id.get())); assert!(!self.spender_txs.iter().any(|(_, id)| *id == tx_id.get())); } } @@ -423,7 +423,7 @@ impl MempoolStore { } fn drop_tx(&mut self, entry: &TxMempoolEntry) { - self.txs_by_fee.entry(entry.fee).and_modify(|entries| { + self.txs_by_descendant_score.entry(entry.fee).and_modify(|entries| { entries.remove(&entry.tx_id()).then(|| ()).expect("Inconsistent mempool store") }); self.txs_by_creation_time.entry(entry.creation_time).and_modify(|entries| { @@ -836,8 +836,13 @@ where let mut removed_fees = Vec::new(); while !self.store.is_empty() && self.get_memory_usage() > self.max_size { // TODO sort by descendant score, not by fee - let removed_id = - self.store.txs_by_fee.values().flatten().next().expect("pool not empty"); + let removed_id = self + .store + .txs_by_descendant_score + .values() + .flatten() + .next() + .expect("pool not empty"); let removed = self.store.txs_by_id.get(removed_id).expect("tx with id should exist").clone(); @@ -929,7 +934,7 @@ where fn get_all(&self) -> Vec<&Transaction> { self.store - .txs_by_fee + .txs_by_descendant_score .values() .flatten() .map(|id| &self.store.get_entry(id).expect("entry").tx) From 870a6368758ddc6cc43fcdc477dbdf9c0f85f3bd Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Wed, 30 Mar 2022 17:09:26 +0700 Subject: [PATCH 094/153] new function update_ancestor_state --- mempool/src/pool.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index f511b64350..6731eb5137 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -380,6 +380,10 @@ impl MempoolStore { } } + fn update_ancestor_state(&mut self, entry: &TxMempoolEntry) { + self.update_ancestor_count(entry); + } + fn mark_outpoints_as_spent(&mut self, entry: &TxMempoolEntry) { let id = entry.tx_id(); for outpoint in entry.tx.inputs().iter().map(|input| input.outpoint()) { @@ -393,7 +397,7 @@ impl MempoolStore { fn add_tx(&mut self, entry: TxMempoolEntry) { self.append_to_parents(&entry); - self.update_ancestor_count(&entry); + self.update_ancestor_state(&entry); self.mark_outpoints_as_spent(&entry); self.txs_by_descendant_score.entry(entry.fee).or_default().insert(entry.tx_id()); From 136f3004f82742a248cf3d4d8a667906f5a496ee Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Wed, 30 Mar 2022 17:11:08 +0700 Subject: [PATCH 095/153] Update fees_with_descendants field of ancestors when new entry is inserted --- mempool/src/error.rs | 2 ++ mempool/src/pool.rs | 54 +++++++++++++++++++++++++++++++++++--------- 2 files changed, 45 insertions(+), 11 deletions(-) diff --git a/mempool/src/error.rs b/mempool/src/error.rs index 59cca905a7..5bf76207c1 100644 --- a/mempool/src/error.rs +++ b/mempool/src/error.rs @@ -66,6 +66,8 @@ pub enum TxValidationError { RollingFeeThresholdNotMet { minimum_fee: Amount, tx_fee: Amount }, #[error("FeeRate error")] FeeRateError, + #[error("AncestorFeeUpdateOverflow")] + AncestorFeeUpdateOverflow, #[error("Internal Error")] InternalError, } diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 6731eb5137..88f5ead4ab 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -380,8 +380,18 @@ impl MempoolStore { } } - fn update_ancestor_state(&mut self, entry: &TxMempoolEntry) { + fn update_ancestor_fees(&mut self, entry: &TxMempoolEntry) -> Result<(), Error> { + for ancestor in entry.unconfirmed_ancestors(self).0 { + let ancestor = self.txs_by_id.get_mut(&ancestor).expect("ancestor"); + ancestor.fees_with_descendants = (ancestor.fees_with_descendants + entry.fee) + .ok_or(TxValidationError::AncestorFeeUpdateOverflow)? + } + Ok(()) + } + + fn update_ancestor_state(&mut self, entry: &TxMempoolEntry) -> Result<(), Error> { self.update_ancestor_count(entry); + self.update_ancestor_fees(entry) } fn mark_outpoints_as_spent(&mut self, entry: &TxMempoolEntry) { @@ -395,19 +405,41 @@ impl MempoolStore { self.spender_txs.retain(|_, id| *id != entry.tx_id()) } - fn add_tx(&mut self, entry: TxMempoolEntry) { + fn add_tx(&mut self, entry: TxMempoolEntry) -> Result<(), Error> { self.append_to_parents(&entry); - self.update_ancestor_state(&entry); + self.update_ancestor_state(&entry)?; self.mark_outpoints_as_spent(&entry); - self.txs_by_descendant_score.entry(entry.fee).or_default().insert(entry.tx_id()); - self.txs_by_creation_time - .entry(entry.creation_time) - .or_default() - .insert(entry.tx_id()); + let creation_time = entry.creation_time; let tx_id = entry.tx_id(); - self.txs_by_id.insert(entry.tx_id(), entry); + + self.txs_by_id.insert(tx_id, entry.clone()); + + self.add_to_descendant_score_index(&entry); + self.txs_by_creation_time.entry(creation_time).or_default().insert(tx_id); assert!(self.txs_by_id.get(&tx_id).is_some()); + Ok(()) + } + + fn add_to_descendant_score_index(&mut self, entry: &TxMempoolEntry) { + // Since the ancestors of `entry` have had their descendant score modified, their ordering + // in txs_by_descendant_score may no longer be correct. We thus remove all ancestors and + // reinsert them, taking the new, update fees into account + let ancestors = entry.unconfirmed_ancestors(self); + for entries in self.txs_by_descendant_score.values_mut() { + entries.retain(|id| !ancestors.contains(id)) + } + for ancestor_id in ancestors.0 { + let ancestor = self.txs_by_id.get(&ancestor_id).expect("Inconsistent mempool state"); + self.txs_by_descendant_score + .entry(ancestor.fee) + .or_default() + .insert(ancestor_id); + } + self.txs_by_descendant_score + .entry(entry.fees_with_descendants) + .or_default() + .insert(entry.tx_id()); } fn remove_tx(&mut self, tx_id: &Id) { @@ -786,7 +818,7 @@ where self.store.drop_conflicts(conflicts); let entry = self.create_entry(tx)?; let id = entry.tx.get_id().get(); - self.store.add_tx(entry); + self.store.add_tx(entry)?; self.limit_mempool_size()?; self.store.txs_by_id.contains_key(&id).then(|| ()).ok_or(Error::MempoolFull) } @@ -1933,7 +1965,7 @@ mod tests { let ids = entries.clone().into_iter().map(|entry| entry.tx_id()).collect::>(); for entry in entries.into_iter() { - mempool.store.add_tx(entry); + mempool.store.add_tx(entry)?; } let entry1 = mempool.store.get_entry(ids.get(0).expect("index")).expect("entry"); From 6f2d69aeb4e5f8b07f1f0542c700d24660e6568e Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Thu, 31 Mar 2022 15:30:35 +0700 Subject: [PATCH 096/153] remove outdated TODO --- mempool/src/pool.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 88f5ead4ab..f9fb1d853a 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -1839,8 +1839,6 @@ mod tests { #[test] fn one_ancestor_signal_is_enough() -> anyhow::Result<()> { let mut mempool = setup(); - // TODO add a function which calculates the tx size based on number of outputs and inputs, - // so that we can calculate the minimum relay fee let tx = TxGenerator::new() .with_num_outputs(2) .generate_tx(&mempool) From 2d7772761fdf6648cc5bb35e400d4971b7e61bfb Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Fri, 1 Apr 2022 13:14:16 +0700 Subject: [PATCH 097/153] Introduce DescendantScore type and sort according to it --- mempool/src/pool.rs | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index f9fb1d853a..c89900e5a6 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -238,6 +238,11 @@ impl Ord for TxMempoolEntry { } } +newtype!( + #[derive(Debug, PartialEq, Eq, Ord, PartialOrd)] + struct DescendantScore(Amount) +); + #[derive(Clone, Copy, Debug)] struct RollingFeeRate { block_since_last_rolling_fee_bump: bool, @@ -295,7 +300,7 @@ pub struct MempoolImpl { #[derive(Debug)] struct MempoolStore { txs_by_id: HashMap, - txs_by_descendant_score: BTreeMap>, + txs_by_descendant_score: BTreeMap>, txs_by_creation_time: BTreeMap>, spender_txs: BTreeMap, } @@ -422,9 +427,17 @@ impl MempoolStore { } fn add_to_descendant_score_index(&mut self, entry: &TxMempoolEntry) { + self.refresh_ancestors(entry); + self.txs_by_descendant_score + .entry(entry.fees_with_descendants.into()) + .or_default() + .insert(entry.tx_id()); + } + + fn refresh_ancestors(&mut self, entry: &TxMempoolEntry) { // Since the ancestors of `entry` have had their descendant score modified, their ordering // in txs_by_descendant_score may no longer be correct. We thus remove all ancestors and - // reinsert them, taking the new, update fees into account + // reinsert them, taking the new, updated fees into account let ancestors = entry.unconfirmed_ancestors(self); for entries in self.txs_by_descendant_score.values_mut() { entries.retain(|id| !ancestors.contains(id)) @@ -432,14 +445,12 @@ impl MempoolStore { for ancestor_id in ancestors.0 { let ancestor = self.txs_by_id.get(&ancestor_id).expect("Inconsistent mempool state"); self.txs_by_descendant_score - .entry(ancestor.fee) + .entry(ancestor.fees_with_descendants.into()) .or_default() .insert(ancestor_id); } - self.txs_by_descendant_score - .entry(entry.fees_with_descendants) - .or_default() - .insert(entry.tx_id()); + + self.txs_by_descendant_score.retain(|_score, txs| !txs.is_empty()); } fn remove_tx(&mut self, tx_id: &Id) { @@ -459,9 +470,11 @@ impl MempoolStore { } fn drop_tx(&mut self, entry: &TxMempoolEntry) { - self.txs_by_descendant_score.entry(entry.fee).and_modify(|entries| { - entries.remove(&entry.tx_id()).then(|| ()).expect("Inconsistent mempool store") - }); + self.txs_by_descendant_score + .entry(entry.fees_with_descendants.into()) + .and_modify(|entries| { + entries.remove(&entry.tx_id()).then(|| ()).expect("Inconsistent mempool store") + }); self.txs_by_creation_time.entry(entry.creation_time).and_modify(|entries| { entries.remove(&entry.tx_id()).then(|| ()).expect("Inconsistent mempool store") }); @@ -883,9 +896,9 @@ where self.store.txs_by_id.get(removed_id).expect("tx with id should exist").clone(); log::debug!( - "Mempool trim: Evicting tx {} which pays a fee of {:?} and has size {}", + "Mempool trim: Evicting tx {} which has a descendant score of {:?} and has size {}", removed.tx_id(), - removed.fee, + removed.fees_with_descendants, removed.tx.encoded_size() ); removed_fees.push(FeeRate::of_tx(removed.fee, removed.tx.encoded_size())?); From 80dd6118255a04d094a12a355ba5df17e662961f Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Mon, 4 Apr 2022 10:38:23 +0700 Subject: [PATCH 098/153] Update descendant score index when an entry is dropped --- mempool/src/pool.rs | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index c89900e5a6..9063f774cd 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -470,17 +470,29 @@ impl MempoolStore { } fn drop_tx(&mut self, entry: &TxMempoolEntry) { - self.txs_by_descendant_score - .entry(entry.fees_with_descendants.into()) - .and_modify(|entries| { - entries.remove(&entry.tx_id()).then(|| ()).expect("Inconsistent mempool store") - }); + self.remove_from_descendant_score_index(entry); self.txs_by_creation_time.entry(entry.creation_time).and_modify(|entries| { entries.remove(&entry.tx_id()).then(|| ()).expect("Inconsistent mempool store") }); self.unspend_outpoints(entry) } + fn remove_from_descendant_score_index(&mut self, entry: &TxMempoolEntry) { + self.refresh_ancestors(entry); + self.txs_by_descendant_score + .entry(entry.fees_with_descendants.into()) + .or_default() + .remove(&entry.tx_id()); + if self + .txs_by_descendant_score + .get(&entry.fees_with_descendants.into()) + .expect("key must exist") + .is_empty() + { + self.txs_by_descendant_score.remove(&entry.fees_with_descendants.into()); + } + } + fn drop_conflicts(&mut self, conflicts: Conflicts) { for conflict in conflicts.0 { self.remove_tx(&Id::new(conflict)) From 487ec7ec120b9f6690224768f4f095a20055fcc7 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Mon, 4 Apr 2022 11:13:26 +0700 Subject: [PATCH 099/153] Refactor and rename udpate_ancestor_state -> update_ancestor_state_for_add --- mempool/src/pool.rs | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 9063f774cd..d4850b5e10 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -378,27 +378,16 @@ impl MempoolStore { } } - fn update_ancestor_count(&mut self, entry: &TxMempoolEntry) { - for ancestor in entry.unconfirmed_ancestors(self).0 { - let ancestor = self.txs_by_id.get_mut(&ancestor).expect("ancestor"); - ancestor.count_with_descendants += 1; - } - } - - fn update_ancestor_fees(&mut self, entry: &TxMempoolEntry) -> Result<(), Error> { + fn update_ancestor_state_for_add(&mut self, entry: &TxMempoolEntry) -> Result<(), Error> { for ancestor in entry.unconfirmed_ancestors(self).0 { let ancestor = self.txs_by_id.get_mut(&ancestor).expect("ancestor"); ancestor.fees_with_descendants = (ancestor.fees_with_descendants + entry.fee) - .ok_or(TxValidationError::AncestorFeeUpdateOverflow)? + .ok_or(TxValidationError::AncestorFeeUpdateOverflow)?; + ancestor.count_with_descendants += 1; } Ok(()) } - fn update_ancestor_state(&mut self, entry: &TxMempoolEntry) -> Result<(), Error> { - self.update_ancestor_count(entry); - self.update_ancestor_fees(entry) - } - fn mark_outpoints_as_spent(&mut self, entry: &TxMempoolEntry) { let id = entry.tx_id(); for outpoint in entry.tx.inputs().iter().map(|input| input.outpoint()) { @@ -412,7 +401,7 @@ impl MempoolStore { fn add_tx(&mut self, entry: TxMempoolEntry) -> Result<(), Error> { self.append_to_parents(&entry); - self.update_ancestor_state(&entry)?; + self.update_ancestor_state_for_add(&entry)?; self.mark_outpoints_as_spent(&entry); let creation_time = entry.creation_time; From a30b7bd9f21c4f96ad1fc882914ba76dc118e963 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Mon, 4 Apr 2022 11:23:35 +0700 Subject: [PATCH 100/153] Update ancestor state when entry is dropped --- mempool/src/pool.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index d4850b5e10..76ea2cfa36 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -388,6 +388,15 @@ impl MempoolStore { Ok(()) } + fn update_ancestor_state_for_drop(&mut self, entry: &TxMempoolEntry) { + for ancestor in entry.unconfirmed_ancestors(self).0 { + let ancestor = self.txs_by_id.get_mut(&ancestor).expect("ancestor"); + ancestor.fees_with_descendants = + (ancestor.fees_with_descendants - entry.fee).expect("fee with descendants"); + ancestor.count_with_descendants -= 1; + } + } + fn mark_outpoints_as_spent(&mut self, entry: &TxMempoolEntry) { let id = entry.tx_id(); for outpoint in entry.tx.inputs().iter().map(|input| input.outpoint()) { @@ -446,6 +455,7 @@ impl MempoolStore { log::info!("remove_tx: {}", tx_id.get()); if let Some(entry) = self.txs_by_id.remove(&tx_id.get()) { self.update_for_drop(&entry); + self.update_ancestor_state_for_drop(&entry); self.drop_tx(&entry); } else { assert!(!self.txs_by_descendant_score.values().flatten().any(|id| *id == tx_id.get())); From 98125c2b9ebaa6ce0de183f01be8e638cd76400f Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Thu, 31 Mar 2022 15:50:02 +0700 Subject: [PATCH 101/153] TESTS: descendant score --- mempool/src/pool.rs | 104 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 76ea2cfa36..974e185da8 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -2497,4 +2497,108 @@ mod tests { log::info!("Total time spent creating: {:?}", time_creating_txs); Ok(()) } + + #[test] + fn descendant_score() -> anyhow::Result<()> { + let mut mempool = setup(); + let tx = TxGenerator::new() + .with_num_outputs(2) + .generate_tx(&mempool) + .expect("generate_replaceable_tx"); + let tx_id = tx.get_id(); + mempool.add_transaction(tx)?; + + let outpoint_source_id = OutPointSourceId::Transaction(tx_id); + + let flags = 0; + let locktime = 0; + + let tx_b_fee = Amount::from(get_relay_fee_from_tx_size(estimate_tx_size(1, 2))); + let tx_a_fee = (tx_b_fee + Amount::from(1000)).unwrap(); + let tx_c_fee = (tx_a_fee + Amount::from(1000)).unwrap(); + let tx_a = tx_spend_input( + &mempool, + TxInput::new(outpoint_source_id.clone(), 0, DUMMY_WITNESS_MSG.to_vec()), + tx_a_fee, + flags, + locktime, + )?; + let tx_a_id = tx_a.get_id(); + log::debug!("tx_a_id : {}", tx_a_id.get()); + log::debug!("tx_a fee : {:?}", mempool.try_get_fee(&tx_a)?); + mempool.add_transaction(tx_a)?; + + let tx_b = tx_spend_input( + &mempool, + TxInput::new(outpoint_source_id, 1, DUMMY_WITNESS_MSG.to_vec()), + tx_b_fee, + flags, + locktime, + )?; + let tx_b_id = tx_b.get_id(); + log::debug!("tx_b_id : {}", tx_b_id.get()); + log::debug!("tx_b fee : {:?}", mempool.try_get_fee(&tx_b)?); + mempool.add_transaction(tx_b)?; + + let tx_c = tx_spend_input( + &mempool, + TxInput::new( + OutPointSourceId::Transaction(tx_b_id.clone()), + 0, + DUMMY_WITNESS_MSG.to_vec(), + ), + tx_c_fee, + flags, + locktime, + )?; + let tx_c_id = tx_c.get_id(); + log::debug!("tx_c_id : {}", tx_c_id.get()); + log::debug!("tx_c fee : {:?}", mempool.try_get_fee(&tx_c)?); + mempool.add_transaction(tx_c)?; + + let entry_a = mempool.store.txs_by_id.get(&tx_a_id.get()).expect("tx_a"); + log::debug!("entry a has score {:?}", entry_a.fees_with_descendants); + let entry_b = mempool.store.txs_by_id.get(&tx_b_id.get()).expect("tx_b"); + log::debug!("entry b has score {:?}", entry_b.fees_with_descendants); + let entry_c = mempool.store.txs_by_id.get(&tx_c_id.get()).expect("tx_c").clone(); + log::debug!("entry c has score {:?}", entry_c.fees_with_descendants); + assert_eq!(entry_a.fee, entry_a.fees_with_descendants); + assert_eq!( + entry_b.fees_with_descendants, + (entry_b.fee + entry_c.fee).unwrap() + ); + assert!(!mempool.store.txs_by_descendant_score.contains_key(&tx_b_fee.into())); + log::debug!( + "raw_txs_by_descendant_score {:?}", + mempool.store.txs_by_descendant_score + ); + check_txs_sorted_by_descendant_sore(&mempool); + + mempool.drop_transaction(&entry_c.tx.get_id()); + assert!(!mempool.store.txs_by_descendant_score.contains_key(&tx_c_fee.into())); + let entry_b = mempool.store.txs_by_id.get(&tx_b_id.get()).expect("tx_b"); + assert_eq!(entry_b.fees_with_descendants, entry_b.fee); + + check_txs_sorted_by_descendant_sore(&mempool); + + Ok(()) + } + + fn check_txs_sorted_by_descendant_sore( + mempool: &MempoolImpl, + ) { + let txs_by_descendant_score = + mempool.store.txs_by_descendant_score.values().flatten().collect::>(); + for i in 0..(txs_by_descendant_score.len() - 1) { + log::debug!("i = {}", i); + let tx_id = txs_by_descendant_score.get(i).unwrap(); + let next_tx_id = txs_by_descendant_score.get(i + 1).unwrap(); + let entry_score = mempool.store.txs_by_id.get(tx_id).unwrap().fees_with_descendants; + let next_entry_score = + mempool.store.txs_by_id.get(next_tx_id).unwrap().fees_with_descendants; + log::debug!("entry_score: {:?}", entry_score); + log::debug!("next_entry_score: {:?}", next_entry_score); + assert!(entry_score <= next_entry_score) + } + } } From aa3e2afc612451318f0dba4b600bdc7e92ce0314 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Mon, 4 Apr 2022 14:33:21 +0700 Subject: [PATCH 102/153] TESTS: add test try_replace_irreplaceable --- mempool/src/pool.rs | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 974e185da8..4653f0b53b 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -1732,6 +1732,47 @@ mod tests { Ok(()) } + #[test] + fn try_replace_irreplaceable() -> anyhow::Result<()> { + let mut mempool = setup(); + let outpoint = mempool + .available_outpoints(true) + .iter() + .next() + .expect("there should be an outpoint since setup creates the genesis transaction") + .outpoint + .to_owned(); + + let outpoint_source_id = OutPointSourceId::from( + outpoint.get_tx_id().get_tx_id().expect("Not Coinbase").to_owned(), + ); + + let input = TxInput::new(outpoint_source_id, 0, DUMMY_WITNESS_MSG.to_vec()); + let flags = 0; + let locktime = 0; + let original_fee = Amount::from(get_relay_fee_from_tx_size(TX_SPEND_INPUT_SIZE)); + let original = tx_spend_input(&mempool, input.clone(), original_fee, flags, locktime) + .expect("should be able to spend here"); + let original_id = original.get_id(); + mempool.add_transaction(original)?; + + let flags = 0; + let replacement_fee = (original_fee + 1000.into()).unwrap(); + let replacement = tx_spend_input(&mempool, input, replacement_fee, flags, locktime) + .expect("should be able to spend here"); + assert!(matches!( + mempool.add_transaction(replacement.clone()), + Err(Error::TxValidationError( + TxValidationError::ConflictWithIrreplaceableTransaction + )) + )); + + mempool.drop_transaction(&original_id); + mempool.add_transaction(replacement)?; + + Ok(()) + } + #[test] fn tx_replace() -> anyhow::Result<()> { let relay_fee = get_relay_fee_from_tx_size(TX_SPEND_INPUT_SIZE); From 7d1434cce6aa59dd9b8f12230996b5c944f9341d Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Tue, 5 Apr 2022 11:59:26 +0700 Subject: [PATCH 103/153] Refactor finalize_tx --- mempool/src/pool.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 4653f0b53b..7d5b174bc6 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -838,8 +838,7 @@ where Ok(replacements_with_descendants) } - fn finalize_tx(&mut self, tx: Transaction, conflicts: Conflicts) -> Result<(), Error> { - self.store.drop_conflicts(conflicts); + fn finalize_tx(&mut self, tx: Transaction) -> Result<(), Error> { let entry = self.create_entry(tx)?; let id = entry.tx.get_id().get(); self.store.add_tx(entry)?; @@ -988,7 +987,8 @@ where fn add_transaction(&mut self, tx: Transaction) -> Result<(), Error> { let conflicts = self.validate_transaction(&tx)?; - self.finalize_tx(tx, conflicts)?; + self.store.drop_conflicts(conflicts); + self.finalize_tx(tx)?; Ok(()) } From e8cb3c47d11e2b3f9bd34f2bc90a70d193f94190 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Tue, 5 Apr 2022 11:53:16 +0700 Subject: [PATCH 104/153] Proper error for descendant of expired transaction --- mempool/src/error.rs | 2 ++ mempool/src/pool.rs | 8 +++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/mempool/src/error.rs b/mempool/src/error.rs index 5bf76207c1..a1e569d052 100644 --- a/mempool/src/error.rs +++ b/mempool/src/error.rs @@ -68,6 +68,8 @@ pub enum TxValidationError { FeeRateError, #[error("AncestorFeeUpdateOverflow")] AncestorFeeUpdateOverflow, + #[error("Descendant of expired transaction")] + DescendantOfExpiredTransaction, #[error("Internal Error")] InternalError, } diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 7d5b174bc6..b195a60594 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -842,12 +842,18 @@ where let entry = self.create_entry(tx)?; let id = entry.tx.get_id().get(); self.store.add_tx(entry)?; + self.remove_expired_transactions(); + self.store + .txs_by_id + .contains_key(&id) + .then(|| ()) + .ok_or(TxValidationError::DescendantOfExpiredTransaction)?; + self.limit_mempool_size()?; self.store.txs_by_id.contains_key(&id).then(|| ()).ok_or(Error::MempoolFull) } fn limit_mempool_size(&mut self) -> Result<(), Error> { - self.remove_expired_transactions(); let removed_fees = self.trim()?; if !removed_fees.is_empty() { let new_minimum_fee_rate = From e14393674d213ec359782ab9cf6420a6b9169131 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Tue, 5 Apr 2022 12:12:22 +0700 Subject: [PATCH 105/153] TESTS: descendant of expired entry --- mempool/src/pool.rs | 47 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index b195a60594..16a50a4d5a 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -2648,4 +2648,51 @@ mod tests { assert!(entry_score <= next_entry_score) } } + + #[test] + fn descendant_of_expired_entry() -> anyhow::Result<()> { + let mock_clock = MockClock::new(); + logging::try_init_logging::<&str>(None); + + let mut mempool = MempoolImpl::create( + ChainStateMock::new(), + mock_clock.clone(), + SystemUsageEstimator {}, + ); + + let num_inputs = 1; + let num_outputs = 2; + let fee = get_relay_fee_from_tx_size(estimate_tx_size(num_inputs, num_outputs)); + let parent = TxGenerator::new() + .with_num_inputs(num_inputs) + .with_num_outputs(num_outputs) + .with_fee(fee.into()) + .generate_tx(&mempool)?; + let parent_id = parent.get_id(); + mempool.add_transaction(parent)?; + + let flags = 0; + let locktime = 0; + let outpoint_source_id = OutPointSourceId::Transaction(parent_id.clone()); + let child = tx_spend_input( + &mempool, + TxInput::new(outpoint_source_id, 0, DUMMY_WITNESS_MSG.to_vec()), + None, + flags, + locktime, + )?; + let child_id = child.get_id(); + mock_clock.set(DEFAULT_MEMPOOL_EXPIRY + 1); + + assert!(matches!( + mempool.add_transaction(child), + Err(Error::TxValidationError( + TxValidationError::DescendantOfExpiredTransaction + )) + )); + + assert!(!mempool.contains_transaction(&parent_id)); + assert!(!mempool.contains_transaction(&child_id)); + Ok(()) + } } From 027d82c821be38e3cda71e9f1b3d9a0a844894b0 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Tue, 5 Apr 2022 12:18:48 +0700 Subject: [PATCH 106/153] TESTS: mempool full --- mempool/src/pool.rs | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 16a50a4d5a..de07bc992a 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -2695,4 +2695,25 @@ mod tests { assert!(!mempool.contains_transaction(&child_id)); Ok(()) } + + #[test] + fn mempool_full() -> anyhow::Result<()> { + logging::try_init_logging::<&str>(None); + let mut mock_usage = MockGetMemoryUsage::new(); + mock_usage + .expect_get_memory_usage() + .times(1) + .return_const(MAX_MEMPOOL_SIZE_BYTES + 1); + + let chain_state = ChainStateMock::new(); + let mut mempool = MempoolImpl::create(chain_state, SystemClock, mock_usage); + + let tx = TxGenerator::new().generate_tx(&mempool)?; + log::debug!("mempool_full: tx has is {}", tx.get_id().get()); + assert!(matches!( + mempool.add_transaction(tx), + Err(Error::MempoolFull) + )); + Ok(()) + } } From 7961096bf0670261b1d896e2699ec45d0fb3f495 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Tue, 5 Apr 2022 13:15:16 +0700 Subject: [PATCH 107/153] TESTS: no empty bags in descendant score index --- mempool/src/pool.rs | 49 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index de07bc992a..04d8d330bb 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -2716,4 +2716,53 @@ mod tests { )); Ok(()) } + + #[test] + fn no_empty_bags_in_descendant_score_index() -> anyhow::Result<()> { + let mut mempool = setup(); + + let num_inputs = 1; + let num_outputs = 100; + let fee = get_relay_fee_from_tx_size(estimate_tx_size(num_inputs, num_outputs)); + let parent = TxGenerator::new() + .with_num_inputs(num_inputs) + .with_num_outputs(num_outputs) + .with_fee(fee.into()) + .generate_tx(&mempool)?; + let parent_id = parent.get_id(); + + let outpoint_source_id = OutPointSourceId::Transaction(parent.get_id()); + mempool.add_transaction(parent)?; + let num_child_txs = num_outputs; + let flags = 0; + let locktime = 0; + let txs = (0..num_child_txs) + .into_iter() + .map(|i| { + tx_spend_input( + &mempool, + TxInput::new( + outpoint_source_id.clone(), + u32::try_from(i).unwrap(), + DUMMY_WITNESS_MSG.to_vec(), + ), + Amount::from(fee + u128::try_from(i).unwrap()), + flags, + locktime, + ) + }) + .collect::, _>>()?; + let ids = txs.iter().map(|tx| tx.get_id()).collect::>(); + + for tx in txs { + mempool.add_transaction(tx)?; + } + + mempool.drop_transaction(&parent_id); + for id in ids { + mempool.drop_transaction(&id) + } + assert!(mempool.store.txs_by_descendant_score.is_empty()); + Ok(()) + } } From 6136347f691294dc437c7acd66f8a964b3ac695e Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Tue, 5 Apr 2022 13:15:31 +0700 Subject: [PATCH 108/153] Mempool: Documentation --- mempool/src/pool.rs | 126 ++++++++++++++++++++++++++++++-------- notes/time_lock_notes.txt | 33 ++++++++++ 2 files changed, 135 insertions(+), 24 deletions(-) create mode 100644 notes/time_lock_notes.txt diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 04d8d330bb..9ad9f7a5e9 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -299,9 +299,35 @@ pub struct MempoolImpl { #[derive(Debug)] struct MempoolStore { + // This is the "main" data structure storing Mempool entries. All other structures in the + // MempoolStore contain ids (hashes) of entries, sorted according to some order of interest. txs_by_id: HashMap, + + // Mempool entries sorted by descendant score. + // We keep this index so that when the mempool grows full, we know which transactions are the + // most economically reasonable to evict. When an entry is removed from the mempool for + // fullness reasons, it must be removed together with all of its descendants (as these descendants + // would no longer be valid to mine). Entries with a lower descendant score will be evicted + // first. + // + // FIXME currently, the descendant score is the sum fee of the transaction to gether with all + // of its descendants. If we wish to follow Bitcoin Core, we should use: + // max(feerate(tx, tx_with_descendants)), + // Where feerate is computed as fee(tx)/size(tx) + // Note that if we wish to follow Bitcoin Bore, "size" is not simply the encoded size, but + // rather a value that takes into account witdess and sigop data (see CTxMemPoolEntry::GetTxSize). txs_by_descendant_score: BTreeMap>, + + // Entries that have remained in the mempool for a long time (see DEFAULT_MEMPOOL_EXPIRY) are + // evicted. To efficiently know which entries to evict, we store the mempool entries sorted by + // their creation time, from earliest to latest. txs_by_creation_time: BTreeMap>, + + // TODO add txs_by_ancestor_score index, which will be used by the block production subsystem + // to select the best transactions for the next block + // + // We keep the information of which outpoints are spent by entries currently in the mempool. + // This allows us to recognize conflicts (double-spends) and handle them spender_txs: BTreeMap, } @@ -640,6 +666,46 @@ where } fn validate_transaction(&self, tx: &Transaction) -> Result { + // This validation function is based on Bitcoin Core's MemPoolAccept::PreChecks. + // However, as of this stage it does not cover everything covered in Bitcoin Core + // + // Currently, the items we want covered which are NOT yet covered are: + // + // - Checking if a transaction is "standard" (see `IsStandardTx`, `AreInputsStandard` in Bitcoin Core). We have yet to decide on Mintlayer's + // definition of "standard" + // + // - Time locks: Briefly, the corresponding functions in Bitcoin Core are `CheckFinalTx` and + // `CheckSequenceLocks`. See mempool/src/time_lock_notes.txt for more details on our + // brainstorming on this topic thus far. + // + // - Bitcoin Core does not relay transactions smaller than 82 bytes (see + // MIN_STANDARD_TX_NONWITNESS_SIZE in Bitcoin Core's policy.h) + // + // - Checking that coinbase inputs have matured + // + // - Checking the signature operations cost (Bitcoin Core: `GetTransactionSigOpCost`) + // + // - We have yet to understand and implement this comment pertaining to chain limits and + // calculation of in-mempool ancestors: + // https://github.com/bitcoin/bitcoin/blob/7c08d81e119570792648fe95bbacddbb1d5f9ae2/src/validation.cpp#L829 + // + // - Bitcoin Core's `EntriesAndTxidsDisjoint` check + // + // - Bitcoin Core's `PolicyScriptChecks` + // + // - Bitcoin Core's `ConsensusScriptChecks` + + // Deviations from Bitcoin Core + // + // - In our FeeRate calculations, we use the `encoded_size`of a transaction. In contrast, + // see Bitcoin Core's `CTxMemPoolEntry::GetTxSize()`, which takes into account sigops cost + // and witness data. This deviation is not intentional, but rather the result of wanting to + // get a basic implementation working. TODO: weigh what notion of size/weight we wish to + // use and whether to follow Bitcoin Core in this regard + // + // We don't have a notion of MAX_MONEY like Bitcoin Core does, so we also don't have a + // `MoneyRange` check like the one found in Bitcoin Core's `CheckTransaction` + if tx.inputs().is_empty() { return Err(TxValidationError::NoInputs); } @@ -648,9 +714,6 @@ where return Err(TxValidationError::NoOutputs); } - // TODO consier a MAX_MONEY check reminiscent of bitcoin's - // TODO consider rejecting non-standard transactions (for some definition of standard) - let outpoints = tx.inputs().iter().map(|input| input.outpoint()).cloned(); if has_duplicate_entry(outpoints) { @@ -1747,23 +1810,26 @@ mod tests { .next() .expect("there should be an outpoint since setup creates the genesis transaction") .outpoint - .to_owned(); + .clone(); - let outpoint_source_id = OutPointSourceId::from( - outpoint.get_tx_id().get_tx_id().expect("Not Coinbase").to_owned(), - ); + let outpoint_source_id = + OutPointSourceId::from(*outpoint.tx_id().get_tx_id().expect("Not Coinbase")); - let input = TxInput::new(outpoint_source_id, 0, DUMMY_WITNESS_MSG.to_vec()); + let input = TxInput::new( + outpoint_source_id, + 0, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ); let flags = 0; let locktime = 0; - let original_fee = Amount::from(get_relay_fee_from_tx_size(TX_SPEND_INPUT_SIZE)); + let original_fee = Amount::from_atoms(get_relay_fee_from_tx_size(TX_SPEND_INPUT_SIZE)); let original = tx_spend_input(&mempool, input.clone(), original_fee, flags, locktime) .expect("should be able to spend here"); let original_id = original.get_id(); mempool.add_transaction(original)?; let flags = 0; - let replacement_fee = (original_fee + 1000.into()).unwrap(); + let replacement_fee = (original_fee + Amount::from_atoms(1000)).unwrap(); let replacement = tx_spend_input(&mempool, input, replacement_fee, flags, locktime) .expect("should be able to spend here"); assert!(matches!( @@ -2560,12 +2626,16 @@ mod tests { let flags = 0; let locktime = 0; - let tx_b_fee = Amount::from(get_relay_fee_from_tx_size(estimate_tx_size(1, 2))); - let tx_a_fee = (tx_b_fee + Amount::from(1000)).unwrap(); - let tx_c_fee = (tx_a_fee + Amount::from(1000)).unwrap(); + let tx_b_fee = Amount::from_atoms(get_relay_fee_from_tx_size(estimate_tx_size(1, 2))); + let tx_a_fee = (tx_b_fee + Amount::from_atoms(1000)).unwrap(); + let tx_c_fee = (tx_a_fee + Amount::from_atoms(1000)).unwrap(); let tx_a = tx_spend_input( &mempool, - TxInput::new(outpoint_source_id.clone(), 0, DUMMY_WITNESS_MSG.to_vec()), + TxInput::new( + outpoint_source_id.clone(), + 0, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ), tx_a_fee, flags, locktime, @@ -2577,7 +2647,11 @@ mod tests { let tx_b = tx_spend_input( &mempool, - TxInput::new(outpoint_source_id, 1, DUMMY_WITNESS_MSG.to_vec()), + TxInput::new( + outpoint_source_id, + 1, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ), tx_b_fee, flags, locktime, @@ -2590,9 +2664,9 @@ mod tests { let tx_c = tx_spend_input( &mempool, TxInput::new( - OutPointSourceId::Transaction(tx_b_id.clone()), + OutPointSourceId::Transaction(tx_b_id), 0, - DUMMY_WITNESS_MSG.to_vec(), + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), ), tx_c_fee, flags, @@ -2666,23 +2740,27 @@ mod tests { let parent = TxGenerator::new() .with_num_inputs(num_inputs) .with_num_outputs(num_outputs) - .with_fee(fee.into()) + .with_fee(Amount::from_atoms(fee)) .generate_tx(&mempool)?; let parent_id = parent.get_id(); mempool.add_transaction(parent)?; let flags = 0; let locktime = 0; - let outpoint_source_id = OutPointSourceId::Transaction(parent_id.clone()); + let outpoint_source_id = OutPointSourceId::Transaction(parent_id); let child = tx_spend_input( &mempool, - TxInput::new(outpoint_source_id, 0, DUMMY_WITNESS_MSG.to_vec()), + TxInput::new( + outpoint_source_id, + 0, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ), None, flags, locktime, )?; let child_id = child.get_id(); - mock_clock.set(DEFAULT_MEMPOOL_EXPIRY + 1); + mock_clock.set(DEFAULT_MEMPOOL_EXPIRY + Duration::new(1, 0)); assert!(matches!( mempool.add_transaction(child), @@ -2727,7 +2805,7 @@ mod tests { let parent = TxGenerator::new() .with_num_inputs(num_inputs) .with_num_outputs(num_outputs) - .with_fee(fee.into()) + .with_fee(Amount::from_atoms(fee)) .generate_tx(&mempool)?; let parent_id = parent.get_id(); @@ -2744,9 +2822,9 @@ mod tests { TxInput::new( outpoint_source_id.clone(), u32::try_from(i).unwrap(), - DUMMY_WITNESS_MSG.to_vec(), + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), ), - Amount::from(fee + u128::try_from(i).unwrap()), + Amount::from_atoms(fee + u128::try_from(i).unwrap()), flags, locktime, ) diff --git a/notes/time_lock_notes.txt b/notes/time_lock_notes.txt new file mode 100644 index 0000000000..103161ec9b --- /dev/null +++ b/notes/time_lock_notes.txt @@ -0,0 +1,33 @@ +TIMELOCK NOTES + +1. There are two kinds of timelocks, absolute and relative. Absolute timelocks prevent transmitting a tx over the network or spending a utxo before a specific point in time (measured either in Unix Epoch Time or block height). Relative timelocks prevent spending an outputs before it has aged by a certain amount of time. + +2. Absolute timelocks: + +* Transaction-level timelocks +In Bitcoin Core transactions have the nLocktime field. When a transaction is validated, if nLocktime is less than the current time, it won't be relayed to the network. +Notes: + In this way Alice can create a transaction in which she sends "postdated" funds to Bob, send Bob that transaction, and now Bob can submit the transaction when enough time has passed (before that it will just be rejected). + +* UTXO-level timelocks +There is an opcode, CHECKLOCKTIMEVERIFY. When a transaction is submitted, for every output being spent a script is run (locking, unlocking). CLTV can appear in the script and make it fail if the locktime parameter in the script is greater than the locktime of the transaction. +So CLTV actually RELIES ON nLocktime. + +3. Relative Timelocks +- the nSequence field was originally intended for something else +NOTE: need to check the whole deal about it being used as a way to signal if CLTV applied. +- Transaction validation fails unless all inputs have sufficiently aged (each according to its timelock type? need to check). +- CHECKSEQUENCEVERIFY opcode. The use of this opcode makes reference to nSequence, need to see how they work together. + +The transaction structure will use an +enum Locktime { + BlockHeight(RangedInt<0, 500,000,000>), + TimeStamp(UnixEpoch(Ranged)) +} + +We will need some conversions. +QUESTION: need to map out the conversions. + +QUESTIONS: + Do we want to implement absolute tx level timelocks the exact same way Bitcoin Core implements them? + Do we want to do things differently with relative timelocks since nSequence is kind of a mess? From b564d87f40db4b60d110112a7bcd91680164e828 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Mon, 11 Apr 2022 15:55:45 +0700 Subject: [PATCH 109/153] Improve style in ancestor and descendant functions --- mempool/src/pool.rs | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 9ad9f7a5e9..d460ceb987 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -193,10 +193,7 @@ impl TxMempoolEntry { fn unconfirmed_ancestors_inner(&self, visited: &mut Ancestors, store: &MempoolStore) { for parent in self.parents.iter() { - if visited.0.contains(parent) { - continue; - } else { - visited.0.insert(*parent); + if visited.insert(*parent) { store .get_entry(parent) .expect("entry") @@ -213,10 +210,7 @@ impl TxMempoolEntry { fn unconfirmed_descendants_inner(&self, visited: &mut Descendants, store: &MempoolStore) { for child in self.children.iter() { - if visited.0.contains(child) { - continue; - } else { - visited.0.insert(*child); + if visited.insert(*child) { store .get_entry(child) .expect("entry") From 2cab6cee102e6d91fc8a27a93c1b641fc70b3177 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Mon, 11 Apr 2022 19:14:47 +0700 Subject: [PATCH 110/153] Fix FeeRate calculations to consistently use tokens/kb and update rolling_fee test accordingly --- mempool/src/feerate.rs | 30 ++++++++++------ mempool/src/pool.rs | 78 +++++++++++++++++++++++++++++++++--------- 2 files changed, 81 insertions(+), 27 deletions(-) diff --git a/mempool/src/feerate.rs b/mempool/src/feerate.rs index 6767565a42..4ad416868d 100644 --- a/mempool/src/feerate.rs +++ b/mempool/src/feerate.rs @@ -4,29 +4,33 @@ use crate::error::TxValidationError; lazy_static::lazy_static! { pub(crate) static ref INCREMENTAL_RELAY_FEE_RATE: FeeRate = FeeRate::new(Amount::from_atoms(1000)); + pub(crate) static ref INCREMENTAL_RELAY_THRESHOLD: FeeRate = FeeRate::new(Amount::from_atoms(500)); } #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub(crate) struct FeeRate { - tokens_per_byte: Amount, + tokens_per_kb: Amount, } impl FeeRate { - pub(crate) fn new(tokens_per_byte: Amount) -> Self { - Self { tokens_per_byte } + pub(crate) fn new(tokens_per_kb: Amount) -> Self { + Self { tokens_per_kb } } pub(crate) fn of_tx(fee: Amount, tx_size: usize) -> Result { let tokens_per_byte = Self::div_up(fee, tx_size)?; - Ok(Self { tokens_per_byte }) + Ok(Self { + tokens_per_kb: (tokens_per_byte * 1000).unwrap(), + }) } pub(crate) fn compute_fee(&self, size: usize) -> Result { - (self.tokens_per_byte * size as u128).ok_or(TxValidationError::FeeRateError) + ((self.tokens_per_kb * u128::try_from(size).unwrap()).unwrap() / 1000) + .ok_or(TxValidationError::FeeRateError) } - pub(crate) fn tokens_per_byte(&self) -> Amount { - self.tokens_per_byte + pub(crate) fn tokens_per_kb(&self) -> Amount { + self.tokens_per_kb } fn div_up(fee: Amount, tx_size: usize) -> Result { @@ -42,15 +46,19 @@ impl FeeRate { impl std::ops::Add for FeeRate { type Output = Option; fn add(self, other: Self) -> Self::Output { - let tokens_per_byte = self.tokens_per_byte + other.tokens_per_byte; - tokens_per_byte.map(|tokens_per_byte| FeeRate { tokens_per_byte }) + let tokens_per_byte = self.tokens_per_kb + other.tokens_per_kb; + tokens_per_byte.map(|tokens_per_byte| FeeRate { + tokens_per_kb: tokens_per_byte, + }) } } impl std::ops::Div for FeeRate { type Output = Option; fn div(self, other: Self) -> Self::Output { - let tokens_per_byte = self.tokens_per_byte / other.tokens_per_byte.into(); - tokens_per_byte.map(|tokens_per_byte| FeeRate { tokens_per_byte }) + let tokens_per_byte = self.tokens_per_kb / other.tokens_per_kb.into(); + tokens_per_byte.map(|tokens_per_byte| FeeRate { + tokens_per_kb: tokens_per_byte, + }) } } diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index d460ceb987..bbdb770bc7 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -25,6 +25,7 @@ use crate::error::Error; use crate::error::TxValidationError; use crate::feerate::FeeRate; use crate::feerate::INCREMENTAL_RELAY_FEE_RATE; +use crate::feerate::INCREMENTAL_RELAY_THRESHOLD; const ROLLING_FEE_BASE_HALFLIFE: Time = Duration::new(60 * 60 * 12, 1); // TODO this willbe defined elsewhere (some of limits.rs file) @@ -259,7 +260,7 @@ impl RollingFeeRate { / (halflife.as_secs() as f64), ); self.rolling_minimum_fee_rate = - FeeRate::new(self.rolling_minimum_fee_rate.tokens_per_byte().div_by_float(divisor)); + FeeRate::new(self.rolling_minimum_fee_rate.tokens_per_kb().div_by_float(divisor)); log::trace!( "decay_fee: new fee rate: {:?}", @@ -585,10 +586,7 @@ where self.rolling_fee_rate ); - if self.rolling_fee_rate.get().rolling_minimum_fee_rate - < (*INCREMENTAL_RELAY_FEE_RATE / FeeRate::new(Amount::from_atoms(2))) - .expect("not division by zero") - { + if self.rolling_fee_rate.get().rolling_minimum_fee_rate < *INCREMENTAL_RELAY_THRESHOLD { log::trace!("rolling fee rate {:?} less than half of the incremental fee rate, dropping the fee", self.rolling_fee_rate.get().rolling_minimum_fee_rate); self.drop_rolling_fee(); return Ok(self.rolling_fee_rate.get().rolling_minimum_fee_rate); @@ -654,9 +652,14 @@ where ) -> Result { let minimum_fee_rate = self.get_update_min_fee_rate()?; log::debug!("minimum fee rate {:?}", minimum_fee_rate); - log::debug!("tx_size: {:?}", tx.encoded_size()); - - minimum_fee_rate.compute_fee(tx.encoded_size()) + log::debug!( + "tx_size: {:?}, tx_fee {:?}", + tx.encoded_size(), + self.try_get_fee(tx)? + ); + let res = minimum_fee_rate.compute_fee(tx.encoded_size()); + log::debug!("minimum_mempool_fee for tx: {:?}", res); + res } fn validate_transaction(&self, tx: &Transaction) -> Result { @@ -2494,6 +2497,11 @@ mod tests { assert!(mempool.contains_transaction(&child_1_id)); assert!(!mempool.contains_transaction(&child_0_id)); let rolling_fee = mempool.get_minimum_rolling_fee(); + log::debug!("FeeRate of child_0 {:?}", mempool.try_get_fee(&child_0)?); + log::debug!( + "minimum rolling fee after child_0's eviction {:?}", + rolling_fee + ); assert_eq!( rolling_fee, (FeeRate::of_tx(mempool.try_get_fee(&child_0)?, child_0.encoded_size())? @@ -2514,9 +2522,16 @@ mod tests { flags, locktime, )?; - log::debug!("before child2"); + log::debug!( + "before child2: fee = {:?}, size = {}, minimum fee rate = {:?}", + mempool.try_get_fee(&child_2)?, + child_2.encoded_size(), + mempool.get_minimum_rolling_fee() + ); + let res = mempool.add_transaction(child_2); + log::debug!("result of adding child2 {:?}", res); assert!(matches!( - mempool.add_transaction(child_2), + res, Err(Error::TxValidationError( TxValidationError::RollingFeeThresholdNotMet { .. } )) @@ -2545,12 +2560,22 @@ mod tests { // Since memory usage is now zero, it is less than 1/4 of the max size // and ROLLING_FEE_BASE_HALFLIFE / 4 is the time it will take for the fee to halve + // We are going to submit dummy txs to the mempool incrementing time by this halflife + // between txs. Finally, when the fee rate falls under INCREMENTAL_RELAY_THRESHOLD, we + // observer that it is set to zero let halflife = ROLLING_FEE_BASE_HALFLIFE / 4; mock_clock.increment(halflife); - let dummy_tx = TxGenerator::new().generate_tx(&mempool)?; - log::debug!("first attempt to add dummy"); + let dummy_tx = + TxGenerator::new().with_fee(Amount::from_atoms(100)).generate_tx(&mempool)?; + log::debug!( + "First attempt to add dummy which pays a fee of {:?}", + mempool.try_get_fee(&dummy_tx)? + ); + let res = mempool.add_transaction(dummy_tx.clone()); + + log::debug!("Result of first attempt to add dummy: {:?}", res); assert!(matches!( - mempool.add_transaction(dummy_tx.clone()), + res, Err(Error::TxValidationError( TxValidationError::RollingFeeThresholdNotMet { .. } )) @@ -2561,10 +2586,31 @@ mod tests { ); mock_clock.increment(halflife); - // Fee will have dropped under INCREMENTAL_RELAY_FEE_RATE / 2 by now, so it will be set to - // zero and our tx will be submitted successfully - log::debug!("second attempt to add dummy"); + log::debug!("Second attempt to add dummy"); mempool.add_transaction(dummy_tx)?; + assert_eq!( + mempool.get_minimum_rolling_fee(), + (rolling_fee / FeeRate::new(Amount::from_atoms(4))).unwrap() + ); + log::debug!( + "After successful addition of dummy, rolling fee rate is {:?}", + mempool.get_minimum_rolling_fee() + ); + + // Add more dummies until rolling feerate drops to zero + mock_clock.increment(halflife); + let another_dummy = + TxGenerator::new().with_fee(Amount::from_atoms(100)).generate_tx(&mempool)?; + mempool.add_transaction(another_dummy)?; + assert_eq!( + mempool.get_minimum_rolling_fee(), + (rolling_fee / FeeRate::new(Amount::from_atoms(8))).unwrap() + ); + + mock_clock.increment(halflife); + let final_dummy = + TxGenerator::new().with_fee(Amount::from_atoms(100)).generate_tx(&mempool)?; + mempool.add_transaction(final_dummy)?; assert_eq!( mempool.get_minimum_rolling_fee(), FeeRate::new(Amount::from_atoms(0)) From c6f2cd56240881d9f78e004488688ef84cb42b34 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Tue, 12 Apr 2022 16:19:00 +0700 Subject: [PATCH 111/153] Remove Amount::random, which is no longer needed --- common/Cargo.toml | 1 - common/src/primitives/amount.rs | 5 ----- mempool/src/pool.rs | 4 ++-- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/common/Cargo.toml b/common/Cargo.toml index 16d7a047a4..4786214592 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -23,7 +23,6 @@ sscanf = "0.2" static_assertions = "1.1" thiserror = "1.0" hex = "0.4" -rand = "0.8.5" # for fixed_hash arbitrary = "1.1" diff --git a/common/src/primitives/amount.rs b/common/src/primitives/amount.rs index cdc1e7d129..68d1aa046a 100644 --- a/common/src/primitives/amount.rs +++ b/common/src/primitives/amount.rs @@ -15,7 +15,6 @@ #![allow(clippy::eq_op)] -use rand::Rng; use serialization::{Decode, Encode}; use std::iter::Sum; @@ -131,10 +130,6 @@ impl Amount { } } - pub fn random(range: std::ops::RangeInclusive) -> Amount { - Amount::from_atoms(rand::thread_rng().gen_range(range.start().val..=range.end().val)) - } - // TODO this looks risky, consult Ben/Sam pub fn div_by_float(self, divisor: f64) -> Self { Self { diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index bbdb770bc7..a0ac02041d 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -2766,7 +2766,7 @@ mod tests { #[test] fn descendant_of_expired_entry() -> anyhow::Result<()> { let mock_clock = MockClock::new(); - logging::try_init_logging::<&str>(None); + logging::init_logging::<&str>(None); let mut mempool = MempoolImpl::create( ChainStateMock::new(), @@ -2816,7 +2816,7 @@ mod tests { #[test] fn mempool_full() -> anyhow::Result<()> { - logging::try_init_logging::<&str>(None); + logging::init_logging::<&str>(None); let mut mock_usage = MockGetMemoryUsage::new(); mock_usage .expect_get_memory_usage() From b4f6dc2197eca190849ba836db2239c168c8b84f Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Tue, 12 Apr 2022 16:28:09 +0700 Subject: [PATCH 112/153] TESTS Remove time measurements from different_size_txs --- mempool/src/pool.rs | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index a0ac02041d..43182c243d 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -2620,8 +2620,6 @@ mod tests { #[test] fn different_size_txs() -> anyhow::Result<()> { - use std::time::Duration; - use std::time::Instant; let mut mempool = setup(); let initial_tx = TxGenerator::new() .with_num_inputs(1) @@ -2630,24 +2628,16 @@ mod tests { mempool.add_transaction(initial_tx)?; let target_txs = 100; - let mut time_processing_txs = Duration::from_millis(0); - let mut time_creating_txs = Duration::from_millis(0); for i in 0..target_txs { let num_inputs = i + 1; let num_outputs = i + 1; - let before_creating = Instant::now(); let tx = TxGenerator::new() .with_num_inputs(num_inputs) .with_num_outputs(num_outputs) .generate_tx(&mempool)?; - time_creating_txs += before_creating.elapsed(); - let before_processing = Instant::now(); mempool.add_transaction(tx)?; - time_processing_txs += before_processing.elapsed() } - log::info!("Total time spent processing: {:?}", time_processing_txs); - log::info!("Total time spent creating: {:?}", time_creating_txs); Ok(()) } From 61c9daa7fb2fb622b758d2ecbd350efd1336d422 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Wed, 13 Apr 2022 14:11:45 +0700 Subject: [PATCH 113/153] Use u128 as FeeRate's underlying type --- common/src/primitives/amount.rs | 7 --- mempool/src/feerate.rs | 58 ++++++++++------------- mempool/src/pool.rs | 82 +++++++++++++-------------------- 3 files changed, 56 insertions(+), 91 deletions(-) diff --git a/common/src/primitives/amount.rs b/common/src/primitives/amount.rs index 68d1aa046a..b1af89c6d2 100644 --- a/common/src/primitives/amount.rs +++ b/common/src/primitives/amount.rs @@ -129,13 +129,6 @@ impl Amount { atoms_str.parse::().ok().map(|v| Amount { val: v }) } } - - // TODO this looks risky, consult Ben/Sam - pub fn div_by_float(self, divisor: f64) -> Self { - Self { - val: (self.val as f64 / divisor) as u128, - } - } } impl From for u128 { diff --git a/mempool/src/feerate.rs b/mempool/src/feerate.rs index 4ad416868d..6978cacbae 100644 --- a/mempool/src/feerate.rs +++ b/mempool/src/feerate.rs @@ -1,64 +1,54 @@ use common::primitives::amount::Amount; -use crate::error::TxValidationError; - lazy_static::lazy_static! { - pub(crate) static ref INCREMENTAL_RELAY_FEE_RATE: FeeRate = FeeRate::new(Amount::from_atoms(1000)); - pub(crate) static ref INCREMENTAL_RELAY_THRESHOLD: FeeRate = FeeRate::new(Amount::from_atoms(500)); + pub(crate) static ref INCREMENTAL_RELAY_FEE_RATE: FeeRate = FeeRate::new(1000); + pub(crate) static ref INCREMENTAL_RELAY_THRESHOLD: FeeRate = FeeRate::new(500); } #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub(crate) struct FeeRate { - tokens_per_kb: Amount, + tokens_per_kb: u128, } impl FeeRate { - pub(crate) fn new(tokens_per_kb: Amount) -> Self { + pub(crate) fn new(tokens_per_kb: u128) -> Self { Self { tokens_per_kb } } - pub(crate) fn of_tx(fee: Amount, tx_size: usize) -> Result { - let tokens_per_byte = Self::div_up(fee, tx_size)?; - Ok(Self { - tokens_per_kb: (tokens_per_byte * 1000).unwrap(), - }) + pub(crate) fn of_tx(fee: Amount, tx_size: usize) -> Self { + let tokens_per_byte = Self::div_up(fee.into(), tx_size); + Self { + tokens_per_kb: tokens_per_byte * 1000, + } } - pub(crate) fn compute_fee(&self, size: usize) -> Result { - ((self.tokens_per_kb * u128::try_from(size).unwrap()).unwrap() / 1000) - .ok_or(TxValidationError::FeeRateError) + pub(crate) fn compute_fee(&self, size: usize) -> Amount { + Amount::from_atoms(self.tokens_per_kb * u128::try_from(size).unwrap() / 1000) } - pub(crate) fn tokens_per_kb(&self) -> Amount { + pub(crate) fn tokens_per_kb(&self) -> u128 { self.tokens_per_kb } - fn div_up(fee: Amount, tx_size: usize) -> Result { - let tx_size = tx_size as u128; - (((fee + Amount::from_atoms(tx_size)).ok_or(TxValidationError::FeeRateError)? - - Amount::from_atoms(1)) - .ok_or(TxValidationError::FeeRateError)? - / tx_size) - .ok_or(TxValidationError::FeeRateError) + fn div_up(fee: u128, tx_size: usize) -> u128 { + let tx_size = u128::try_from(tx_size).unwrap(); + (fee + tx_size - 1) / tx_size } } impl std::ops::Add for FeeRate { - type Output = Option; + type Output = FeeRate; fn add(self, other: Self) -> Self::Output { - let tokens_per_byte = self.tokens_per_kb + other.tokens_per_kb; - tokens_per_byte.map(|tokens_per_byte| FeeRate { - tokens_per_kb: tokens_per_byte, - }) + let tokens_per_kb = self.tokens_per_kb + other.tokens_per_kb; + FeeRate { tokens_per_kb } } } -impl std::ops::Div for FeeRate { - type Output = Option; - fn div(self, other: Self) -> Self::Output { - let tokens_per_byte = self.tokens_per_kb / other.tokens_per_kb.into(); - tokens_per_byte.map(|tokens_per_byte| FeeRate { - tokens_per_kb: tokens_per_byte, - }) +impl std::ops::Div for FeeRate { + type Output = FeeRate; + fn div(self, rhs: u128) -> Self::Output { + FeeRate { + tokens_per_kb: self.tokens_per_kb / rhs, + } } } diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 43182c243d..ff838ef25d 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -246,7 +246,7 @@ struct RollingFeeRate { } impl RollingFeeRate { - fn decay_fee(mut self, halflife: Time, current_time: Time) -> Result { + fn decay_fee(mut self, halflife: Time, current_time: Time) -> Self { log::trace!( "decay_fee: old fee rate: {:?}\nCurrent time: {:?}\nLast Rolling Fee Update: {:?}\nHalflife: {:?}", self.rolling_minimum_fee_rate, @@ -260,14 +260,14 @@ impl RollingFeeRate { / (halflife.as_secs() as f64), ); self.rolling_minimum_fee_rate = - FeeRate::new(self.rolling_minimum_fee_rate.tokens_per_kb().div_by_float(divisor)); + FeeRate::new((self.rolling_minimum_fee_rate.tokens_per_kb() as f64 / divisor) as u128); log::trace!( "decay_fee: new fee rate: {:?}", self.rolling_minimum_fee_rate ); self.last_rolling_fee_update = current_time; - Ok(self) + self } } @@ -275,7 +275,7 @@ impl RollingFeeRate { pub(crate) fn new(creation_time: Time) -> Self { Self { block_since_last_rolling_fee_bump: false, - rolling_minimum_fee_rate: FeeRate::new(Amount::from_atoms(0)), + rolling_minimum_fee_rate: FeeRate::new(0), last_rolling_fee_update: creation_time, } } @@ -570,17 +570,17 @@ where self.rolling_fee_rate.set(rolling_fee_rate) } - pub(crate) fn get_update_min_fee_rate(&self) -> Result { + pub(crate) fn get_update_min_fee_rate(&self) -> FeeRate { let rolling_fee_rate = self.rolling_fee_rate.get(); if !rolling_fee_rate.block_since_last_rolling_fee_bump - || rolling_fee_rate.rolling_minimum_fee_rate == FeeRate::new(Amount::from_atoms(0)) + || rolling_fee_rate.rolling_minimum_fee_rate == FeeRate::new(0) { - return Ok(rolling_fee_rate.rolling_minimum_fee_rate); + return rolling_fee_rate.rolling_minimum_fee_rate; } else if self.clock.get_time() > rolling_fee_rate.last_rolling_fee_update + ROLLING_FEE_DECAY_INTERVAL { // Decay the rolling fee - self.decay_rolling_fee_rate()?; + self.decay_rolling_fee_rate(); log::debug!( "rolling fee rate after decay_rolling_fee_rate {:?}", self.rolling_fee_rate @@ -589,28 +589,26 @@ where if self.rolling_fee_rate.get().rolling_minimum_fee_rate < *INCREMENTAL_RELAY_THRESHOLD { log::trace!("rolling fee rate {:?} less than half of the incremental fee rate, dropping the fee", self.rolling_fee_rate.get().rolling_minimum_fee_rate); self.drop_rolling_fee(); - return Ok(self.rolling_fee_rate.get().rolling_minimum_fee_rate); + return self.rolling_fee_rate.get().rolling_minimum_fee_rate; } } - Ok(std::cmp::max( + std::cmp::max( self.rolling_fee_rate.get().rolling_minimum_fee_rate, *INCREMENTAL_RELAY_FEE_RATE, - )) + ) } fn drop_rolling_fee(&self) { let mut rolling_fee_rate = self.rolling_fee_rate.get(); - rolling_fee_rate.rolling_minimum_fee_rate = FeeRate::new(Amount::from_atoms(0)); + rolling_fee_rate.rolling_minimum_fee_rate = FeeRate::new(0); self.rolling_fee_rate.set(rolling_fee_rate) } - fn decay_rolling_fee_rate(&self) -> Result<(), TxValidationError> { + fn decay_rolling_fee_rate(&self) { let halflife = self.rolling_fee_halflife(); let time = self.clock.get_time(); - self.rolling_fee_rate - .set(self.rolling_fee_rate.get().decay_fee(halflife, time)?); - Ok(()) + self.rolling_fee_rate.set(self.rolling_fee_rate.get().decay_fee(halflife, time)); } fn verify_inputs_available(&self, tx: &Transaction) -> Result<(), TxValidationError> { @@ -646,17 +644,15 @@ where Ok(TxMempoolEntry::new(tx, fee, parents, time)) } - fn get_update_minimum_mempool_fee( - &self, - tx: &Transaction, - ) -> Result { - let minimum_fee_rate = self.get_update_min_fee_rate()?; + fn get_update_minimum_mempool_fee(&self, tx: &Transaction) -> Amount { + let minimum_fee_rate = self.get_update_min_fee_rate(); log::debug!("minimum fee rate {:?}", minimum_fee_rate); - log::debug!( + /*log::debug!( "tx_size: {:?}, tx_fee {:?}", tx.encoded_size(), self.try_get_fee(tx)? ); + */ let res = minimum_fee_rate.compute_fee(tx.encoded_size()); log::debug!("minimum_mempool_fee for tx: {:?}", res); res @@ -738,7 +734,7 @@ where fn pays_minimum_mempool_fee(&self, tx: &Transaction) -> Result<(), TxValidationError> { let tx_fee = self.try_get_fee(tx)?; - let minimum_fee = self.get_update_minimum_mempool_fee(tx)?; + let minimum_fee = self.get_update_minimum_mempool_fee(tx); (tx_fee >= minimum_fee) .then(|| ()) .ok_or(TxValidationError::RollingFeeThresholdNotMet { @@ -914,12 +910,11 @@ where } fn limit_mempool_size(&mut self) -> Result<(), Error> { - let removed_fees = self.trim()?; + let removed_fees = self.trim(); if !removed_fees.is_empty() { let new_minimum_fee_rate = - (*removed_fees.iter().max().expect("removed_fees should not be empty") - + *INCREMENTAL_RELAY_FEE_RATE) - .ok_or(TxValidationError::FeeRateError)?; + *removed_fees.iter().max().expect("removed_fees should not be empty") + + *INCREMENTAL_RELAY_FEE_RATE; if new_minimum_fee_rate > self.rolling_fee_rate.get().rolling_minimum_fee_rate { self.update_min_fee_rate(new_minimum_fee_rate) } @@ -957,7 +952,7 @@ where } } - fn trim(&mut self) -> Result, Error> { + fn trim(&mut self) -> Vec { let mut removed_fees = Vec::new(); while !self.store.is_empty() && self.get_memory_usage() > self.max_size { // TODO sort by descendant score, not by fee @@ -977,10 +972,10 @@ where removed.fees_with_descendants, removed.tx.encoded_size() ); - removed_fees.push(FeeRate::of_tx(removed.fee, removed.tx.encoded_size())?); + removed_fees.push(FeeRate::of_tx(removed.fee, removed.tx.encoded_size())); self.store.drop_tx_and_descendants(removed.tx.get_id()); } - Ok(removed_fees) + removed_fees } } @@ -2504,9 +2499,8 @@ mod tests { ); assert_eq!( rolling_fee, - (FeeRate::of_tx(mempool.try_get_fee(&child_0)?, child_0.encoded_size())? - + *INCREMENTAL_RELAY_FEE_RATE) - .ok_or(TxValidationError::FeeRateError)? + FeeRate::of_tx(mempool.try_get_fee(&child_0)?, child_0.encoded_size()) + + *INCREMENTAL_RELAY_FEE_RATE ); // Now that the minimum rolling fee has been bumped up, a low-fee tx will not pass @@ -2545,7 +2539,7 @@ mod tests { 2, InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), ), - mempool.get_minimum_rolling_fee().compute_fee(estimate_tx_size(1, 1))?, + mempool.get_minimum_rolling_fee().compute_fee(estimate_tx_size(1, 1)), flags, locktime, )?; @@ -2580,18 +2574,12 @@ mod tests { TxValidationError::RollingFeeThresholdNotMet { .. } )) )); - assert_eq!( - mempool.get_minimum_rolling_fee(), - (rolling_fee / FeeRate::new(Amount::from_atoms(2))).unwrap() - ); + assert_eq!(mempool.get_minimum_rolling_fee(), rolling_fee / 2); mock_clock.increment(halflife); log::debug!("Second attempt to add dummy"); mempool.add_transaction(dummy_tx)?; - assert_eq!( - mempool.get_minimum_rolling_fee(), - (rolling_fee / FeeRate::new(Amount::from_atoms(4))).unwrap() - ); + assert_eq!(mempool.get_minimum_rolling_fee(), rolling_fee / 4); log::debug!( "After successful addition of dummy, rolling fee rate is {:?}", mempool.get_minimum_rolling_fee() @@ -2602,19 +2590,13 @@ mod tests { let another_dummy = TxGenerator::new().with_fee(Amount::from_atoms(100)).generate_tx(&mempool)?; mempool.add_transaction(another_dummy)?; - assert_eq!( - mempool.get_minimum_rolling_fee(), - (rolling_fee / FeeRate::new(Amount::from_atoms(8))).unwrap() - ); + assert_eq!(mempool.get_minimum_rolling_fee(), rolling_fee / 8); mock_clock.increment(halflife); let final_dummy = TxGenerator::new().with_fee(Amount::from_atoms(100)).generate_tx(&mempool)?; mempool.add_transaction(final_dummy)?; - assert_eq!( - mempool.get_minimum_rolling_fee(), - FeeRate::new(Amount::from_atoms(0)) - ); + assert_eq!(mempool.get_minimum_rolling_fee(), FeeRate::new(0)); Ok(()) } From b3d5a66d0551d19b0577ba594e6e99bef52ab09e Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Mon, 18 Apr 2022 11:22:44 +0700 Subject: [PATCH 114/153] FeeRate multiply before divide in computing tokens_per_kb --- mempool/src/feerate.rs | 3 +-- mempool/src/pool.rs | 25 +++++++++++++++++-------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/mempool/src/feerate.rs b/mempool/src/feerate.rs index 6978cacbae..5cb5716d36 100644 --- a/mempool/src/feerate.rs +++ b/mempool/src/feerate.rs @@ -16,9 +16,8 @@ impl FeeRate { } pub(crate) fn of_tx(fee: Amount, tx_size: usize) -> Self { - let tokens_per_byte = Self::div_up(fee.into(), tx_size); Self { - tokens_per_kb: tokens_per_byte * 1000, + tokens_per_kb: Self::div_up(1000 * u128::try_from(fee).expect("of_tx"), tx_size), } } diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index ff838ef25d..018bdf37e6 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -2492,7 +2492,13 @@ mod tests { assert!(mempool.contains_transaction(&child_1_id)); assert!(!mempool.contains_transaction(&child_0_id)); let rolling_fee = mempool.get_minimum_rolling_fee(); - log::debug!("FeeRate of child_0 {:?}", mempool.try_get_fee(&child_0)?); + let child_0_fee = mempool.try_get_fee(&child_0)?; + log::debug!("FeeRate of child_0 {:?}", child_0_fee); + assert_eq!( + rolling_fee, + *INCREMENTAL_RELAY_FEE_RATE + FeeRate::of_tx(child_0_fee, child_0.encoded_size()) + ); + assert_eq!(rolling_fee, FeeRate::new(3561)); log::debug!( "minimum rolling fee after child_0's eviction {:?}", rolling_fee @@ -2574,29 +2580,32 @@ mod tests { TxValidationError::RollingFeeThresholdNotMet { .. } )) )); + log::debug!( + "minimum rolling fee after first attempt to add dummy: {:?}", + mempool.get_minimum_rolling_fee() + ); assert_eq!(mempool.get_minimum_rolling_fee(), rolling_fee / 2); mock_clock.increment(halflife); log::debug!("Second attempt to add dummy"); mempool.add_transaction(dummy_tx)?; + log::debug!( + "minimum rolling fee after first second to add dummy: {:?}", + mempool.get_minimum_rolling_fee() + ); assert_eq!(mempool.get_minimum_rolling_fee(), rolling_fee / 4); log::debug!( "After successful addition of dummy, rolling fee rate is {:?}", mempool.get_minimum_rolling_fee() ); - // Add more dummies until rolling feerate drops to zero + // Add another dummmy until rolling feerate drops to zero mock_clock.increment(halflife); let another_dummy = TxGenerator::new().with_fee(Amount::from_atoms(100)).generate_tx(&mempool)?; mempool.add_transaction(another_dummy)?; - assert_eq!(mempool.get_minimum_rolling_fee(), rolling_fee / 8); - - mock_clock.increment(halflife); - let final_dummy = - TxGenerator::new().with_fee(Amount::from_atoms(100)).generate_tx(&mempool)?; - mempool.add_transaction(final_dummy)?; assert_eq!(mempool.get_minimum_rolling_fee(), FeeRate::new(0)); + Ok(()) } From fe8020e4dbd89a5884d932ef86454236f57e0d1a Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Mon, 18 Apr 2022 11:27:39 +0700 Subject: [PATCH 115/153] Replace unwrap calls with expect --- mempool/src/feerate.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mempool/src/feerate.rs b/mempool/src/feerate.rs index 5cb5716d36..30fcb123f7 100644 --- a/mempool/src/feerate.rs +++ b/mempool/src/feerate.rs @@ -22,7 +22,9 @@ impl FeeRate { } pub(crate) fn compute_fee(&self, size: usize) -> Amount { - Amount::from_atoms(self.tokens_per_kb * u128::try_from(size).unwrap() / 1000) + Amount::from_atoms( + self.tokens_per_kb * u128::try_from(size).expect("compute_fee conversion") / 1000, + ) } pub(crate) fn tokens_per_kb(&self) -> u128 { @@ -30,7 +32,7 @@ impl FeeRate { } fn div_up(fee: u128, tx_size: usize) -> u128 { - let tx_size = u128::try_from(tx_size).unwrap(); + let tx_size = u128::try_from(tx_size).expect("div_up conversion"); (fee + tx_size - 1) / tx_size } } From 7b1c81542bc0f70c90e3fe90a7970d799f1b608f Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Thu, 21 Jul 2022 14:14:30 +0300 Subject: [PATCH 116/153] Fix tests that broke during rebase --- mempool/src/pool.rs | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 018bdf37e6..2a34fd92e2 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -88,6 +88,7 @@ where fn get_relay_fee(tx: &Transaction) -> Amount { // TODO we should never reach the expect, but should this be an error anyway? + eprintln!("get_relay_fee - encoded_size {:?}", tx.encoded_size()); Amount::from_atoms(u128::try_from(tx.encoded_size() * RELAY_FEE_PER_BYTE).expect("Overflow")) } @@ -753,6 +754,7 @@ where } fn rbf_checks(&self, tx: &Transaction) -> Result { + eprintln!("rbf_checks"); let conflicts = tx .inputs() .iter() @@ -803,9 +805,14 @@ where tx: &Transaction, total_conflict_fees: Amount, ) -> Result<(), TxValidationError> { + eprintln!("tx fee is {:?}", self.try_get_fee(tx)?); let additional_fees = (self.try_get_fee(tx)? - total_conflict_fees) .ok_or(TxValidationError::AdditionalFeesUnderflow)?; let relay_fee = get_relay_fee(tx); + eprintln!( + "conflict fees: {:?}, additional fee: {:?}, relay_fee {:?}", + total_conflict_fees, additional_fees, relay_fee + ); (additional_fees >= relay_fee) .then(|| ()) .ok_or(TxValidationError::InsufficientFeesToRelayRBF) @@ -1760,6 +1767,10 @@ mod tests { } fn test_replace_tx(original_fee: Amount, replacement_fee: Amount) -> Result<(), Error> { + eprintln!( + "tx_replace_tx: original_fee: {:?}, replacement_fee {:?}", + original_fee, replacement_fee + ); let mut mempool = setup(); let outpoint = mempool .available_outpoints(true) @@ -1782,11 +1793,16 @@ mod tests { let original = tx_spend_input(&mempool, input.clone(), original_fee, flags, locktime) .expect("should be able to spend here"); let original_id = original.get_id(); + eprintln!("created a tx with fee {:?}", mempool.try_get_fee(&original)); mempool.add_transaction(original)?; let flags = 0; let replacement = tx_spend_input(&mempool, input, replacement_fee, flags, locktime) .expect("should be able to spend here"); + eprintln!( + "created a replacement with fee {:?}", + mempool.try_get_fee(&replacement) + ); mempool.add_transaction(replacement)?; assert!(!mempool.contains_transaction(&original_id)); @@ -1842,7 +1858,7 @@ mod tests { let relay_fee = get_relay_fee_from_tx_size(TX_SPEND_INPUT_SIZE); let replacement_fee = Amount::from_atoms(relay_fee + 100); test_replace_tx(Amount::from_atoms(100), replacement_fee)?; - let res = test_replace_tx(Amount::from_atoms(100), Amount::from_atoms(relay_fee + 99)); + let res = test_replace_tx(Amount::from_atoms(300), replacement_fee); assert!(matches!( res, Err(Error::TxValidationError( @@ -1903,7 +1919,7 @@ mod tests { } // To test our validation of BIP125 Rule#4 (replacement transaction pays for its own bandwidth), we need to know the necessary relay fee before creating the transaction. The relay fee depends on the size of the transaction. The usual way to get the size of a transaction is to call `tx.encoded_size` but we cannot do this until we have created the transaction itself. To get around this cycle, we have precomputed the size of all transaction created by `tx_spend_input`. This value will be the same for all transactions created by this function. - const TX_SPEND_INPUT_SIZE: usize = 84; + const TX_SPEND_INPUT_SIZE: usize = 213; fn tx_spend_input( mempool: &MempoolImpl, @@ -2498,7 +2514,7 @@ mod tests { rolling_fee, *INCREMENTAL_RELAY_FEE_RATE + FeeRate::of_tx(child_0_fee, child_0.encoded_size()) ); - assert_eq!(rolling_fee, FeeRate::new(3561)); + assert_eq!(rolling_fee, FeeRate::new(3582)); log::debug!( "minimum rolling fee after child_0's eviction {:?}", rolling_fee From c02d1db6961d0617592f6e7831f9a670e775a0a0 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Tue, 26 Jul 2022 18:09:11 +0300 Subject: [PATCH 117/153] Move mempool tests to separate file --- mempool/src/pool.rs | 1793 +------------------------------------ mempool/src/pool/tests.rs | 1787 ++++++++++++++++++++++++++++++++++++ 2 files changed, 1788 insertions(+), 1792 deletions(-) create mode 100644 mempool/src/pool/tests.rs diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 2a34fd92e2..1fd1874edd 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -1089,1795 +1089,4 @@ where } #[cfg(test)] -mod tests { - use super::*; - use common::chain::signature::inputsig::InputWitness; - use common::chain::transaction::{Destination, TxInput, TxOutput}; - use common::chain::OutPointSourceId; - use common::chain::OutputPurpose; - use core::panic; - use std::sync::atomic::{AtomicU64, Ordering}; - use std::sync::Arc; - - const DUMMY_WITNESS_MSG: &[u8] = b"dummy_witness_msg"; - - #[derive(Debug, PartialEq, Eq, Clone)] - struct ValuedOutPoint { - outpoint: OutPoint, - value: Amount, - } - - impl std::cmp::PartialOrd for ValuedOutPoint { - fn partial_cmp(&self, other: &Self) -> Option { - other.value.partial_cmp(&self.value) - } - } - - impl std::cmp::Ord for ValuedOutPoint { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - other.value.cmp(&self.value) - } - } - - fn dummy_input() -> TxInput { - let outpoint_source_id = OutPointSourceId::Transaction(Id::new(H256::zero())); - let output_index = 0; - let witness = DUMMY_WITNESS_MSG.to_vec(); - TxInput::new( - outpoint_source_id, - output_index, - InputWitness::NoSignature(Some(witness)), - ) - } - - fn dummy_output() -> TxOutput { - let value = Amount::from_atoms(0); - let purpose = OutputPurpose::Transfer(Destination::AnyoneCanSpend); - TxOutput::new(value, purpose) - } - - fn estimate_tx_size(num_inputs: usize, num_outputs: usize) -> usize { - let inputs = (0..num_inputs).into_iter().map(|_| dummy_input()).collect(); - let outputs = (0..num_outputs).into_iter().map(|_| dummy_output()).collect(); - let flags = 0; - let locktime = 0; - let size = Transaction::new(flags, inputs, outputs, locktime).unwrap().encoded_size(); - // Take twice the encoded size of the dummy tx.Real Txs are larger than these dummy ones, - // but taking 3 times the size seems to ensure our txs won't fail the minimum relay fee - // validation (see the function `pays_minimum_relay_fees`) - let result = 3 * size; - log::debug!( - "estimated size for tx with {} inputs and {} outputs: {}", - num_inputs, - num_outputs, - result - ); - result - } - - #[test] - fn dummy_size() { - logging::init_logging::<&str>(None); - log::debug!("1, 1: {}", estimate_tx_size(1, 1)); - log::debug!("1, 2: {}", estimate_tx_size(1, 2)); - log::debug!("1, 400: {}", estimate_tx_size(1, 400)); - } - - #[test] - fn real_size() -> anyhow::Result<()> { - let mempool = setup(); - let tx = TxGenerator::new() - .with_num_inputs(1) - .with_num_outputs(400) - .generate_tx(&mempool)?; - log::debug!("real size of tx {}", tx.encoded_size()); - Ok(()) - } - - fn valued_outpoint( - tx_id: &Id, - outpoint_index: u32, - output: &TxOutput, - ) -> ValuedOutPoint { - let outpoint_source_id = OutPointSourceId::Transaction(*tx_id); - let outpoint = OutPoint::new(outpoint_source_id, outpoint_index); - let value = output.value(); - ValuedOutPoint { outpoint, value } - } - - pub(crate) fn create_genesis_tx() -> Transaction { - const TOTAL_SUPPLY: u128 = 10_000_000_000_000; - let genesis_message = b"".to_vec(); - let outpoint_source_id = OutPointSourceId::Transaction(Id::new(H256::zero())); - let input = TxInput::new( - outpoint_source_id, - 0, - InputWitness::NoSignature(Some(genesis_message)), - ); - let output = TxOutput::new( - Amount::from_atoms(TOTAL_SUPPLY), - OutputPurpose::Transfer(Destination::AnyoneCanSpend), - ); - Transaction::new(0, vec![input], vec![output], 0) - .expect("Failed to create genesis coinbase transaction") - } - - impl TxMempoolEntry { - fn outpoints_created(&self) -> BTreeSet { - let id = self.tx.get_id(); - std::iter::repeat(id) - .zip(self.tx.outputs().iter().enumerate()) - .map(|(id, (index, output))| valued_outpoint(&id, index as u32, output)) - .collect() - } - } - - impl MempoolStore { - fn unconfirmed_outpoints(&self) -> BTreeSet { - self.txs_by_id - .values() - .cloned() - .flat_map(|entry| entry.outpoints_created()) - .collect() - } - } - - impl MempoolImpl - where - T: GetTime, - M: GetMemoryUsage, - { - fn available_outpoints(&self, allow_double_spend: bool) -> BTreeSet { - let mut available = self - .store - .unconfirmed_outpoints() - .into_iter() - .chain(self.chain_state.confirmed_outpoints()) - .collect::>(); - if !allow_double_spend { - available.retain(|valued_outpoint| { - !self.store.spender_txs.contains_key(&valued_outpoint.outpoint) - }); - } - available - } - - fn get_input_value(&self, input: &TxInput) -> anyhow::Result { - let allow_double_spend = true; - self.available_outpoints(allow_double_spend) - .iter() - .find_map(|valued_outpoint| { - (valued_outpoint.outpoint == *input.outpoint()).then(|| valued_outpoint.value) - }) - .ok_or_else(|| anyhow::anyhow!("No such unconfirmed output")) - } - - fn get_minimum_rolling_fee(&self) -> FeeRate { - self.rolling_fee_rate.get().rolling_minimum_fee_rate - } - - fn process_block(&mut self, tx_id: &Id) -> anyhow::Result<()> { - let mut chain_state = self.chain_state.clone(); - chain_state.add_confirmed_tx( - self.store - .txs_by_id - .get(&tx_id.get()) - .cloned() - .ok_or_else(|| { - anyhow::anyhow!("process_block: tx {} not found in mempool", tx_id.get()) - })? - .tx, - ); - log::debug!("Setting tip to {:?}", chain_state); - self.new_tip_set(chain_state); - self.drop_transaction(tx_id); - Ok(()) - } - } - - #[derive(Debug, Clone)] - pub(crate) struct ChainStateMock { - confirmed_txs: HashMap, - available_outpoints: BTreeSet, - } - - impl ChainStateMock { - pub(crate) fn new() -> Self { - let genesis_tx = create_genesis_tx(); - let outpoint_source_id = OutPointSourceId::Transaction(genesis_tx.get_id()); - let outpoints = genesis_tx - .outputs() - .iter() - .enumerate() - .map(|(index, _)| OutPoint::new(outpoint_source_id.clone(), index as u32)) - .collect(); - Self { - confirmed_txs: std::iter::once((genesis_tx.get_id().get(), genesis_tx)).collect(), - available_outpoints: outpoints, - } - } - - fn confirmed_txs(&self) -> &HashMap { - &self.confirmed_txs - } - - fn confirmed_outpoints(&self) -> BTreeSet { - self.available_outpoints - .iter() - .map(|outpoint| { - let tx_id = outpoint - .tx_id() - .get_tx_id() - .cloned() - .expect("Outpoints in these tests are created from TXs"); - let index = outpoint.output_index(); - let tx = - self.confirmed_txs.get(&tx_id.get()).expect("Inconsistent Chain State"); - let output = tx - .outputs() - .get(index as usize) - .expect("Inconsistent Chain State: output not found"); - - valued_outpoint(&tx_id, index, output) - }) - .collect() - } - - fn add_confirmed_tx(&mut self, tx: Transaction) { - let outpoints_spent: BTreeSet<_> = - tx.inputs().iter().map(|input| input.outpoint()).collect(); - let outpoints_created: BTreeSet<_> = tx - .outputs() - .iter() - .enumerate() - .map(|(i, _)| OutPoint::new(OutPointSourceId::Transaction(tx.get_id()), i as u32)) - .collect(); - self.available_outpoints.extend(outpoints_created); - self.available_outpoints.retain(|outpoint| !outpoints_spent.contains(outpoint)); - self.confirmed_txs.insert(tx.get_id().get(), tx); - } - } - - impl ChainState for ChainStateMock { - fn contains_outpoint(&self, outpoint: &OutPoint) -> bool { - self.available_outpoints.iter().any(|value| *value == *outpoint) - } - - fn get_outpoint_value(&self, outpoint: &OutPoint) -> Result { - self.confirmed_txs - .get(&outpoint.tx_id().get_tx_id().expect("Not coinbase").get()) - .ok_or_else(|| anyhow::anyhow!("tx for outpoint sought in chain state, not found")) - .and_then(|tx| { - tx.outputs() - .get(outpoint.output_index() as usize) - .ok_or_else(|| anyhow::anyhow!("outpoint index out of bounds")) - .map(|output| output.value()) - }) - } - } - - /* FIXME The second call in the following flow sometimes returns TransactionAlreadyInMempool - let tx1 = TxGenerator::new(&mempool).generate_tx()?; - mempool.add_transaction(tx1)?; - - let tx2 = TxGenerator::new(&mempool).generate_tx()?; - mempool.add_transaction(tx2)?; - */ - struct TxGenerator { - coin_pool: BTreeSet, - num_inputs: usize, - num_outputs: usize, - tx_fee: Option, - replaceable: bool, - allow_double_spend: bool, - } - - impl TxGenerator { - fn with_num_inputs(mut self, num_inputs: usize) -> Self { - self.num_inputs = num_inputs; - self - } - - fn with_num_outputs(mut self, num_outputs: usize) -> Self { - self.num_outputs = num_outputs; - self - } - - fn replaceable(mut self) -> Self { - self.replaceable = true; - self - } - - fn with_fee(mut self, fee: Amount) -> Self { - self.tx_fee = Some(fee); - self - } - - fn new() -> Self { - Self { - coin_pool: BTreeSet::new(), - num_inputs: 1, - num_outputs: 1, - tx_fee: None, - replaceable: false, - allow_double_spend: false, - } - } - - fn generate_tx( - &mut self, - mempool: &MempoolImpl, - ) -> anyhow::Result { - self.coin_pool = mempool.available_outpoints(self.allow_double_spend); - let fee = if let Some(tx_fee) = self.tx_fee { - tx_fee - } else { - Amount::from_atoms(get_relay_fee_from_tx_size(estimate_tx_size( - self.num_inputs, - self.num_outputs, - ))) - }; - log::debug!( - "Trying to build a tx with {} inputs, {} outputs, and a fee of {:?}", - self.num_inputs, - self.num_outputs, - fee - ); - let valued_inputs = self.generate_tx_inputs(fee)?; - let outputs = self.generate_tx_outputs(&valued_inputs, fee)?; - let locktime = 0; - let flags = if self.replaceable { 1 } else { 0 }; - let (inputs, _): (Vec, Vec) = valued_inputs.into_iter().unzip(); - let spent_outpoints = - inputs.iter().map(|input| input.outpoint()).collect::>(); - self.coin_pool.retain(|outpoint| { - !spent_outpoints.iter().any(|spent| **spent == outpoint.outpoint) - }); - let tx = Transaction::new(flags, inputs, outputs.clone(), locktime) - .map_err(anyhow::Error::from)?; - self.coin_pool.extend( - std::iter::repeat(tx.get_id()) - .zip(outputs.iter().enumerate()) - .map(|(id, (i, output))| valued_outpoint(&id, i as u32, output)), - ); - - Ok(tx) - } - - fn generate_tx_inputs(&mut self, fee: Amount) -> anyhow::Result> { - Ok(self - .get_unspent_outpoints(self.num_inputs, fee)? - .iter() - .map(|valued_outpoint| { - let ValuedOutPoint { outpoint, value } = valued_outpoint; - ( - TxInput::new( - outpoint.tx_id(), - outpoint.output_index(), - InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), - ), - *value, - ) - }) - .collect()) - } - - fn generate_tx_outputs( - &self, - inputs: &[(TxInput, Amount)], - tx_fee: Amount, - ) -> anyhow::Result> { - if self.num_outputs == 0 { - return Ok(vec![]); - } - - let inputs: Vec<_> = inputs.to_owned(); - let (inputs, values): (Vec, Vec) = inputs.into_iter().unzip(); - if inputs.is_empty() { - return Ok(vec![]); - } - let sum_of_inputs = - values.into_iter().sum::>().expect("Overflow in sum of input values"); - - let total_to_spend = (sum_of_inputs - tx_fee).ok_or_else(||anyhow::anyhow!( - "generate_tx_outputs: underflow computing total_to_spend - sum_of_inputs = {:?}, fee = {:?}", sum_of_inputs, tx_fee - ))?; - - let value = (sum_of_inputs / u128::try_from(self.num_outputs).expect("conversion")) - .expect("not dividing by zero"); - - let mut left_to_spend = total_to_spend; - let mut outputs = Vec::new(); - - for _ in 0..self.num_outputs - 1 { - outputs.push(TxOutput::new( - value, - OutputPurpose::Transfer(Destination::AnyoneCanSpend), - )); - left_to_spend = (left_to_spend - value).expect("subtraction failed"); - } - - outputs.push(TxOutput::new( - left_to_spend, - OutputPurpose::Transfer(Destination::AnyoneCanSpend), - )); - Ok(outputs) - } - - fn get_unspent_outpoints( - &self, - num_outputs: usize, - fee: Amount, - ) -> anyhow::Result> { - log::debug!( - "get_unspent_outpoints: num_outputs: {}, fee: {:?}", - num_outputs, - fee - ); - let num_available_outpoints = self.coin_pool.len(); - let outpoints: Vec<_> = (num_available_outpoints >= num_outputs) - .then(|| self.coin_pool.iter().take(num_outputs).cloned().collect()) - .ok_or_else(|| anyhow::anyhow!("no outpoints left"))?; - let sum_of_outputs = outpoints - .iter() - .map(|valued_outpoint| valued_outpoint.value) - .sum::>() - .expect("sum error"); - if fee > sum_of_outputs { - Err(anyhow::Error::msg( - "get_unspent_outpoints:: fee is {:?} but sum of outputs is {:?}", - )) - } else { - Ok(outpoints) - } - } - } - - fn get_relay_fee_from_tx_size(tx_size: usize) -> u128 { - u128::try_from(tx_size * RELAY_FEE_PER_BYTE).expect("relay fee overflow") - } - - #[test] - fn add_single_tx() -> anyhow::Result<()> { - let mut mempool = setup(); - - let genesis_tx = mempool - .chain_state - .confirmed_txs() - .values() - .next() - .expect("genesis tx not found"); - - let outpoint_source_id = OutPointSourceId::Transaction(genesis_tx.get_id()); - - let flags = 0; - let locktime = 0; - let input = TxInput::new( - outpoint_source_id, - 0, - InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), - ); - let relay_fee = Amount::from_atoms(get_relay_fee_from_tx_size(TX_SPEND_INPUT_SIZE)); - let tx = tx_spend_input(&mempool, input, relay_fee, flags, locktime)?; - - let tx_clone = tx.clone(); - let tx_id = tx.get_id(); - mempool.add_transaction(tx)?; - assert!(mempool.contains_transaction(&tx_id)); - let all_txs = mempool.get_all(); - assert_eq!(all_txs, vec![&tx_clone]); - mempool.drop_transaction(&tx_id); - assert!(!mempool.contains_transaction(&tx_id)); - let all_txs = mempool.get_all(); - assert_eq!(all_txs, Vec::<&Transaction>::new()); - Ok(()) - } - - #[test] - fn txs_sorted() -> anyhow::Result<()> { - let mut mempool = setup(); - let mut tx_generator = TxGenerator::new(); - let target_txs = 10; - - for _ in 0..target_txs { - match tx_generator.generate_tx(&mempool) { - Ok(tx) => { - mempool.add_transaction(tx.clone())?; - } - _ => break, - } - } - - let fees = mempool - .get_all() - .iter() - .map(|tx| mempool.try_get_fee(tx)) - .collect::, _>>()?; - let mut fees_sorted = fees.clone(); - fees_sorted.sort_by(|a, b| b.cmp(a)); - assert_eq!(fees, fees_sorted); - Ok(()) - } - - #[test] - fn tx_no_inputs() -> anyhow::Result<()> { - let mut mempool = setup(); - let tx = TxGenerator::new() - .with_num_inputs(0) - .with_fee(Amount::from_atoms(0)) - .generate_tx(&mempool) - .expect("generate_tx failed"); - assert!(matches!( - mempool.add_transaction(tx), - Err(Error::TxValidationError(TxValidationError::NoInputs)) - )); - Ok(()) - } - - fn setup() -> MempoolImpl { - logging::init_logging::<&str>(None); - MempoolImpl::create( - ChainStateMock::new(), - SystemClock {}, - SystemUsageEstimator {}, - ) - } - - #[test] - fn tx_no_outputs() -> anyhow::Result<()> { - let mut mempool = setup(); - let tx = TxGenerator::new() - .with_num_outputs(0) - .generate_tx(&mempool) - .expect("generate_tx failed"); - assert!(matches!( - mempool.add_transaction(tx), - Err(Error::TxValidationError(TxValidationError::NoOutputs)) - )); - Ok(()) - } - - #[test] - fn tx_duplicate_inputs() -> anyhow::Result<()> { - let mut mempool = setup(); - - let genesis_tx = mempool - .chain_state - .confirmed_txs() - .values() - .next() - .expect("genesis tx not found"); - - let outpoint_source_id = OutPointSourceId::Transaction(genesis_tx.get_id()); - let input = TxInput::new( - outpoint_source_id.clone(), - 0, - InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), - ); - let witness = b"attempted_double_spend".to_vec(); - let duplicate_input = TxInput::new( - outpoint_source_id, - 0, - InputWitness::NoSignature(Some(witness)), - ); - let flags = 0; - let locktime = 0; - let outputs = tx_spend_input(&mempool, input.clone(), None, flags, locktime)? - .outputs() - .clone(); - let inputs = vec![input, duplicate_input]; - let tx = Transaction::new(flags, inputs, outputs, locktime)?; - - assert!(matches!( - mempool.add_transaction(tx), - Err(Error::TxValidationError(TxValidationError::DuplicateInputs)) - )); - Ok(()) - } - - #[test] - fn tx_already_in_mempool() -> anyhow::Result<()> { - let mut mempool = setup(); - - let genesis_tx = mempool - .chain_state - .confirmed_txs() - .values() - .next() - .expect("genesis tx not found"); - - let outpoint_source_id = OutPointSourceId::Transaction(genesis_tx.get_id()); - let input = TxInput::new( - outpoint_source_id, - 0, - InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), - ); - - let flags = 0; - let locktime = 0; - let tx = tx_spend_input(&mempool, input, None, flags, locktime)?; - - mempool.add_transaction(tx.clone())?; - assert!(matches!( - mempool.add_transaction(tx), - Err(Error::TxValidationError( - TxValidationError::TransactionAlreadyInMempool - )) - )); - Ok(()) - } - - #[test] - fn outpoint_not_found() -> anyhow::Result<()> { - let mut mempool = setup(); - - let genesis_tx = mempool - .chain_state - .confirmed_txs() - .values() - .next() - .expect("genesis tx not found"); - - let outpoint_source_id = OutPointSourceId::Transaction(genesis_tx.get_id()); - - let good_input = TxInput::new( - outpoint_source_id.clone(), - 0, - InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), - ); - let flags = 0; - let locktime = 0; - let outputs = - tx_spend_input(&mempool, good_input, None, flags, locktime)?.outputs().clone(); - - let bad_outpoint_index = 1; - let bad_input = TxInput::new( - outpoint_source_id, - bad_outpoint_index, - InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), - ); - - let inputs = vec![bad_input]; - let tx = Transaction::new(flags, inputs, outputs, locktime)?; - - assert!(matches!( - mempool.add_transaction(tx), - Err(Error::TxValidationError( - TxValidationError::OutPointNotFound { .. } - )) - )); - - Ok(()) - } - - #[test] - fn tx_too_big() -> anyhow::Result<()> { - let mut mempool = setup(); - let tx = TxGenerator::new() - .with_num_outputs(400_000) - .generate_tx(&mempool) - .expect("generate_tx failed"); - assert!(matches!( - mempool.add_transaction(tx), - Err(Error::TxValidationError( - TxValidationError::ExceedsMaxBlockSize - )) - )); - Ok(()) - } - - fn test_replace_tx(original_fee: Amount, replacement_fee: Amount) -> Result<(), Error> { - eprintln!( - "tx_replace_tx: original_fee: {:?}, replacement_fee {:?}", - original_fee, replacement_fee - ); - let mut mempool = setup(); - let outpoint = mempool - .available_outpoints(true) - .iter() - .next() - .expect("there should be an outpoint since setup creates the genesis transaction") - .outpoint - .clone(); - - let outpoint_source_id = - OutPointSourceId::from(*outpoint.tx_id().get_tx_id().expect("Not Coinbase")); - - let input = TxInput::new( - outpoint_source_id, - 0, - InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), - ); - let flags = 1; - let locktime = 0; - let original = tx_spend_input(&mempool, input.clone(), original_fee, flags, locktime) - .expect("should be able to spend here"); - let original_id = original.get_id(); - eprintln!("created a tx with fee {:?}", mempool.try_get_fee(&original)); - mempool.add_transaction(original)?; - - let flags = 0; - let replacement = tx_spend_input(&mempool, input, replacement_fee, flags, locktime) - .expect("should be able to spend here"); - eprintln!( - "created a replacement with fee {:?}", - mempool.try_get_fee(&replacement) - ); - mempool.add_transaction(replacement)?; - assert!(!mempool.contains_transaction(&original_id)); - - Ok(()) - } - - #[test] - fn try_replace_irreplaceable() -> anyhow::Result<()> { - let mut mempool = setup(); - let outpoint = mempool - .available_outpoints(true) - .iter() - .next() - .expect("there should be an outpoint since setup creates the genesis transaction") - .outpoint - .clone(); - - let outpoint_source_id = - OutPointSourceId::from(*outpoint.tx_id().get_tx_id().expect("Not Coinbase")); - - let input = TxInput::new( - outpoint_source_id, - 0, - InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), - ); - let flags = 0; - let locktime = 0; - let original_fee = Amount::from_atoms(get_relay_fee_from_tx_size(TX_SPEND_INPUT_SIZE)); - let original = tx_spend_input(&mempool, input.clone(), original_fee, flags, locktime) - .expect("should be able to spend here"); - let original_id = original.get_id(); - mempool.add_transaction(original)?; - - let flags = 0; - let replacement_fee = (original_fee + Amount::from_atoms(1000)).unwrap(); - let replacement = tx_spend_input(&mempool, input, replacement_fee, flags, locktime) - .expect("should be able to spend here"); - assert!(matches!( - mempool.add_transaction(replacement.clone()), - Err(Error::TxValidationError( - TxValidationError::ConflictWithIrreplaceableTransaction - )) - )); - - mempool.drop_transaction(&original_id); - mempool.add_transaction(replacement)?; - - Ok(()) - } - - #[test] - fn tx_replace() -> anyhow::Result<()> { - let relay_fee = get_relay_fee_from_tx_size(TX_SPEND_INPUT_SIZE); - let replacement_fee = Amount::from_atoms(relay_fee + 100); - test_replace_tx(Amount::from_atoms(100), replacement_fee)?; - let res = test_replace_tx(Amount::from_atoms(300), replacement_fee); - assert!(matches!( - res, - Err(Error::TxValidationError( - TxValidationError::InsufficientFeesToRelayRBF - )) - )); - let res = test_replace_tx(Amount::from_atoms(100), Amount::from_atoms(100)); - assert!(matches!( - res, - Err(Error::TxValidationError( - TxValidationError::ReplacementFeeLowerThanOriginal { .. } - )) - )); - let res = test_replace_tx(Amount::from_atoms(100), Amount::from_atoms(90)); - assert!(matches!( - res, - Err(Error::TxValidationError( - TxValidationError::ReplacementFeeLowerThanOriginal { .. } - )) - )); - Ok(()) - } - - #[test] - fn tx_replace_child() -> anyhow::Result<()> { - let mut mempool = setup(); - let tx = TxGenerator::new() - .replaceable() - .generate_tx(&mempool) - .expect("generate_replaceable_tx"); - mempool.add_transaction(tx.clone())?; - - let outpoint_source_id = OutPointSourceId::Transaction(tx.get_id()); - let child_tx_input = TxInput::new( - outpoint_source_id, - 0, - InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), - ); - // We want to test that even though child_tx doesn't signal replaceability directly, it is replaceable because its parent signalled replaceability - // replaced - let flags = 0; - let locktime = 0; - let child_tx = tx_spend_input( - &mempool, - child_tx_input.clone(), - Amount::from_atoms(100), - flags, - locktime, - )?; - mempool.add_transaction(child_tx)?; - - let relay_fee = get_relay_fee_from_tx_size(TX_SPEND_INPUT_SIZE); - let replacement_fee = Amount::from_atoms(relay_fee + 100); - let replacement_tx = - tx_spend_input(&mempool, child_tx_input, replacement_fee, flags, locktime)?; - mempool.add_transaction(replacement_tx)?; - Ok(()) - } - - // To test our validation of BIP125 Rule#4 (replacement transaction pays for its own bandwidth), we need to know the necessary relay fee before creating the transaction. The relay fee depends on the size of the transaction. The usual way to get the size of a transaction is to call `tx.encoded_size` but we cannot do this until we have created the transaction itself. To get around this cycle, we have precomputed the size of all transaction created by `tx_spend_input`. This value will be the same for all transactions created by this function. - const TX_SPEND_INPUT_SIZE: usize = 213; - - fn tx_spend_input( - mempool: &MempoolImpl, - input: TxInput, - fee: impl Into>, - flags: u32, - locktime: u32, - ) -> anyhow::Result { - let fee = fee.into().map_or_else( - || Amount::from_atoms(get_relay_fee_from_tx_size(estimate_tx_size(1, 2))), - std::convert::identity, - ); - tx_spend_several_inputs(mempool, &[input], fee, flags, locktime) - } - - fn tx_spend_several_inputs( - mempool: &MempoolImpl, - inputs: &[TxInput], - fee: Amount, - flags: u32, - locktime: u32, - ) -> anyhow::Result { - let input_value = inputs - .iter() - .map(|input| mempool.get_input_value(input)) - .collect::, _>>()? - .into_iter() - .sum::>() - .ok_or_else(|| { - let msg = String::from("tx_spend_input: overflow"); - log::error!("{}", msg); - anyhow::Error::msg(msg) - })?; - - let available_for_spending = (input_value - fee).ok_or_else(|| { - let msg = format!( - "tx_spend_several_inputs: input_value ({:?}) lower than fee ({:?})", - input_value, fee - ); - log::error!("{}", msg); - anyhow::Error::msg(msg) - })?; - let spent = (available_for_spending / 2).expect("division error"); - - let change = (available_for_spending - spent).ok_or_else(|| { - let msg = String::from("Error computing change"); - anyhow::Error::msg(msg) - })?; - - Transaction::new( - flags, - inputs.to_owned(), - vec![ - TxOutput::new(spent, OutputPurpose::Transfer(Destination::AnyoneCanSpend)), - TxOutput::new(change, OutputPurpose::Transfer(Destination::AnyoneCanSpend)), - ], - locktime, - ) - .map_err(Into::into) - } - - #[test] - fn one_ancestor_signal_is_enough() -> anyhow::Result<()> { - let mut mempool = setup(); - let tx = TxGenerator::new() - .with_num_outputs(2) - .generate_tx(&mempool) - .expect("generate_replaceable_tx"); - - mempool.add_transaction(tx.clone())?; - - let flags_replaceable = 1; - let flags_irreplaceable = 0; - let locktime = 0; - - let outpoint_source_id = OutPointSourceId::Transaction(tx.get_id()); - let ancestor_with_signal = tx_spend_input( - &mempool, - TxInput::new( - outpoint_source_id.clone(), - 0, - InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), - ), - None, - flags_replaceable, - locktime, - )?; - - let ancestor_without_signal = tx_spend_input( - &mempool, - TxInput::new( - outpoint_source_id, - 1, - InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), - ), - None, - flags_irreplaceable, - locktime, - )?; - - mempool.add_transaction(ancestor_with_signal.clone())?; - mempool.add_transaction(ancestor_without_signal.clone())?; - - let input_with_replaceable_parent = TxInput::new( - OutPointSourceId::Transaction(ancestor_with_signal.get_id()), - 0, - InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), - ); - - let input_with_irreplaceable_parent = TxInput::new( - OutPointSourceId::Transaction(ancestor_without_signal.get_id()), - 0, - InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), - ); - - // TODO compute minimum necessary relay fee instead of just overestimating it - let original_fee = Amount::from_atoms(200); - let dummy_output = TxOutput::new( - original_fee, - OutputPurpose::Transfer(Destination::AnyoneCanSpend), - ); - let replaced_tx = tx_spend_several_inputs( - &mempool, - &[input_with_irreplaceable_parent.clone(), input_with_replaceable_parent], - original_fee, - flags_irreplaceable, - locktime, - )?; - let replaced_tx_id = replaced_tx.get_id(); - - mempool.add_transaction(replaced_tx)?; - - let replacing_tx = Transaction::new( - flags_irreplaceable, - vec![input_with_irreplaceable_parent], - vec![dummy_output], - locktime, - )?; - - mempool.add_transaction(replacing_tx)?; - assert!(!mempool.contains_transaction(&replaced_tx_id)); - - Ok(()) - } - - #[test] - fn tx_mempool_entry() -> anyhow::Result<()> { - use common::primitives::time; - let mut mempool = setup(); - // Input different flag values just to make the hashes of these dummy transactions - // different - let txs = (1..=6) - .into_iter() - .map(|i| Transaction::new(i, vec![], vec![], 0).unwrap_or_else(|_| panic!("tx {}", i))) - .collect::>(); - let fee = Amount::from_atoms(0); - - // Generation 1 - let tx1_parents = BTreeSet::default(); - let entry1 = - TxMempoolEntry::new(txs.get(0).unwrap().clone(), fee, tx1_parents, time::get()); - let tx2_parents = BTreeSet::default(); - let entry2 = - TxMempoolEntry::new(txs.get(1).unwrap().clone(), fee, tx2_parents, time::get()); - - // Generation 2 - let tx3_parents = vec![entry1.tx_id(), entry2.tx_id()].into_iter().collect(); - let entry3 = - TxMempoolEntry::new(txs.get(2).unwrap().clone(), fee, tx3_parents, time::get()); - - // Generation 3 - let tx4_parents = vec![entry3.tx_id()].into_iter().collect(); - let tx5_parents = vec![entry3.tx_id()].into_iter().collect(); - let entry4 = - TxMempoolEntry::new(txs.get(3).unwrap().clone(), fee, tx4_parents, time::get()); - let entry5 = - TxMempoolEntry::new(txs.get(4).unwrap().clone(), fee, tx5_parents, time::get()); - - // Generation 4 - let tx6_parents = - vec![entry3.tx_id(), entry4.tx_id(), entry5.tx_id()].into_iter().collect(); - let entry6 = - TxMempoolEntry::new(txs.get(5).unwrap().clone(), fee, tx6_parents, time::get()); - - let entries = vec![entry1, entry2, entry3, entry4, entry5, entry6]; - let ids = entries.clone().into_iter().map(|entry| entry.tx_id()).collect::>(); - - for entry in entries.into_iter() { - mempool.store.add_tx(entry)?; - } - - let entry1 = mempool.store.get_entry(ids.get(0).expect("index")).expect("entry"); - let entry2 = mempool.store.get_entry(ids.get(1).expect("index")).expect("entry"); - let entry3 = mempool.store.get_entry(ids.get(2).expect("index")).expect("entry"); - let entry4 = mempool.store.get_entry(ids.get(3).expect("index")).expect("entry"); - let entry5 = mempool.store.get_entry(ids.get(4).expect("index")).expect("entry"); - let entry6 = mempool.store.get_entry(ids.get(5).expect("index")).expect("entry"); - assert_eq!(entry1.unconfirmed_ancestors(&mempool.store).0.len(), 0); - assert_eq!(entry2.unconfirmed_ancestors(&mempool.store).0.len(), 0); - assert_eq!(entry3.unconfirmed_ancestors(&mempool.store).0.len(), 2); - assert_eq!(entry4.unconfirmed_ancestors(&mempool.store).0.len(), 3); - assert_eq!(entry5.unconfirmed_ancestors(&mempool.store).0.len(), 3); - assert_eq!(entry6.unconfirmed_ancestors(&mempool.store).0.len(), 5); - - assert_eq!(entry1.count_with_descendants(), 5); - assert_eq!(entry2.count_with_descendants(), 5); - assert_eq!(entry3.count_with_descendants(), 4); - assert_eq!(entry4.count_with_descendants(), 2); - assert_eq!(entry5.count_with_descendants(), 2); - assert_eq!(entry6.count_with_descendants(), 1); - - Ok(()) - } - - fn test_bip125_max_replacements( - mempool: &mut MempoolImpl, - num_potential_replacements: usize, - ) -> anyhow::Result<()> { - let tx = TxGenerator::new() - .with_num_outputs(num_potential_replacements - 1) - .replaceable() - .generate_tx(mempool) - .expect("generate_tx failed"); - let input = tx.inputs().first().expect("one input").clone(); - let outputs = tx.outputs().clone(); - let tx_id = tx.get_id(); - mempool.add_transaction(tx)?; - - let flags = 0; - let locktime = 0; - let outpoint_source_id = OutPointSourceId::Transaction(tx_id); - let fee = get_relay_fee_from_tx_size(TX_SPEND_INPUT_SIZE); - for (index, _) in outputs.iter().enumerate() { - let input = TxInput::new( - outpoint_source_id.clone(), - index.try_into().unwrap(), - InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), - ); - let tx = tx_spend_input(mempool, input, Amount::from_atoms(fee), flags, locktime)?; - mempool.add_transaction(tx)?; - } - let mempool_size_before_replacement = mempool.store.txs_by_id.len(); - - let replacement_fee = Amount::from_atoms(1000) * fee; - let replacement_tx = tx_spend_input(mempool, input, replacement_fee, flags, locktime)?; - mempool.add_transaction(replacement_tx).map_err(anyhow::Error::from)?; - let mempool_size_after_replacement = mempool.store.txs_by_id.len(); - - assert_eq!( - mempool_size_after_replacement, - mempool_size_before_replacement - num_potential_replacements + 1 - ); - Ok(()) - } - - #[test] - fn too_many_conflicts() -> anyhow::Result<()> { - let mut mempool = setup(); - let num_potential_replacements = MAX_BIP125_REPLACEMENT_CANDIDATES + 1; - let err = test_bip125_max_replacements(&mut mempool, num_potential_replacements) - .expect_err("expected error TooManyPotentialReplacements") - .downcast() - .expect("failed to downcast"); - assert!(matches!( - err, - Error::TxValidationError(TxValidationError::TooManyPotentialReplacements) - )); - Ok(()) - } - - #[test] - fn not_too_many_conflicts() -> anyhow::Result<()> { - let mut mempool = setup(); - let num_potential_replacements = MAX_BIP125_REPLACEMENT_CANDIDATES; - test_bip125_max_replacements(&mut mempool, num_potential_replacements) - } - - #[test] - fn spends_new_unconfirmed() -> anyhow::Result<()> { - let mut mempool = setup(); - let tx = TxGenerator::new() - .with_num_outputs(2) - .replaceable() - .generate_tx(&mempool) - .expect("generate_replaceable_tx"); - let outpoint_source_id = OutPointSourceId::Transaction(tx.get_id()); - mempool.add_transaction(tx)?; - - let input1 = TxInput::new( - outpoint_source_id.clone(), - 0, - InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), - ); - let input2 = TxInput::new( - outpoint_source_id, - 1, - InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), - ); - - let locktime = 0; - let flags = 0; - let original_fee = Amount::from_atoms(100); - let replaced_tx = tx_spend_input(&mempool, input1.clone(), original_fee, flags, locktime)?; - mempool.add_transaction(replaced_tx)?; - let relay_fee = get_relay_fee_from_tx_size(TX_SPEND_INPUT_SIZE); - let replacement_fee = Amount::from_atoms(100 + relay_fee); - let incoming_tx = tx_spend_several_inputs( - &mempool, - &[input1, input2], - replacement_fee, - flags, - locktime, - )?; - - let res = mempool.add_transaction(incoming_tx); - assert!(matches!( - res, - Err(Error::TxValidationError( - TxValidationError::SpendsNewUnconfirmedOutput - )) - )); - Ok(()) - } - - #[test] - fn pays_more_than_conflicts_with_descendants() -> anyhow::Result<()> { - let mut mempool = setup(); - let tx = TxGenerator::new().generate_tx(&mempool).expect("generate_replaceable_tx"); - let tx_id = tx.get_id(); - mempool.add_transaction(tx)?; - - let outpoint_source_id = OutPointSourceId::Transaction(tx_id); - let input = TxInput::new( - outpoint_source_id, - 0, - InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), - ); - - let locktime = 0; - let rbf = 1; - let no_rbf = 0; - - // Create transaction that we will attempt to replace - let original_fee = Amount::from_atoms(100); - let replaced_tx = tx_spend_input(&mempool, input.clone(), original_fee, rbf, locktime)?; - let replaced_tx_fee = mempool.try_get_fee(&replaced_tx)?; - let replaced_id = replaced_tx.get_id(); - mempool.add_transaction(replaced_tx)?; - - // Create some children for this transaction - let descendant_outpoint_source_id = OutPointSourceId::Transaction(replaced_id); - - let descendant1_fee = Amount::from_atoms(100); - let descendant1 = tx_spend_input( - &mempool, - TxInput::new( - descendant_outpoint_source_id.clone(), - 0, - InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), - ), - descendant1_fee, - no_rbf, - locktime, - )?; - let descendant1_id = descendant1.get_id(); - mempool.add_transaction(descendant1)?; - - let descendant2_fee = Amount::from_atoms(100); - let descendant2 = tx_spend_input( - &mempool, - TxInput::new( - descendant_outpoint_source_id, - 1, - InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), - ), - descendant2_fee, - no_rbf, - locktime, - )?; - let descendant2_id = descendant2.get_id(); - mempool.add_transaction(descendant2)?; - - //Create a new incoming transaction that conflicts with `replaced_tx` because it spends - //`input`. It will be rejected because its fee exactly equals (so is not greater than) the - //sum of the fees of the conflict together with its descendants - let insufficient_rbf_fee = [replaced_tx_fee, descendant1_fee, descendant2_fee] - .into_iter() - .sum::>() - .unwrap(); - let incoming_tx = tx_spend_input( - &mempool, - input.clone(), - insufficient_rbf_fee, - no_rbf, - locktime, - )?; - - assert!(matches!( - mempool.add_transaction(incoming_tx), - Err(Error::TxValidationError( - TxValidationError::TransactionFeeLowerThanConflictsWithDescendants - )) - )); - - let relay_fee = get_relay_fee_from_tx_size(TX_SPEND_INPUT_SIZE); - let sufficient_rbf_fee = insufficient_rbf_fee + Amount::from_atoms(relay_fee); - let incoming_tx = tx_spend_input(&mempool, input, sufficient_rbf_fee, no_rbf, locktime)?; - mempool.add_transaction(incoming_tx)?; - - assert!(!mempool.contains_transaction(&replaced_id)); - assert!(!mempool.contains_transaction(&descendant1_id)); - assert!(!mempool.contains_transaction(&descendant2_id)); - Ok(()) - } - - #[derive(Clone)] - struct MockClock { - time: Arc, - } - - impl MockClock { - fn new() -> Self { - Self { - time: Arc::new(AtomicU64::new(0)), - } - } - - fn set(&self, time: Time) { - self.time.store(time.as_secs(), Ordering::SeqCst) - } - - fn increment(&self, inc: Time) { - self.time.store( - self.time.load(Ordering::SeqCst) + inc.as_secs(), - Ordering::SeqCst, - ) - } - } - - impl GetTime for MockClock { - fn get_time(&self) -> Time { - Duration::new(self.time.load(Ordering::SeqCst), 0) - } - } - - #[test] - fn only_expired_entries_removed() -> anyhow::Result<()> { - let mock_clock = MockClock::new(); - - let mut mempool = MempoolImpl::create( - ChainStateMock::new(), - mock_clock.clone(), - SystemUsageEstimator {}, - ); - - let num_inputs = 1; - let num_outputs = 2; - let big_fee = get_relay_fee_from_tx_size(estimate_tx_size(num_inputs, num_outputs)) + 100; - let parent = TxGenerator::new() - .with_num_inputs(num_inputs) - .with_num_outputs(num_outputs) - .with_fee(Amount::from_atoms(big_fee)) - .generate_tx(&mempool)?; - let parent_id = parent.get_id(); - mempool.add_transaction(parent)?; - - let flags = 0; - let locktime = 0; - let outpoint_source_id = OutPointSourceId::Transaction(parent_id); - let child_0 = tx_spend_input( - &mempool, - TxInput::new( - outpoint_source_id.clone(), - 0, - InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), - ), - None, - flags, - locktime, - )?; - - let child_1 = tx_spend_input( - &mempool, - TxInput::new( - outpoint_source_id, - 1, - InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), - ), - None, - flags, - locktime, - )?; - let child_1_id = child_1.get_id(); - - let expired_tx_id = child_0.get_id(); - mempool.add_transaction(child_0)?; - - // Simulate the parent being added to a block - // We have to do this because if we leave this parent in the mempool then it will be - // expired, and so removed along with both its children, and thus the addition of child_1 to - // the mempool will fail - mempool.process_block(&parent_id)?; - mock_clock.set(DEFAULT_MEMPOOL_EXPIRY + Duration::new(1, 0)); - - mempool.add_transaction(child_1)?; - assert!(!mempool.contains_transaction(&expired_tx_id)); - assert!(mempool.contains_transaction(&child_1_id)); - Ok(()) - } - - #[test] - fn rolling_fee() -> anyhow::Result<()> { - logging::init_logging::<&str>(None); - let mock_clock = MockClock::new(); - let mut mock_usage = MockGetMemoryUsage::new(); - // Add parent - // Add first child - mock_usage.expect_get_memory_usage().times(2).return_const(0usize); - // Add second child, triggering the trimming process - mock_usage - .expect_get_memory_usage() - .times(1) - .return_const(MAX_MEMPOOL_SIZE_BYTES + 1); - // After removing one entry, cause the code to exit the loop by showing a small usage - mock_usage.expect_get_memory_usage().return_const(0usize); - - let chain_state = ChainStateMock::new(); - let mut mempool = MempoolImpl::create(chain_state, mock_clock.clone(), mock_usage); - - let num_inputs = 1; - let num_outputs = 3; - - // Use a higher than default fee because we don't want this transction to be evicted during - // the trimming process - let parent = TxGenerator::new() - .with_num_inputs(num_inputs) - .with_num_outputs(num_outputs) - .generate_tx(&mempool)?; - let parent_id = parent.get_id(); - log::debug!("parent_id: {}", parent_id.get()); - log::debug!("before adding parent"); - mempool.add_transaction(parent)?; - log::debug!("after adding parent"); - - let flags = 0; - let locktime = 0; - let outpoint_source_id = OutPointSourceId::Transaction(parent_id); - - // child_0 has the lower fee so it will be evicted when memory usage is too high - let child_0 = tx_spend_input( - &mempool, - TxInput::new( - outpoint_source_id.clone(), - 0, - InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), - ), - None, - flags, - locktime, - )?; - let child_0_id = child_0.get_id(); - log::debug!("child_0_id {}", child_0_id.get()); - - let big_fee = Amount::from_atoms( - get_relay_fee_from_tx_size(estimate_tx_size(num_inputs, num_outputs)) + 100, - ); - let child_1 = tx_spend_input( - &mempool, - TxInput::new( - outpoint_source_id.clone(), - 1, - InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), - ), - big_fee, - flags, - locktime, - )?; - let child_1_id = child_1.get_id(); - log::debug!("child_1_id {}", child_1_id.get()); - mempool.add_transaction(child_0.clone())?; - log::debug!("added child_0"); - mempool.add_transaction(child_1)?; - log::debug!("added child_1"); - - assert_eq!(mempool.store.txs_by_id.len(), 2); - assert!(mempool.contains_transaction(&child_1_id)); - assert!(!mempool.contains_transaction(&child_0_id)); - let rolling_fee = mempool.get_minimum_rolling_fee(); - let child_0_fee = mempool.try_get_fee(&child_0)?; - log::debug!("FeeRate of child_0 {:?}", child_0_fee); - assert_eq!( - rolling_fee, - *INCREMENTAL_RELAY_FEE_RATE + FeeRate::of_tx(child_0_fee, child_0.encoded_size()) - ); - assert_eq!(rolling_fee, FeeRate::new(3582)); - log::debug!( - "minimum rolling fee after child_0's eviction {:?}", - rolling_fee - ); - assert_eq!( - rolling_fee, - FeeRate::of_tx(mempool.try_get_fee(&child_0)?, child_0.encoded_size()) - + *INCREMENTAL_RELAY_FEE_RATE - ); - - // Now that the minimum rolling fee has been bumped up, a low-fee tx will not pass - // validation - let child_2 = tx_spend_input( - &mempool, - TxInput::new( - outpoint_source_id.clone(), - 2, - InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), - ), - None, - flags, - locktime, - )?; - log::debug!( - "before child2: fee = {:?}, size = {}, minimum fee rate = {:?}", - mempool.try_get_fee(&child_2)?, - child_2.encoded_size(), - mempool.get_minimum_rolling_fee() - ); - let res = mempool.add_transaction(child_2); - log::debug!("result of adding child2 {:?}", res); - assert!(matches!( - res, - Err(Error::TxValidationError( - TxValidationError::RollingFeeThresholdNotMet { .. } - )) - )); - - // We provide a sufficient fee for the tx to pass the minimum rolling fee requirement - let child_2_high_fee = tx_spend_input( - &mempool, - TxInput::new( - outpoint_source_id, - 2, - InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), - ), - mempool.get_minimum_rolling_fee().compute_fee(estimate_tx_size(1, 1)), - flags, - locktime, - )?; - log::debug!("before child2_high_fee"); - mempool.add_transaction(child_2_high_fee)?; - - // We simulate a block being accepted so the rolling fee will begin to decay - mempool.process_block(&parent_id)?; - - // Because the rolling fee is only updated when we attempt to add a tx to the mempool - // we need to submit a "dummy" tx to trigger these updates. - - // Since memory usage is now zero, it is less than 1/4 of the max size - // and ROLLING_FEE_BASE_HALFLIFE / 4 is the time it will take for the fee to halve - // We are going to submit dummy txs to the mempool incrementing time by this halflife - // between txs. Finally, when the fee rate falls under INCREMENTAL_RELAY_THRESHOLD, we - // observer that it is set to zero - let halflife = ROLLING_FEE_BASE_HALFLIFE / 4; - mock_clock.increment(halflife); - let dummy_tx = - TxGenerator::new().with_fee(Amount::from_atoms(100)).generate_tx(&mempool)?; - log::debug!( - "First attempt to add dummy which pays a fee of {:?}", - mempool.try_get_fee(&dummy_tx)? - ); - let res = mempool.add_transaction(dummy_tx.clone()); - - log::debug!("Result of first attempt to add dummy: {:?}", res); - assert!(matches!( - res, - Err(Error::TxValidationError( - TxValidationError::RollingFeeThresholdNotMet { .. } - )) - )); - log::debug!( - "minimum rolling fee after first attempt to add dummy: {:?}", - mempool.get_minimum_rolling_fee() - ); - assert_eq!(mempool.get_minimum_rolling_fee(), rolling_fee / 2); - - mock_clock.increment(halflife); - log::debug!("Second attempt to add dummy"); - mempool.add_transaction(dummy_tx)?; - log::debug!( - "minimum rolling fee after first second to add dummy: {:?}", - mempool.get_minimum_rolling_fee() - ); - assert_eq!(mempool.get_minimum_rolling_fee(), rolling_fee / 4); - log::debug!( - "After successful addition of dummy, rolling fee rate is {:?}", - mempool.get_minimum_rolling_fee() - ); - - // Add another dummmy until rolling feerate drops to zero - mock_clock.increment(halflife); - let another_dummy = - TxGenerator::new().with_fee(Amount::from_atoms(100)).generate_tx(&mempool)?; - mempool.add_transaction(another_dummy)?; - assert_eq!(mempool.get_minimum_rolling_fee(), FeeRate::new(0)); - - Ok(()) - } - - #[test] - fn different_size_txs() -> anyhow::Result<()> { - let mut mempool = setup(); - let initial_tx = TxGenerator::new() - .with_num_inputs(1) - .with_num_outputs(10_000) - .generate_tx(&mempool)?; - mempool.add_transaction(initial_tx)?; - - let target_txs = 100; - for i in 0..target_txs { - let num_inputs = i + 1; - let num_outputs = i + 1; - let tx = TxGenerator::new() - .with_num_inputs(num_inputs) - .with_num_outputs(num_outputs) - .generate_tx(&mempool)?; - mempool.add_transaction(tx)?; - } - - Ok(()) - } - - #[test] - fn descendant_score() -> anyhow::Result<()> { - let mut mempool = setup(); - let tx = TxGenerator::new() - .with_num_outputs(2) - .generate_tx(&mempool) - .expect("generate_replaceable_tx"); - let tx_id = tx.get_id(); - mempool.add_transaction(tx)?; - - let outpoint_source_id = OutPointSourceId::Transaction(tx_id); - - let flags = 0; - let locktime = 0; - - let tx_b_fee = Amount::from_atoms(get_relay_fee_from_tx_size(estimate_tx_size(1, 2))); - let tx_a_fee = (tx_b_fee + Amount::from_atoms(1000)).unwrap(); - let tx_c_fee = (tx_a_fee + Amount::from_atoms(1000)).unwrap(); - let tx_a = tx_spend_input( - &mempool, - TxInput::new( - outpoint_source_id.clone(), - 0, - InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), - ), - tx_a_fee, - flags, - locktime, - )?; - let tx_a_id = tx_a.get_id(); - log::debug!("tx_a_id : {}", tx_a_id.get()); - log::debug!("tx_a fee : {:?}", mempool.try_get_fee(&tx_a)?); - mempool.add_transaction(tx_a)?; - - let tx_b = tx_spend_input( - &mempool, - TxInput::new( - outpoint_source_id, - 1, - InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), - ), - tx_b_fee, - flags, - locktime, - )?; - let tx_b_id = tx_b.get_id(); - log::debug!("tx_b_id : {}", tx_b_id.get()); - log::debug!("tx_b fee : {:?}", mempool.try_get_fee(&tx_b)?); - mempool.add_transaction(tx_b)?; - - let tx_c = tx_spend_input( - &mempool, - TxInput::new( - OutPointSourceId::Transaction(tx_b_id), - 0, - InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), - ), - tx_c_fee, - flags, - locktime, - )?; - let tx_c_id = tx_c.get_id(); - log::debug!("tx_c_id : {}", tx_c_id.get()); - log::debug!("tx_c fee : {:?}", mempool.try_get_fee(&tx_c)?); - mempool.add_transaction(tx_c)?; - - let entry_a = mempool.store.txs_by_id.get(&tx_a_id.get()).expect("tx_a"); - log::debug!("entry a has score {:?}", entry_a.fees_with_descendants); - let entry_b = mempool.store.txs_by_id.get(&tx_b_id.get()).expect("tx_b"); - log::debug!("entry b has score {:?}", entry_b.fees_with_descendants); - let entry_c = mempool.store.txs_by_id.get(&tx_c_id.get()).expect("tx_c").clone(); - log::debug!("entry c has score {:?}", entry_c.fees_with_descendants); - assert_eq!(entry_a.fee, entry_a.fees_with_descendants); - assert_eq!( - entry_b.fees_with_descendants, - (entry_b.fee + entry_c.fee).unwrap() - ); - assert!(!mempool.store.txs_by_descendant_score.contains_key(&tx_b_fee.into())); - log::debug!( - "raw_txs_by_descendant_score {:?}", - mempool.store.txs_by_descendant_score - ); - check_txs_sorted_by_descendant_sore(&mempool); - - mempool.drop_transaction(&entry_c.tx.get_id()); - assert!(!mempool.store.txs_by_descendant_score.contains_key(&tx_c_fee.into())); - let entry_b = mempool.store.txs_by_id.get(&tx_b_id.get()).expect("tx_b"); - assert_eq!(entry_b.fees_with_descendants, entry_b.fee); - - check_txs_sorted_by_descendant_sore(&mempool); - - Ok(()) - } - - fn check_txs_sorted_by_descendant_sore( - mempool: &MempoolImpl, - ) { - let txs_by_descendant_score = - mempool.store.txs_by_descendant_score.values().flatten().collect::>(); - for i in 0..(txs_by_descendant_score.len() - 1) { - log::debug!("i = {}", i); - let tx_id = txs_by_descendant_score.get(i).unwrap(); - let next_tx_id = txs_by_descendant_score.get(i + 1).unwrap(); - let entry_score = mempool.store.txs_by_id.get(tx_id).unwrap().fees_with_descendants; - let next_entry_score = - mempool.store.txs_by_id.get(next_tx_id).unwrap().fees_with_descendants; - log::debug!("entry_score: {:?}", entry_score); - log::debug!("next_entry_score: {:?}", next_entry_score); - assert!(entry_score <= next_entry_score) - } - } - - #[test] - fn descendant_of_expired_entry() -> anyhow::Result<()> { - let mock_clock = MockClock::new(); - logging::init_logging::<&str>(None); - - let mut mempool = MempoolImpl::create( - ChainStateMock::new(), - mock_clock.clone(), - SystemUsageEstimator {}, - ); - - let num_inputs = 1; - let num_outputs = 2; - let fee = get_relay_fee_from_tx_size(estimate_tx_size(num_inputs, num_outputs)); - let parent = TxGenerator::new() - .with_num_inputs(num_inputs) - .with_num_outputs(num_outputs) - .with_fee(Amount::from_atoms(fee)) - .generate_tx(&mempool)?; - let parent_id = parent.get_id(); - mempool.add_transaction(parent)?; - - let flags = 0; - let locktime = 0; - let outpoint_source_id = OutPointSourceId::Transaction(parent_id); - let child = tx_spend_input( - &mempool, - TxInput::new( - outpoint_source_id, - 0, - InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), - ), - None, - flags, - locktime, - )?; - let child_id = child.get_id(); - mock_clock.set(DEFAULT_MEMPOOL_EXPIRY + Duration::new(1, 0)); - - assert!(matches!( - mempool.add_transaction(child), - Err(Error::TxValidationError( - TxValidationError::DescendantOfExpiredTransaction - )) - )); - - assert!(!mempool.contains_transaction(&parent_id)); - assert!(!mempool.contains_transaction(&child_id)); - Ok(()) - } - - #[test] - fn mempool_full() -> anyhow::Result<()> { - logging::init_logging::<&str>(None); - let mut mock_usage = MockGetMemoryUsage::new(); - mock_usage - .expect_get_memory_usage() - .times(1) - .return_const(MAX_MEMPOOL_SIZE_BYTES + 1); - - let chain_state = ChainStateMock::new(); - let mut mempool = MempoolImpl::create(chain_state, SystemClock, mock_usage); - - let tx = TxGenerator::new().generate_tx(&mempool)?; - log::debug!("mempool_full: tx has is {}", tx.get_id().get()); - assert!(matches!( - mempool.add_transaction(tx), - Err(Error::MempoolFull) - )); - Ok(()) - } - - #[test] - fn no_empty_bags_in_descendant_score_index() -> anyhow::Result<()> { - let mut mempool = setup(); - - let num_inputs = 1; - let num_outputs = 100; - let fee = get_relay_fee_from_tx_size(estimate_tx_size(num_inputs, num_outputs)); - let parent = TxGenerator::new() - .with_num_inputs(num_inputs) - .with_num_outputs(num_outputs) - .with_fee(Amount::from_atoms(fee)) - .generate_tx(&mempool)?; - let parent_id = parent.get_id(); - - let outpoint_source_id = OutPointSourceId::Transaction(parent.get_id()); - mempool.add_transaction(parent)?; - let num_child_txs = num_outputs; - let flags = 0; - let locktime = 0; - let txs = (0..num_child_txs) - .into_iter() - .map(|i| { - tx_spend_input( - &mempool, - TxInput::new( - outpoint_source_id.clone(), - u32::try_from(i).unwrap(), - InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), - ), - Amount::from_atoms(fee + u128::try_from(i).unwrap()), - flags, - locktime, - ) - }) - .collect::, _>>()?; - let ids = txs.iter().map(|tx| tx.get_id()).collect::>(); - - for tx in txs { - mempool.add_transaction(tx)?; - } - - mempool.drop_transaction(&parent_id); - for id in ids { - mempool.drop_transaction(&id) - } - assert!(mempool.store.txs_by_descendant_score.is_empty()); - Ok(()) - } -} +mod tests; diff --git a/mempool/src/pool/tests.rs b/mempool/src/pool/tests.rs new file mode 100644 index 0000000000..5277a40ad6 --- /dev/null +++ b/mempool/src/pool/tests.rs @@ -0,0 +1,1787 @@ +use super::*; +use common::chain::signature::inputsig::InputWitness; +use common::chain::transaction::{Destination, TxInput, TxOutput}; +use common::chain::OutPointSourceId; +use common::chain::OutputPurpose; +use core::panic; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; + +const DUMMY_WITNESS_MSG: &[u8] = b"dummy_witness_msg"; + +#[derive(Debug, PartialEq, Eq, Clone)] +struct ValuedOutPoint { + outpoint: OutPoint, + value: Amount, +} + +impl std::cmp::PartialOrd for ValuedOutPoint { + fn partial_cmp(&self, other: &Self) -> Option { + other.value.partial_cmp(&self.value) + } +} + +impl std::cmp::Ord for ValuedOutPoint { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + other.value.cmp(&self.value) + } +} + +fn dummy_input() -> TxInput { + let outpoint_source_id = OutPointSourceId::Transaction(Id::new(H256::zero())); + let output_index = 0; + let witness = DUMMY_WITNESS_MSG.to_vec(); + TxInput::new( + outpoint_source_id, + output_index, + InputWitness::NoSignature(Some(witness)), + ) +} + +fn dummy_output() -> TxOutput { + let value = Amount::from_atoms(0); + let purpose = OutputPurpose::Transfer(Destination::AnyoneCanSpend); + TxOutput::new(value, purpose) +} + +fn estimate_tx_size(num_inputs: usize, num_outputs: usize) -> usize { + let inputs = (0..num_inputs).into_iter().map(|_| dummy_input()).collect(); + let outputs = (0..num_outputs).into_iter().map(|_| dummy_output()).collect(); + let flags = 0; + let locktime = 0; + let size = Transaction::new(flags, inputs, outputs, locktime).unwrap().encoded_size(); + // Take twice the encoded size of the dummy tx.Real Txs are larger than these dummy ones, + // but taking 3 times the size seems to ensure our txs won't fail the minimum relay fee + // validation (see the function `pays_minimum_relay_fees`) + let result = 3 * size; + log::debug!( + "estimated size for tx with {} inputs and {} outputs: {}", + num_inputs, + num_outputs, + result + ); + result +} + +#[test] +fn dummy_size() { + logging::init_logging::<&str>(None); + log::debug!("1, 1: {}", estimate_tx_size(1, 1)); + log::debug!("1, 2: {}", estimate_tx_size(1, 2)); + log::debug!("1, 400: {}", estimate_tx_size(1, 400)); +} + +#[test] +fn real_size() -> anyhow::Result<()> { + let mempool = setup(); + let tx = TxGenerator::new() + .with_num_inputs(1) + .with_num_outputs(400) + .generate_tx(&mempool)?; + log::debug!("real size of tx {}", tx.encoded_size()); + Ok(()) +} + +fn valued_outpoint( + tx_id: &Id, + outpoint_index: u32, + output: &TxOutput, +) -> ValuedOutPoint { + let outpoint_source_id = OutPointSourceId::Transaction(*tx_id); + let outpoint = OutPoint::new(outpoint_source_id, outpoint_index); + let value = output.value(); + ValuedOutPoint { outpoint, value } +} + +pub(crate) fn create_genesis_tx() -> Transaction { + const TOTAL_SUPPLY: u128 = 10_000_000_000_000; + let genesis_message = b"".to_vec(); + let outpoint_source_id = OutPointSourceId::Transaction(Id::new(H256::zero())); + let input = TxInput::new( + outpoint_source_id, + 0, + InputWitness::NoSignature(Some(genesis_message)), + ); + let output = TxOutput::new( + Amount::from_atoms(TOTAL_SUPPLY), + OutputPurpose::Transfer(Destination::AnyoneCanSpend), + ); + Transaction::new(0, vec![input], vec![output], 0) + .expect("Failed to create genesis coinbase transaction") +} + +impl TxMempoolEntry { + fn outpoints_created(&self) -> BTreeSet { + let id = self.tx.get_id(); + std::iter::repeat(id) + .zip(self.tx.outputs().iter().enumerate()) + .map(|(id, (index, output))| valued_outpoint(&id, index as u32, output)) + .collect() + } +} + +impl MempoolStore { + fn unconfirmed_outpoints(&self) -> BTreeSet { + self.txs_by_id + .values() + .cloned() + .flat_map(|entry| entry.outpoints_created()) + .collect() + } +} + +impl MempoolImpl +where + T: GetTime, + M: GetMemoryUsage, +{ + fn available_outpoints(&self, allow_double_spend: bool) -> BTreeSet { + let mut available = self + .store + .unconfirmed_outpoints() + .into_iter() + .chain(self.chain_state.confirmed_outpoints()) + .collect::>(); + if !allow_double_spend { + available.retain(|valued_outpoint| { + !self.store.spender_txs.contains_key(&valued_outpoint.outpoint) + }); + } + available + } + + fn get_input_value(&self, input: &TxInput) -> anyhow::Result { + let allow_double_spend = true; + self.available_outpoints(allow_double_spend) + .iter() + .find_map(|valued_outpoint| { + (valued_outpoint.outpoint == *input.outpoint()).then(|| valued_outpoint.value) + }) + .ok_or_else(|| anyhow::anyhow!("No such unconfirmed output")) + } + + fn get_minimum_rolling_fee(&self) -> FeeRate { + self.rolling_fee_rate.get().rolling_minimum_fee_rate + } + + fn process_block(&mut self, tx_id: &Id) -> anyhow::Result<()> { + let mut chain_state = self.chain_state.clone(); + chain_state.add_confirmed_tx( + self.store + .txs_by_id + .get(&tx_id.get()) + .cloned() + .ok_or_else(|| { + anyhow::anyhow!("process_block: tx {} not found in mempool", tx_id.get()) + })? + .tx, + ); + log::debug!("Setting tip to {:?}", chain_state); + self.new_tip_set(chain_state); + self.drop_transaction(tx_id); + Ok(()) + } +} + +#[derive(Debug, Clone)] +pub(crate) struct ChainStateMock { + confirmed_txs: HashMap, + available_outpoints: BTreeSet, +} + +impl ChainStateMock { + pub(crate) fn new() -> Self { + let genesis_tx = create_genesis_tx(); + let outpoint_source_id = OutPointSourceId::Transaction(genesis_tx.get_id()); + let outpoints = genesis_tx + .outputs() + .iter() + .enumerate() + .map(|(index, _)| OutPoint::new(outpoint_source_id.clone(), index as u32)) + .collect(); + Self { + confirmed_txs: std::iter::once((genesis_tx.get_id().get(), genesis_tx)).collect(), + available_outpoints: outpoints, + } + } + + fn confirmed_txs(&self) -> &HashMap { + &self.confirmed_txs + } + + fn confirmed_outpoints(&self) -> BTreeSet { + self.available_outpoints + .iter() + .map(|outpoint| { + let tx_id = outpoint + .tx_id() + .get_tx_id() + .cloned() + .expect("Outpoints in these tests are created from TXs"); + let index = outpoint.output_index(); + let tx = self.confirmed_txs.get(&tx_id.get()).expect("Inconsistent Chain State"); + let output = tx + .outputs() + .get(index as usize) + .expect("Inconsistent Chain State: output not found"); + + valued_outpoint(&tx_id, index, output) + }) + .collect() + } + + fn add_confirmed_tx(&mut self, tx: Transaction) { + let outpoints_spent: BTreeSet<_> = + tx.inputs().iter().map(|input| input.outpoint()).collect(); + let outpoints_created: BTreeSet<_> = tx + .outputs() + .iter() + .enumerate() + .map(|(i, _)| OutPoint::new(OutPointSourceId::Transaction(tx.get_id()), i as u32)) + .collect(); + self.available_outpoints.extend(outpoints_created); + self.available_outpoints.retain(|outpoint| !outpoints_spent.contains(outpoint)); + self.confirmed_txs.insert(tx.get_id().get(), tx); + } +} + +impl ChainState for ChainStateMock { + fn contains_outpoint(&self, outpoint: &OutPoint) -> bool { + self.available_outpoints.iter().any(|value| *value == *outpoint) + } + + fn get_outpoint_value(&self, outpoint: &OutPoint) -> Result { + self.confirmed_txs + .get(&outpoint.tx_id().get_tx_id().expect("Not coinbase").get()) + .ok_or_else(|| anyhow::anyhow!("tx for outpoint sought in chain state, not found")) + .and_then(|tx| { + tx.outputs() + .get(outpoint.output_index() as usize) + .ok_or_else(|| anyhow::anyhow!("outpoint index out of bounds")) + .map(|output| output.value()) + }) + } +} + +/* FIXME The second call in the following flow sometimes returns TransactionAlreadyInMempool +let tx1 = TxGenerator::new(&mempool).generate_tx()?; +mempool.add_transaction(tx1)?; + +let tx2 = TxGenerator::new(&mempool).generate_tx()?; +mempool.add_transaction(tx2)?; +*/ +struct TxGenerator { + coin_pool: BTreeSet, + num_inputs: usize, + num_outputs: usize, + tx_fee: Option, + replaceable: bool, + allow_double_spend: bool, +} + +impl TxGenerator { + fn with_num_inputs(mut self, num_inputs: usize) -> Self { + self.num_inputs = num_inputs; + self + } + + fn with_num_outputs(mut self, num_outputs: usize) -> Self { + self.num_outputs = num_outputs; + self + } + + fn replaceable(mut self) -> Self { + self.replaceable = true; + self + } + + fn with_fee(mut self, fee: Amount) -> Self { + self.tx_fee = Some(fee); + self + } + + fn new() -> Self { + Self { + coin_pool: BTreeSet::new(), + num_inputs: 1, + num_outputs: 1, + tx_fee: None, + replaceable: false, + allow_double_spend: false, + } + } + + fn generate_tx( + &mut self, + mempool: &MempoolImpl, + ) -> anyhow::Result { + self.coin_pool = mempool.available_outpoints(self.allow_double_spend); + let fee = if let Some(tx_fee) = self.tx_fee { + tx_fee + } else { + Amount::from_atoms(get_relay_fee_from_tx_size(estimate_tx_size( + self.num_inputs, + self.num_outputs, + ))) + }; + log::debug!( + "Trying to build a tx with {} inputs, {} outputs, and a fee of {:?}", + self.num_inputs, + self.num_outputs, + fee + ); + let valued_inputs = self.generate_tx_inputs(fee)?; + let outputs = self.generate_tx_outputs(&valued_inputs, fee)?; + let locktime = 0; + let flags = if self.replaceable { 1 } else { 0 }; + let (inputs, _): (Vec, Vec) = valued_inputs.into_iter().unzip(); + let spent_outpoints = inputs.iter().map(|input| input.outpoint()).collect::>(); + self.coin_pool + .retain(|outpoint| !spent_outpoints.iter().any(|spent| **spent == outpoint.outpoint)); + let tx = Transaction::new(flags, inputs, outputs.clone(), locktime) + .map_err(anyhow::Error::from)?; + self.coin_pool.extend( + std::iter::repeat(tx.get_id()) + .zip(outputs.iter().enumerate()) + .map(|(id, (i, output))| valued_outpoint(&id, i as u32, output)), + ); + + Ok(tx) + } + + fn generate_tx_inputs(&mut self, fee: Amount) -> anyhow::Result> { + Ok(self + .get_unspent_outpoints(self.num_inputs, fee)? + .iter() + .map(|valued_outpoint| { + let ValuedOutPoint { outpoint, value } = valued_outpoint; + ( + TxInput::new( + outpoint.tx_id(), + outpoint.output_index(), + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ), + *value, + ) + }) + .collect()) + } + + fn generate_tx_outputs( + &self, + inputs: &[(TxInput, Amount)], + tx_fee: Amount, + ) -> anyhow::Result> { + if self.num_outputs == 0 { + return Ok(vec![]); + } + + let inputs: Vec<_> = inputs.to_owned(); + let (inputs, values): (Vec, Vec) = inputs.into_iter().unzip(); + if inputs.is_empty() { + return Ok(vec![]); + } + let sum_of_inputs = + values.into_iter().sum::>().expect("Overflow in sum of input values"); + + let total_to_spend = (sum_of_inputs - tx_fee).ok_or_else(||anyhow::anyhow!( + "generate_tx_outputs: underflow computing total_to_spend - sum_of_inputs = {:?}, fee = {:?}", sum_of_inputs, tx_fee + ))?; + + let value = (sum_of_inputs / u128::try_from(self.num_outputs).expect("conversion")) + .expect("not dividing by zero"); + + let mut left_to_spend = total_to_spend; + let mut outputs = Vec::new(); + + for _ in 0..self.num_outputs - 1 { + outputs.push(TxOutput::new( + value, + OutputPurpose::Transfer(Destination::AnyoneCanSpend), + )); + left_to_spend = (left_to_spend - value).expect("subtraction failed"); + } + + outputs.push(TxOutput::new( + left_to_spend, + OutputPurpose::Transfer(Destination::AnyoneCanSpend), + )); + Ok(outputs) + } + + fn get_unspent_outpoints( + &self, + num_outputs: usize, + fee: Amount, + ) -> anyhow::Result> { + log::debug!( + "get_unspent_outpoints: num_outputs: {}, fee: {:?}", + num_outputs, + fee + ); + let num_available_outpoints = self.coin_pool.len(); + let outpoints: Vec<_> = (num_available_outpoints >= num_outputs) + .then(|| self.coin_pool.iter().take(num_outputs).cloned().collect()) + .ok_or_else(|| anyhow::anyhow!("no outpoints left"))?; + let sum_of_outputs = outpoints + .iter() + .map(|valued_outpoint| valued_outpoint.value) + .sum::>() + .expect("sum error"); + if fee > sum_of_outputs { + Err(anyhow::Error::msg( + "get_unspent_outpoints:: fee is {:?} but sum of outputs is {:?}", + )) + } else { + Ok(outpoints) + } + } +} + +fn get_relay_fee_from_tx_size(tx_size: usize) -> u128 { + u128::try_from(tx_size * RELAY_FEE_PER_BYTE).expect("relay fee overflow") +} + +#[test] +fn add_single_tx() -> anyhow::Result<()> { + let mut mempool = setup(); + + let genesis_tx = mempool + .chain_state + .confirmed_txs() + .values() + .next() + .expect("genesis tx not found"); + + let outpoint_source_id = OutPointSourceId::Transaction(genesis_tx.get_id()); + + let flags = 0; + let locktime = 0; + let input = TxInput::new( + outpoint_source_id, + 0, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ); + let relay_fee = Amount::from_atoms(get_relay_fee_from_tx_size(TX_SPEND_INPUT_SIZE)); + let tx = tx_spend_input(&mempool, input, relay_fee, flags, locktime)?; + + let tx_clone = tx.clone(); + let tx_id = tx.get_id(); + mempool.add_transaction(tx)?; + assert!(mempool.contains_transaction(&tx_id)); + let all_txs = mempool.get_all(); + assert_eq!(all_txs, vec![&tx_clone]); + mempool.drop_transaction(&tx_id); + assert!(!mempool.contains_transaction(&tx_id)); + let all_txs = mempool.get_all(); + assert_eq!(all_txs, Vec::<&Transaction>::new()); + Ok(()) +} + +#[test] +fn txs_sorted() -> anyhow::Result<()> { + let mut mempool = setup(); + let mut tx_generator = TxGenerator::new(); + let target_txs = 10; + + for _ in 0..target_txs { + match tx_generator.generate_tx(&mempool) { + Ok(tx) => { + mempool.add_transaction(tx.clone())?; + } + _ => break, + } + } + + let fees = mempool + .get_all() + .iter() + .map(|tx| mempool.try_get_fee(tx)) + .collect::, _>>()?; + let mut fees_sorted = fees.clone(); + fees_sorted.sort_by(|a, b| b.cmp(a)); + assert_eq!(fees, fees_sorted); + Ok(()) +} + +#[test] +fn tx_no_inputs() -> anyhow::Result<()> { + let mut mempool = setup(); + let tx = TxGenerator::new() + .with_num_inputs(0) + .with_fee(Amount::from_atoms(0)) + .generate_tx(&mempool) + .expect("generate_tx failed"); + assert!(matches!( + mempool.add_transaction(tx), + Err(Error::TxValidationError(TxValidationError::NoInputs)) + )); + Ok(()) +} + +fn setup() -> MempoolImpl { + logging::init_logging::<&str>(None); + MempoolImpl::create( + ChainStateMock::new(), + SystemClock {}, + SystemUsageEstimator {}, + ) +} + +#[test] +fn tx_no_outputs() -> anyhow::Result<()> { + let mut mempool = setup(); + let tx = TxGenerator::new() + .with_num_outputs(0) + .generate_tx(&mempool) + .expect("generate_tx failed"); + assert!(matches!( + mempool.add_transaction(tx), + Err(Error::TxValidationError(TxValidationError::NoOutputs)) + )); + Ok(()) +} + +#[test] +fn tx_duplicate_inputs() -> anyhow::Result<()> { + let mut mempool = setup(); + + let genesis_tx = mempool + .chain_state + .confirmed_txs() + .values() + .next() + .expect("genesis tx not found"); + + let outpoint_source_id = OutPointSourceId::Transaction(genesis_tx.get_id()); + let input = TxInput::new( + outpoint_source_id.clone(), + 0, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ); + let witness = b"attempted_double_spend".to_vec(); + let duplicate_input = TxInput::new( + outpoint_source_id, + 0, + InputWitness::NoSignature(Some(witness)), + ); + let flags = 0; + let locktime = 0; + let outputs = tx_spend_input(&mempool, input.clone(), None, flags, locktime)? + .outputs() + .clone(); + let inputs = vec![input, duplicate_input]; + let tx = Transaction::new(flags, inputs, outputs, locktime)?; + + assert!(matches!( + mempool.add_transaction(tx), + Err(Error::TxValidationError(TxValidationError::DuplicateInputs)) + )); + Ok(()) +} + +#[test] +fn tx_already_in_mempool() -> anyhow::Result<()> { + let mut mempool = setup(); + + let genesis_tx = mempool + .chain_state + .confirmed_txs() + .values() + .next() + .expect("genesis tx not found"); + + let outpoint_source_id = OutPointSourceId::Transaction(genesis_tx.get_id()); + let input = TxInput::new( + outpoint_source_id, + 0, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ); + + let flags = 0; + let locktime = 0; + let tx = tx_spend_input(&mempool, input, None, flags, locktime)?; + + mempool.add_transaction(tx.clone())?; + assert!(matches!( + mempool.add_transaction(tx), + Err(Error::TxValidationError( + TxValidationError::TransactionAlreadyInMempool + )) + )); + Ok(()) +} + +#[test] +fn outpoint_not_found() -> anyhow::Result<()> { + let mut mempool = setup(); + + let genesis_tx = mempool + .chain_state + .confirmed_txs() + .values() + .next() + .expect("genesis tx not found"); + + let outpoint_source_id = OutPointSourceId::Transaction(genesis_tx.get_id()); + + let good_input = TxInput::new( + outpoint_source_id.clone(), + 0, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ); + let flags = 0; + let locktime = 0; + let outputs = tx_spend_input(&mempool, good_input, None, flags, locktime)?.outputs().clone(); + + let bad_outpoint_index = 1; + let bad_input = TxInput::new( + outpoint_source_id, + bad_outpoint_index, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ); + + let inputs = vec![bad_input]; + let tx = Transaction::new(flags, inputs, outputs, locktime)?; + + assert!(matches!( + mempool.add_transaction(tx), + Err(Error::TxValidationError( + TxValidationError::OutPointNotFound { .. } + )) + )); + + Ok(()) +} + +#[test] +fn tx_too_big() -> anyhow::Result<()> { + let mut mempool = setup(); + let tx = TxGenerator::new() + .with_num_outputs(400_000) + .generate_tx(&mempool) + .expect("generate_tx failed"); + assert!(matches!( + mempool.add_transaction(tx), + Err(Error::TxValidationError( + TxValidationError::ExceedsMaxBlockSize + )) + )); + Ok(()) +} + +fn test_replace_tx(original_fee: Amount, replacement_fee: Amount) -> Result<(), Error> { + eprintln!( + "tx_replace_tx: original_fee: {:?}, replacement_fee {:?}", + original_fee, replacement_fee + ); + let mut mempool = setup(); + let outpoint = mempool + .available_outpoints(true) + .iter() + .next() + .expect("there should be an outpoint since setup creates the genesis transaction") + .outpoint + .clone(); + + let outpoint_source_id = + OutPointSourceId::from(*outpoint.tx_id().get_tx_id().expect("Not Coinbase")); + + let input = TxInput::new( + outpoint_source_id, + 0, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ); + let flags = 1; + let locktime = 0; + let original = tx_spend_input(&mempool, input.clone(), original_fee, flags, locktime) + .expect("should be able to spend here"); + let original_id = original.get_id(); + eprintln!("created a tx with fee {:?}", mempool.try_get_fee(&original)); + mempool.add_transaction(original)?; + + let flags = 0; + let replacement = tx_spend_input(&mempool, input, replacement_fee, flags, locktime) + .expect("should be able to spend here"); + eprintln!( + "created a replacement with fee {:?}", + mempool.try_get_fee(&replacement) + ); + mempool.add_transaction(replacement)?; + assert!(!mempool.contains_transaction(&original_id)); + + Ok(()) +} + +#[test] +fn try_replace_irreplaceable() -> anyhow::Result<()> { + let mut mempool = setup(); + let outpoint = mempool + .available_outpoints(true) + .iter() + .next() + .expect("there should be an outpoint since setup creates the genesis transaction") + .outpoint + .clone(); + + let outpoint_source_id = + OutPointSourceId::from(*outpoint.tx_id().get_tx_id().expect("Not Coinbase")); + + let input = TxInput::new( + outpoint_source_id, + 0, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ); + let flags = 0; + let locktime = 0; + let original_fee = Amount::from_atoms(get_relay_fee_from_tx_size(TX_SPEND_INPUT_SIZE)); + let original = tx_spend_input(&mempool, input.clone(), original_fee, flags, locktime) + .expect("should be able to spend here"); + let original_id = original.get_id(); + mempool.add_transaction(original)?; + + let flags = 0; + let replacement_fee = (original_fee + Amount::from_atoms(1000)).unwrap(); + let replacement = tx_spend_input(&mempool, input, replacement_fee, flags, locktime) + .expect("should be able to spend here"); + assert!(matches!( + mempool.add_transaction(replacement.clone()), + Err(Error::TxValidationError( + TxValidationError::ConflictWithIrreplaceableTransaction + )) + )); + + mempool.drop_transaction(&original_id); + mempool.add_transaction(replacement)?; + + Ok(()) +} + +#[test] +fn tx_replace() -> anyhow::Result<()> { + let relay_fee = get_relay_fee_from_tx_size(TX_SPEND_INPUT_SIZE); + let replacement_fee = Amount::from_atoms(relay_fee + 100); + test_replace_tx(Amount::from_atoms(100), replacement_fee)?; + let res = test_replace_tx(Amount::from_atoms(300), replacement_fee); + assert!(matches!( + res, + Err(Error::TxValidationError( + TxValidationError::InsufficientFeesToRelayRBF + )) + )); + let res = test_replace_tx(Amount::from_atoms(100), Amount::from_atoms(100)); + assert!(matches!( + res, + Err(Error::TxValidationError( + TxValidationError::ReplacementFeeLowerThanOriginal { .. } + )) + )); + let res = test_replace_tx(Amount::from_atoms(100), Amount::from_atoms(90)); + assert!(matches!( + res, + Err(Error::TxValidationError( + TxValidationError::ReplacementFeeLowerThanOriginal { .. } + )) + )); + Ok(()) +} + +#[test] +fn tx_replace_child() -> anyhow::Result<()> { + let mut mempool = setup(); + let tx = TxGenerator::new() + .replaceable() + .generate_tx(&mempool) + .expect("generate_replaceable_tx"); + mempool.add_transaction(tx.clone())?; + + let outpoint_source_id = OutPointSourceId::Transaction(tx.get_id()); + let child_tx_input = TxInput::new( + outpoint_source_id, + 0, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ); + // We want to test that even though child_tx doesn't signal replaceability directly, it is replaceable because its parent signalled replaceability + // replaced + let flags = 0; + let locktime = 0; + let child_tx = tx_spend_input( + &mempool, + child_tx_input.clone(), + Amount::from_atoms(100), + flags, + locktime, + )?; + mempool.add_transaction(child_tx)?; + + let relay_fee = get_relay_fee_from_tx_size(TX_SPEND_INPUT_SIZE); + let replacement_fee = Amount::from_atoms(relay_fee + 100); + let replacement_tx = + tx_spend_input(&mempool, child_tx_input, replacement_fee, flags, locktime)?; + mempool.add_transaction(replacement_tx)?; + Ok(()) +} + +// To test our validation of BIP125 Rule#4 (replacement transaction pays for its own bandwidth), we need to know the necessary relay fee before creating the transaction. The relay fee depends on the size of the transaction. The usual way to get the size of a transaction is to call `tx.encoded_size` but we cannot do this until we have created the transaction itself. To get around this cycle, we have precomputed the size of all transaction created by `tx_spend_input`. This value will be the same for all transactions created by this function. +const TX_SPEND_INPUT_SIZE: usize = 213; + +fn tx_spend_input( + mempool: &MempoolImpl, + input: TxInput, + fee: impl Into>, + flags: u32, + locktime: u32, +) -> anyhow::Result { + let fee = fee.into().map_or_else( + || Amount::from_atoms(get_relay_fee_from_tx_size(estimate_tx_size(1, 2))), + std::convert::identity, + ); + tx_spend_several_inputs(mempool, &[input], fee, flags, locktime) +} + +fn tx_spend_several_inputs( + mempool: &MempoolImpl, + inputs: &[TxInput], + fee: Amount, + flags: u32, + locktime: u32, +) -> anyhow::Result { + let input_value = inputs + .iter() + .map(|input| mempool.get_input_value(input)) + .collect::, _>>()? + .into_iter() + .sum::>() + .ok_or_else(|| { + let msg = String::from("tx_spend_input: overflow"); + log::error!("{}", msg); + anyhow::Error::msg(msg) + })?; + + let available_for_spending = (input_value - fee).ok_or_else(|| { + let msg = format!( + "tx_spend_several_inputs: input_value ({:?}) lower than fee ({:?})", + input_value, fee + ); + log::error!("{}", msg); + anyhow::Error::msg(msg) + })?; + let spent = (available_for_spending / 2).expect("division error"); + + let change = (available_for_spending - spent).ok_or_else(|| { + let msg = String::from("Error computing change"); + anyhow::Error::msg(msg) + })?; + + Transaction::new( + flags, + inputs.to_owned(), + vec![ + TxOutput::new(spent, OutputPurpose::Transfer(Destination::AnyoneCanSpend)), + TxOutput::new(change, OutputPurpose::Transfer(Destination::AnyoneCanSpend)), + ], + locktime, + ) + .map_err(Into::into) +} + +#[test] +fn one_ancestor_signal_is_enough() -> anyhow::Result<()> { + let mut mempool = setup(); + let tx = TxGenerator::new() + .with_num_outputs(2) + .generate_tx(&mempool) + .expect("generate_replaceable_tx"); + + mempool.add_transaction(tx.clone())?; + + let flags_replaceable = 1; + let flags_irreplaceable = 0; + let locktime = 0; + + let outpoint_source_id = OutPointSourceId::Transaction(tx.get_id()); + let ancestor_with_signal = tx_spend_input( + &mempool, + TxInput::new( + outpoint_source_id.clone(), + 0, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ), + None, + flags_replaceable, + locktime, + )?; + + let ancestor_without_signal = tx_spend_input( + &mempool, + TxInput::new( + outpoint_source_id, + 1, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ), + None, + flags_irreplaceable, + locktime, + )?; + + mempool.add_transaction(ancestor_with_signal.clone())?; + mempool.add_transaction(ancestor_without_signal.clone())?; + + let input_with_replaceable_parent = TxInput::new( + OutPointSourceId::Transaction(ancestor_with_signal.get_id()), + 0, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ); + + let input_with_irreplaceable_parent = TxInput::new( + OutPointSourceId::Transaction(ancestor_without_signal.get_id()), + 0, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ); + + // TODO compute minimum necessary relay fee instead of just overestimating it + let original_fee = Amount::from_atoms(200); + let dummy_output = TxOutput::new( + original_fee, + OutputPurpose::Transfer(Destination::AnyoneCanSpend), + ); + let replaced_tx = tx_spend_several_inputs( + &mempool, + &[input_with_irreplaceable_parent.clone(), input_with_replaceable_parent], + original_fee, + flags_irreplaceable, + locktime, + )?; + let replaced_tx_id = replaced_tx.get_id(); + + mempool.add_transaction(replaced_tx)?; + + let replacing_tx = Transaction::new( + flags_irreplaceable, + vec![input_with_irreplaceable_parent], + vec![dummy_output], + locktime, + )?; + + mempool.add_transaction(replacing_tx)?; + assert!(!mempool.contains_transaction(&replaced_tx_id)); + + Ok(()) +} + +#[test] +fn tx_mempool_entry() -> anyhow::Result<()> { + use common::primitives::time; + let mut mempool = setup(); + // Input different flag values just to make the hashes of these dummy transactions + // different + let txs = (1..=6) + .into_iter() + .map(|i| Transaction::new(i, vec![], vec![], 0).unwrap_or_else(|_| panic!("tx {}", i))) + .collect::>(); + let fee = Amount::from_atoms(0); + + // Generation 1 + let tx1_parents = BTreeSet::default(); + let entry1 = TxMempoolEntry::new(txs.get(0).unwrap().clone(), fee, tx1_parents, time::get()); + let tx2_parents = BTreeSet::default(); + let entry2 = TxMempoolEntry::new(txs.get(1).unwrap().clone(), fee, tx2_parents, time::get()); + + // Generation 2 + let tx3_parents = vec![entry1.tx_id(), entry2.tx_id()].into_iter().collect(); + let entry3 = TxMempoolEntry::new(txs.get(2).unwrap().clone(), fee, tx3_parents, time::get()); + + // Generation 3 + let tx4_parents = vec![entry3.tx_id()].into_iter().collect(); + let tx5_parents = vec![entry3.tx_id()].into_iter().collect(); + let entry4 = TxMempoolEntry::new(txs.get(3).unwrap().clone(), fee, tx4_parents, time::get()); + let entry5 = TxMempoolEntry::new(txs.get(4).unwrap().clone(), fee, tx5_parents, time::get()); + + // Generation 4 + let tx6_parents = vec![entry3.tx_id(), entry4.tx_id(), entry5.tx_id()].into_iter().collect(); + let entry6 = TxMempoolEntry::new(txs.get(5).unwrap().clone(), fee, tx6_parents, time::get()); + + let entries = vec![entry1, entry2, entry3, entry4, entry5, entry6]; + let ids = entries.clone().into_iter().map(|entry| entry.tx_id()).collect::>(); + + for entry in entries.into_iter() { + mempool.store.add_tx(entry)?; + } + + let entry1 = mempool.store.get_entry(ids.get(0).expect("index")).expect("entry"); + let entry2 = mempool.store.get_entry(ids.get(1).expect("index")).expect("entry"); + let entry3 = mempool.store.get_entry(ids.get(2).expect("index")).expect("entry"); + let entry4 = mempool.store.get_entry(ids.get(3).expect("index")).expect("entry"); + let entry5 = mempool.store.get_entry(ids.get(4).expect("index")).expect("entry"); + let entry6 = mempool.store.get_entry(ids.get(5).expect("index")).expect("entry"); + assert_eq!(entry1.unconfirmed_ancestors(&mempool.store).0.len(), 0); + assert_eq!(entry2.unconfirmed_ancestors(&mempool.store).0.len(), 0); + assert_eq!(entry3.unconfirmed_ancestors(&mempool.store).0.len(), 2); + assert_eq!(entry4.unconfirmed_ancestors(&mempool.store).0.len(), 3); + assert_eq!(entry5.unconfirmed_ancestors(&mempool.store).0.len(), 3); + assert_eq!(entry6.unconfirmed_ancestors(&mempool.store).0.len(), 5); + + assert_eq!(entry1.count_with_descendants(), 5); + assert_eq!(entry2.count_with_descendants(), 5); + assert_eq!(entry3.count_with_descendants(), 4); + assert_eq!(entry4.count_with_descendants(), 2); + assert_eq!(entry5.count_with_descendants(), 2); + assert_eq!(entry6.count_with_descendants(), 1); + + Ok(()) +} + +fn test_bip125_max_replacements( + mempool: &mut MempoolImpl, + num_potential_replacements: usize, +) -> anyhow::Result<()> { + let tx = TxGenerator::new() + .with_num_outputs(num_potential_replacements - 1) + .replaceable() + .generate_tx(mempool) + .expect("generate_tx failed"); + let input = tx.inputs().first().expect("one input").clone(); + let outputs = tx.outputs().clone(); + let tx_id = tx.get_id(); + mempool.add_transaction(tx)?; + + let flags = 0; + let locktime = 0; + let outpoint_source_id = OutPointSourceId::Transaction(tx_id); + let fee = get_relay_fee_from_tx_size(TX_SPEND_INPUT_SIZE); + for (index, _) in outputs.iter().enumerate() { + let input = TxInput::new( + outpoint_source_id.clone(), + index.try_into().unwrap(), + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ); + let tx = tx_spend_input(mempool, input, Amount::from_atoms(fee), flags, locktime)?; + mempool.add_transaction(tx)?; + } + let mempool_size_before_replacement = mempool.store.txs_by_id.len(); + + let replacement_fee = Amount::from_atoms(1000) * fee; + let replacement_tx = tx_spend_input(mempool, input, replacement_fee, flags, locktime)?; + mempool.add_transaction(replacement_tx).map_err(anyhow::Error::from)?; + let mempool_size_after_replacement = mempool.store.txs_by_id.len(); + + assert_eq!( + mempool_size_after_replacement, + mempool_size_before_replacement - num_potential_replacements + 1 + ); + Ok(()) +} + +#[test] +fn too_many_conflicts() -> anyhow::Result<()> { + let mut mempool = setup(); + let num_potential_replacements = MAX_BIP125_REPLACEMENT_CANDIDATES + 1; + let err = test_bip125_max_replacements(&mut mempool, num_potential_replacements) + .expect_err("expected error TooManyPotentialReplacements") + .downcast() + .expect("failed to downcast"); + assert!(matches!( + err, + Error::TxValidationError(TxValidationError::TooManyPotentialReplacements) + )); + Ok(()) +} + +#[test] +fn not_too_many_conflicts() -> anyhow::Result<()> { + let mut mempool = setup(); + let num_potential_replacements = MAX_BIP125_REPLACEMENT_CANDIDATES; + test_bip125_max_replacements(&mut mempool, num_potential_replacements) +} + +#[test] +fn spends_new_unconfirmed() -> anyhow::Result<()> { + let mut mempool = setup(); + let tx = TxGenerator::new() + .with_num_outputs(2) + .replaceable() + .generate_tx(&mempool) + .expect("generate_replaceable_tx"); + let outpoint_source_id = OutPointSourceId::Transaction(tx.get_id()); + mempool.add_transaction(tx)?; + + let input1 = TxInput::new( + outpoint_source_id.clone(), + 0, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ); + let input2 = TxInput::new( + outpoint_source_id, + 1, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ); + + let locktime = 0; + let flags = 0; + let original_fee = Amount::from_atoms(100); + let replaced_tx = tx_spend_input(&mempool, input1.clone(), original_fee, flags, locktime)?; + mempool.add_transaction(replaced_tx)?; + let relay_fee = get_relay_fee_from_tx_size(TX_SPEND_INPUT_SIZE); + let replacement_fee = Amount::from_atoms(100 + relay_fee); + let incoming_tx = tx_spend_several_inputs( + &mempool, + &[input1, input2], + replacement_fee, + flags, + locktime, + )?; + + let res = mempool.add_transaction(incoming_tx); + assert!(matches!( + res, + Err(Error::TxValidationError( + TxValidationError::SpendsNewUnconfirmedOutput + )) + )); + Ok(()) +} + +#[test] +fn pays_more_than_conflicts_with_descendants() -> anyhow::Result<()> { + let mut mempool = setup(); + let tx = TxGenerator::new().generate_tx(&mempool).expect("generate_replaceable_tx"); + let tx_id = tx.get_id(); + mempool.add_transaction(tx)?; + + let outpoint_source_id = OutPointSourceId::Transaction(tx_id); + let input = TxInput::new( + outpoint_source_id, + 0, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ); + + let locktime = 0; + let rbf = 1; + let no_rbf = 0; + + // Create transaction that we will attempt to replace + let original_fee = Amount::from_atoms(100); + let replaced_tx = tx_spend_input(&mempool, input.clone(), original_fee, rbf, locktime)?; + let replaced_tx_fee = mempool.try_get_fee(&replaced_tx)?; + let replaced_id = replaced_tx.get_id(); + mempool.add_transaction(replaced_tx)?; + + // Create some children for this transaction + let descendant_outpoint_source_id = OutPointSourceId::Transaction(replaced_id); + + let descendant1_fee = Amount::from_atoms(100); + let descendant1 = tx_spend_input( + &mempool, + TxInput::new( + descendant_outpoint_source_id.clone(), + 0, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ), + descendant1_fee, + no_rbf, + locktime, + )?; + let descendant1_id = descendant1.get_id(); + mempool.add_transaction(descendant1)?; + + let descendant2_fee = Amount::from_atoms(100); + let descendant2 = tx_spend_input( + &mempool, + TxInput::new( + descendant_outpoint_source_id, + 1, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ), + descendant2_fee, + no_rbf, + locktime, + )?; + let descendant2_id = descendant2.get_id(); + mempool.add_transaction(descendant2)?; + + //Create a new incoming transaction that conflicts with `replaced_tx` because it spends + //`input`. It will be rejected because its fee exactly equals (so is not greater than) the + //sum of the fees of the conflict together with its descendants + let insufficient_rbf_fee = [replaced_tx_fee, descendant1_fee, descendant2_fee] + .into_iter() + .sum::>() + .unwrap(); + let incoming_tx = tx_spend_input( + &mempool, + input.clone(), + insufficient_rbf_fee, + no_rbf, + locktime, + )?; + + assert!(matches!( + mempool.add_transaction(incoming_tx), + Err(Error::TxValidationError( + TxValidationError::TransactionFeeLowerThanConflictsWithDescendants + )) + )); + + let relay_fee = get_relay_fee_from_tx_size(TX_SPEND_INPUT_SIZE); + let sufficient_rbf_fee = insufficient_rbf_fee + Amount::from_atoms(relay_fee); + let incoming_tx = tx_spend_input(&mempool, input, sufficient_rbf_fee, no_rbf, locktime)?; + mempool.add_transaction(incoming_tx)?; + + assert!(!mempool.contains_transaction(&replaced_id)); + assert!(!mempool.contains_transaction(&descendant1_id)); + assert!(!mempool.contains_transaction(&descendant2_id)); + Ok(()) +} + +#[derive(Clone)] +struct MockClock { + time: Arc, +} + +impl MockClock { + fn new() -> Self { + Self { + time: Arc::new(AtomicU64::new(0)), + } + } + + fn set(&self, time: Time) { + self.time.store(time.as_secs(), Ordering::SeqCst) + } + + fn increment(&self, inc: Time) { + self.time.store( + self.time.load(Ordering::SeqCst) + inc.as_secs(), + Ordering::SeqCst, + ) + } +} + +impl GetTime for MockClock { + fn get_time(&self) -> Time { + Duration::new(self.time.load(Ordering::SeqCst), 0) + } +} + +#[test] +fn only_expired_entries_removed() -> anyhow::Result<()> { + let mock_clock = MockClock::new(); + + let mut mempool = MempoolImpl::create( + ChainStateMock::new(), + mock_clock.clone(), + SystemUsageEstimator {}, + ); + + let num_inputs = 1; + let num_outputs = 2; + let big_fee = get_relay_fee_from_tx_size(estimate_tx_size(num_inputs, num_outputs)) + 100; + let parent = TxGenerator::new() + .with_num_inputs(num_inputs) + .with_num_outputs(num_outputs) + .with_fee(Amount::from_atoms(big_fee)) + .generate_tx(&mempool)?; + let parent_id = parent.get_id(); + mempool.add_transaction(parent)?; + + let flags = 0; + let locktime = 0; + let outpoint_source_id = OutPointSourceId::Transaction(parent_id); + let child_0 = tx_spend_input( + &mempool, + TxInput::new( + outpoint_source_id.clone(), + 0, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ), + None, + flags, + locktime, + )?; + + let child_1 = tx_spend_input( + &mempool, + TxInput::new( + outpoint_source_id, + 1, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ), + None, + flags, + locktime, + )?; + let child_1_id = child_1.get_id(); + + let expired_tx_id = child_0.get_id(); + mempool.add_transaction(child_0)?; + + // Simulate the parent being added to a block + // We have to do this because if we leave this parent in the mempool then it will be + // expired, and so removed along with both its children, and thus the addition of child_1 to + // the mempool will fail + mempool.process_block(&parent_id)?; + mock_clock.set(DEFAULT_MEMPOOL_EXPIRY + Duration::new(1, 0)); + + mempool.add_transaction(child_1)?; + assert!(!mempool.contains_transaction(&expired_tx_id)); + assert!(mempool.contains_transaction(&child_1_id)); + Ok(()) +} + +#[test] +fn rolling_fee() -> anyhow::Result<()> { + logging::init_logging::<&str>(None); + let mock_clock = MockClock::new(); + let mut mock_usage = MockGetMemoryUsage::new(); + // Add parent + // Add first child + mock_usage.expect_get_memory_usage().times(2).return_const(0usize); + // Add second child, triggering the trimming process + mock_usage + .expect_get_memory_usage() + .times(1) + .return_const(MAX_MEMPOOL_SIZE_BYTES + 1); + // After removing one entry, cause the code to exit the loop by showing a small usage + mock_usage.expect_get_memory_usage().return_const(0usize); + + let chain_state = ChainStateMock::new(); + let mut mempool = MempoolImpl::create(chain_state, mock_clock.clone(), mock_usage); + + let num_inputs = 1; + let num_outputs = 3; + + // Use a higher than default fee because we don't want this transction to be evicted during + // the trimming process + let parent = TxGenerator::new() + .with_num_inputs(num_inputs) + .with_num_outputs(num_outputs) + .generate_tx(&mempool)?; + let parent_id = parent.get_id(); + log::debug!("parent_id: {}", parent_id.get()); + log::debug!("before adding parent"); + mempool.add_transaction(parent)?; + log::debug!("after adding parent"); + + let flags = 0; + let locktime = 0; + let outpoint_source_id = OutPointSourceId::Transaction(parent_id); + + // child_0 has the lower fee so it will be evicted when memory usage is too high + let child_0 = tx_spend_input( + &mempool, + TxInput::new( + outpoint_source_id.clone(), + 0, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ), + None, + flags, + locktime, + )?; + let child_0_id = child_0.get_id(); + log::debug!("child_0_id {}", child_0_id.get()); + + let big_fee = Amount::from_atoms( + get_relay_fee_from_tx_size(estimate_tx_size(num_inputs, num_outputs)) + 100, + ); + let child_1 = tx_spend_input( + &mempool, + TxInput::new( + outpoint_source_id.clone(), + 1, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ), + big_fee, + flags, + locktime, + )?; + let child_1_id = child_1.get_id(); + log::debug!("child_1_id {}", child_1_id.get()); + mempool.add_transaction(child_0.clone())?; + log::debug!("added child_0"); + mempool.add_transaction(child_1)?; + log::debug!("added child_1"); + + assert_eq!(mempool.store.txs_by_id.len(), 2); + assert!(mempool.contains_transaction(&child_1_id)); + assert!(!mempool.contains_transaction(&child_0_id)); + let rolling_fee = mempool.get_minimum_rolling_fee(); + let child_0_fee = mempool.try_get_fee(&child_0)?; + log::debug!("FeeRate of child_0 {:?}", child_0_fee); + assert_eq!( + rolling_fee, + *INCREMENTAL_RELAY_FEE_RATE + FeeRate::of_tx(child_0_fee, child_0.encoded_size()) + ); + assert_eq!(rolling_fee, FeeRate::new(Amount::from_atoms(3582))); + log::debug!( + "minimum rolling fee after child_0's eviction {:?}", + rolling_fee + ); + assert_eq!( + rolling_fee, + FeeRate::of_tx(mempool.try_get_fee(&child_0)?, child_0.encoded_size()) + + *INCREMENTAL_RELAY_FEE_RATE + ); + + // Now that the minimum rolling fee has been bumped up, a low-fee tx will not pass + // validation + let child_2 = tx_spend_input( + &mempool, + TxInput::new( + outpoint_source_id.clone(), + 2, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ), + None, + flags, + locktime, + )?; + log::debug!( + "before child2: fee = {:?}, size = {}, minimum fee rate = {:?}", + mempool.try_get_fee(&child_2)?, + child_2.encoded_size(), + mempool.get_minimum_rolling_fee() + ); + let res = mempool.add_transaction(child_2); + log::debug!("result of adding child2 {:?}", res); + assert!(matches!( + res, + Err(Error::TxValidationError( + TxValidationError::RollingFeeThresholdNotMet { .. } + )) + )); + + // We provide a sufficient fee for the tx to pass the minimum rolling fee requirement + let child_2_high_fee = tx_spend_input( + &mempool, + TxInput::new( + outpoint_source_id, + 2, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ), + mempool.get_minimum_rolling_fee().compute_fee(estimate_tx_size(1, 1)), + flags, + locktime, + )?; + log::debug!("before child2_high_fee"); + mempool.add_transaction(child_2_high_fee)?; + + // We simulate a block being accepted so the rolling fee will begin to decay + mempool.process_block(&parent_id)?; + + // Because the rolling fee is only updated when we attempt to add a tx to the mempool + // we need to submit a "dummy" tx to trigger these updates. + + // Since memory usage is now zero, it is less than 1/4 of the max size + // and ROLLING_FEE_BASE_HALFLIFE / 4 is the time it will take for the fee to halve + // We are going to submit dummy txs to the mempool incrementing time by this halflife + // between txs. Finally, when the fee rate falls under INCREMENTAL_RELAY_THRESHOLD, we + // observer that it is set to zero + let halflife = ROLLING_FEE_BASE_HALFLIFE / 4; + mock_clock.increment(halflife); + let dummy_tx = TxGenerator::new().with_fee(Amount::from_atoms(100)).generate_tx(&mempool)?; + log::debug!( + "First attempt to add dummy which pays a fee of {:?}", + mempool.try_get_fee(&dummy_tx)? + ); + let res = mempool.add_transaction(dummy_tx.clone()); + + log::debug!("Result of first attempt to add dummy: {:?}", res); + assert!(matches!( + res, + Err(Error::TxValidationError( + TxValidationError::RollingFeeThresholdNotMet { .. } + )) + )); + log::debug!( + "minimum rolling fee after first attempt to add dummy: {:?}", + mempool.get_minimum_rolling_fee() + ); + assert_eq!( + mempool.get_minimum_rolling_fee(), + rolling_fee / std::num::NonZeroU128::new(2).expect("nonzero") + ); + + mock_clock.increment(halflife); + log::debug!("Second attempt to add dummy"); + mempool.add_transaction(dummy_tx)?; + log::debug!( + "minimum rolling fee after first second to add dummy: {:?}", + mempool.get_minimum_rolling_fee() + ); + assert_eq!( + mempool.get_minimum_rolling_fee(), + rolling_fee / std::num::NonZeroU128::new(4).expect("nonzero") + ); + log::debug!( + "After successful addition of dummy, rolling fee rate is {:?}", + mempool.get_minimum_rolling_fee() + ); + + // Add another dummmy until rolling feerate drops to zero + mock_clock.increment(halflife); + let another_dummy = + TxGenerator::new().with_fee(Amount::from_atoms(100)).generate_tx(&mempool)?; + mempool.add_transaction(another_dummy)?; + assert_eq!( + mempool.get_minimum_rolling_fee(), + FeeRate::new(Amount::from_atoms(0)) + ); + + Ok(()) +} + +#[test] +fn different_size_txs() -> anyhow::Result<()> { + let mut mempool = setup(); + let initial_tx = TxGenerator::new() + .with_num_inputs(1) + .with_num_outputs(10_000) + .generate_tx(&mempool)?; + mempool.add_transaction(initial_tx)?; + + let target_txs = 100; + for i in 0..target_txs { + let num_inputs = i + 1; + let num_outputs = i + 1; + let tx = TxGenerator::new() + .with_num_inputs(num_inputs) + .with_num_outputs(num_outputs) + .generate_tx(&mempool)?; + mempool.add_transaction(tx)?; + } + + Ok(()) +} + +#[test] +fn descendant_score() -> anyhow::Result<()> { + let mut mempool = setup(); + let tx = TxGenerator::new() + .with_num_outputs(2) + .generate_tx(&mempool) + .expect("generate_replaceable_tx"); + let tx_id = tx.get_id(); + mempool.add_transaction(tx)?; + + let outpoint_source_id = OutPointSourceId::Transaction(tx_id); + + let flags = 0; + let locktime = 0; + + let tx_b_fee = Amount::from_atoms(get_relay_fee_from_tx_size(estimate_tx_size(1, 2))); + let tx_a_fee = (tx_b_fee + Amount::from_atoms(1000)).unwrap(); + let tx_c_fee = (tx_a_fee + Amount::from_atoms(1000)).unwrap(); + let tx_a = tx_spend_input( + &mempool, + TxInput::new( + outpoint_source_id.clone(), + 0, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ), + tx_a_fee, + flags, + locktime, + )?; + let tx_a_id = tx_a.get_id(); + log::debug!("tx_a_id : {}", tx_a_id.get()); + log::debug!("tx_a fee : {:?}", mempool.try_get_fee(&tx_a)?); + mempool.add_transaction(tx_a)?; + + let tx_b = tx_spend_input( + &mempool, + TxInput::new( + outpoint_source_id, + 1, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ), + tx_b_fee, + flags, + locktime, + )?; + let tx_b_id = tx_b.get_id(); + log::debug!("tx_b_id : {}", tx_b_id.get()); + log::debug!("tx_b fee : {:?}", mempool.try_get_fee(&tx_b)?); + mempool.add_transaction(tx_b)?; + + let tx_c = tx_spend_input( + &mempool, + TxInput::new( + OutPointSourceId::Transaction(tx_b_id), + 0, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ), + tx_c_fee, + flags, + locktime, + )?; + let tx_c_id = tx_c.get_id(); + log::debug!("tx_c_id : {}", tx_c_id.get()); + log::debug!("tx_c fee : {:?}", mempool.try_get_fee(&tx_c)?); + mempool.add_transaction(tx_c)?; + + let entry_a = mempool.store.txs_by_id.get(&tx_a_id.get()).expect("tx_a"); + log::debug!("entry a has score {:?}", entry_a.fees_with_descendants); + let entry_b = mempool.store.txs_by_id.get(&tx_b_id.get()).expect("tx_b"); + log::debug!("entry b has score {:?}", entry_b.fees_with_descendants); + let entry_c = mempool.store.txs_by_id.get(&tx_c_id.get()).expect("tx_c").clone(); + log::debug!("entry c has score {:?}", entry_c.fees_with_descendants); + assert_eq!(entry_a.fee, entry_a.fees_with_descendants); + assert_eq!( + entry_b.fees_with_descendants, + (entry_b.fee + entry_c.fee).unwrap() + ); + assert!(!mempool.store.txs_by_descendant_score.contains_key(&tx_b_fee.into())); + log::debug!( + "raw_txs_by_descendant_score {:?}", + mempool.store.txs_by_descendant_score + ); + check_txs_sorted_by_descendant_sore(&mempool); + + mempool.drop_transaction(&entry_c.tx.get_id()); + assert!(!mempool.store.txs_by_descendant_score.contains_key(&tx_c_fee.into())); + let entry_b = mempool.store.txs_by_id.get(&tx_b_id.get()).expect("tx_b"); + assert_eq!(entry_b.fees_with_descendants, entry_b.fee); + + check_txs_sorted_by_descendant_sore(&mempool); + + Ok(()) +} + +fn check_txs_sorted_by_descendant_sore( + mempool: &MempoolImpl, +) { + let txs_by_descendant_score = + mempool.store.txs_by_descendant_score.values().flatten().collect::>(); + for i in 0..(txs_by_descendant_score.len() - 1) { + log::debug!("i = {}", i); + let tx_id = txs_by_descendant_score.get(i).unwrap(); + let next_tx_id = txs_by_descendant_score.get(i + 1).unwrap(); + let entry_score = mempool.store.txs_by_id.get(tx_id).unwrap().fees_with_descendants; + let next_entry_score = + mempool.store.txs_by_id.get(next_tx_id).unwrap().fees_with_descendants; + log::debug!("entry_score: {:?}", entry_score); + log::debug!("next_entry_score: {:?}", next_entry_score); + assert!(entry_score <= next_entry_score) + } +} + +#[test] +fn descendant_of_expired_entry() -> anyhow::Result<()> { + let mock_clock = MockClock::new(); + logging::init_logging::<&str>(None); + + let mut mempool = MempoolImpl::create( + ChainStateMock::new(), + mock_clock.clone(), + SystemUsageEstimator {}, + ); + + let num_inputs = 1; + let num_outputs = 2; + let fee = get_relay_fee_from_tx_size(estimate_tx_size(num_inputs, num_outputs)); + let parent = TxGenerator::new() + .with_num_inputs(num_inputs) + .with_num_outputs(num_outputs) + .with_fee(Amount::from_atoms(fee)) + .generate_tx(&mempool)?; + let parent_id = parent.get_id(); + mempool.add_transaction(parent)?; + + let flags = 0; + let locktime = 0; + let outpoint_source_id = OutPointSourceId::Transaction(parent_id); + let child = tx_spend_input( + &mempool, + TxInput::new( + outpoint_source_id, + 0, + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ), + None, + flags, + locktime, + )?; + let child_id = child.get_id(); + mock_clock.set(DEFAULT_MEMPOOL_EXPIRY + Duration::new(1, 0)); + + assert!(matches!( + mempool.add_transaction(child), + Err(Error::TxValidationError( + TxValidationError::DescendantOfExpiredTransaction + )) + )); + + assert!(!mempool.contains_transaction(&parent_id)); + assert!(!mempool.contains_transaction(&child_id)); + Ok(()) +} + +#[test] +fn mempool_full() -> anyhow::Result<()> { + logging::init_logging::<&str>(None); + let mut mock_usage = MockGetMemoryUsage::new(); + mock_usage + .expect_get_memory_usage() + .times(1) + .return_const(MAX_MEMPOOL_SIZE_BYTES + 1); + + let chain_state = ChainStateMock::new(); + let mut mempool = MempoolImpl::create(chain_state, SystemClock, mock_usage); + + let tx = TxGenerator::new().generate_tx(&mempool)?; + log::debug!("mempool_full: tx has is {}", tx.get_id().get()); + assert!(matches!( + mempool.add_transaction(tx), + Err(Error::MempoolFull) + )); + Ok(()) +} + +#[test] +fn no_empty_bags_in_descendant_score_index() -> anyhow::Result<()> { + let mut mempool = setup(); + + let num_inputs = 1; + let num_outputs = 100; + let fee = get_relay_fee_from_tx_size(estimate_tx_size(num_inputs, num_outputs)); + let parent = TxGenerator::new() + .with_num_inputs(num_inputs) + .with_num_outputs(num_outputs) + .with_fee(Amount::from_atoms(fee)) + .generate_tx(&mempool)?; + let parent_id = parent.get_id(); + + let outpoint_source_id = OutPointSourceId::Transaction(parent.get_id()); + mempool.add_transaction(parent)?; + let num_child_txs = num_outputs; + let flags = 0; + let locktime = 0; + let txs = (0..num_child_txs) + .into_iter() + .map(|i| { + tx_spend_input( + &mempool, + TxInput::new( + outpoint_source_id.clone(), + u32::try_from(i).unwrap(), + InputWitness::NoSignature(Some(DUMMY_WITNESS_MSG.to_vec())), + ), + Amount::from_atoms(fee + u128::try_from(i).unwrap()), + flags, + locktime, + ) + }) + .collect::, _>>()?; + let ids = txs.iter().map(|tx| tx.get_id()).collect::>(); + + for tx in txs { + mempool.add_transaction(tx)?; + } + + mempool.drop_transaction(&parent_id); + for id in ids { + mempool.drop_transaction(&id) + } + assert!(mempool.store.txs_by_descendant_score.is_empty()); + Ok(()) +} From b054ca5db9b9bddbdaa676140ec5203041833ed7 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Tue, 26 Jul 2022 18:18:45 +0300 Subject: [PATCH 118/153] Create FeeRate from Amount rather than integer type --- mempool/src/feerate.rs | 10 ++++++---- mempool/src/pool.rs | 11 ++++++----- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/mempool/src/feerate.rs b/mempool/src/feerate.rs index 30fcb123f7..e466c961ea 100644 --- a/mempool/src/feerate.rs +++ b/mempool/src/feerate.rs @@ -1,8 +1,8 @@ use common::primitives::amount::Amount; lazy_static::lazy_static! { - pub(crate) static ref INCREMENTAL_RELAY_FEE_RATE: FeeRate = FeeRate::new(1000); - pub(crate) static ref INCREMENTAL_RELAY_THRESHOLD: FeeRate = FeeRate::new(500); + pub(crate) static ref INCREMENTAL_RELAY_FEE_RATE: FeeRate = FeeRate::new(Amount::from_atoms(1000)); + pub(crate) static ref INCREMENTAL_RELAY_THRESHOLD: FeeRate = FeeRate::new(Amount::from_atoms(500)); } #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] @@ -11,8 +11,10 @@ pub(crate) struct FeeRate { } impl FeeRate { - pub(crate) fn new(tokens_per_kb: u128) -> Self { - Self { tokens_per_kb } + pub(crate) fn new(tokens_per_kb: Amount) -> Self { + Self { + tokens_per_kb: tokens_per_kb.into(), + } } pub(crate) fn of_tx(fee: Amount, tx_size: usize) -> Self { diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 1fd1874edd..f7a793d09f 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -260,8 +260,9 @@ impl RollingFeeRate { (current_time.as_secs() - self.last_rolling_fee_update.as_secs()) as f64 / (halflife.as_secs() as f64), ); - self.rolling_minimum_fee_rate = - FeeRate::new((self.rolling_minimum_fee_rate.tokens_per_kb() as f64 / divisor) as u128); + self.rolling_minimum_fee_rate = FeeRate::new(Amount::from_atoms( + (self.rolling_minimum_fee_rate.tokens_per_kb() as f64 / divisor) as u128, + )); log::trace!( "decay_fee: new fee rate: {:?}", @@ -276,7 +277,7 @@ impl RollingFeeRate { pub(crate) fn new(creation_time: Time) -> Self { Self { block_since_last_rolling_fee_bump: false, - rolling_minimum_fee_rate: FeeRate::new(0), + rolling_minimum_fee_rate: FeeRate::new(Amount::from_atoms(0)), last_rolling_fee_update: creation_time, } } @@ -574,7 +575,7 @@ where pub(crate) fn get_update_min_fee_rate(&self) -> FeeRate { let rolling_fee_rate = self.rolling_fee_rate.get(); if !rolling_fee_rate.block_since_last_rolling_fee_bump - || rolling_fee_rate.rolling_minimum_fee_rate == FeeRate::new(0) + || rolling_fee_rate.rolling_minimum_fee_rate == FeeRate::new(Amount::from_atoms(0)) { return rolling_fee_rate.rolling_minimum_fee_rate; } else if self.clock.get_time() @@ -602,7 +603,7 @@ where fn drop_rolling_fee(&self) { let mut rolling_fee_rate = self.rolling_fee_rate.get(); - rolling_fee_rate.rolling_minimum_fee_rate = FeeRate::new(0); + rolling_fee_rate.rolling_minimum_fee_rate = FeeRate::new(Amount::from_atoms(0)); self.rolling_fee_rate.set(rolling_fee_rate) } From b0ffd64336a7911cb0577a963791b7955a0489ab Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Tue, 26 Jul 2022 18:20:58 +0300 Subject: [PATCH 119/153] Rename tokens_per_kb field of FeeRate to atoms_per_kb --- mempool/src/feerate.rs | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/mempool/src/feerate.rs b/mempool/src/feerate.rs index e466c961ea..1fbac90cbc 100644 --- a/mempool/src/feerate.rs +++ b/mempool/src/feerate.rs @@ -7,30 +7,30 @@ lazy_static::lazy_static! { #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub(crate) struct FeeRate { - tokens_per_kb: u128, + atoms_per_kb: u128, } impl FeeRate { pub(crate) fn new(tokens_per_kb: Amount) -> Self { Self { - tokens_per_kb: tokens_per_kb.into(), + atoms_per_kb: tokens_per_kb.into(), } } pub(crate) fn of_tx(fee: Amount, tx_size: usize) -> Self { Self { - tokens_per_kb: Self::div_up(1000 * u128::try_from(fee).expect("of_tx"), tx_size), + atoms_per_kb: Self::div_up(1000 * u128::try_from(fee).expect("of_tx"), tx_size), } } pub(crate) fn compute_fee(&self, size: usize) -> Amount { Amount::from_atoms( - self.tokens_per_kb * u128::try_from(size).expect("compute_fee conversion") / 1000, + self.atoms_per_kb * u128::try_from(size).expect("compute_fee conversion") / 1000, ) } pub(crate) fn tokens_per_kb(&self) -> u128 { - self.tokens_per_kb + self.atoms_per_kb } fn div_up(fee: u128, tx_size: usize) -> u128 { @@ -42,8 +42,10 @@ impl FeeRate { impl std::ops::Add for FeeRate { type Output = FeeRate; fn add(self, other: Self) -> Self::Output { - let tokens_per_kb = self.tokens_per_kb + other.tokens_per_kb; - FeeRate { tokens_per_kb } + let tokens_per_kb = self.atoms_per_kb + other.atoms_per_kb; + FeeRate { + atoms_per_kb: tokens_per_kb, + } } } @@ -51,7 +53,7 @@ impl std::ops::Div for FeeRate { type Output = FeeRate; fn div(self, rhs: u128) -> Self::Output { FeeRate { - tokens_per_kb: self.tokens_per_kb / rhs, + atoms_per_kb: self.atoms_per_kb / rhs, } } } From 0f0365d3320b5e8c868d747f7a90cb12f5d2abef Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Tue, 26 Jul 2022 18:26:07 +0300 Subject: [PATCH 120/153] Style changes to newtype macro --- mempool/src/pool.rs | 24 ++++++++++++------------ utils/src/newtype.rs | 9 +++++---- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index f7a793d09f..4f9a566aa5 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -110,20 +110,20 @@ trait TryGetFee { fn try_get_fee(&self, tx: &Transaction) -> Result; } -newtype!( +newtype! { #[derive(Debug)] - struct Ancestors(BTreeSet) -); + struct Ancestors(BTreeSet); +} -newtype!( +newtype! { #[derive(Debug)] - struct Descendants(BTreeSet) -); + struct Descendants(BTreeSet); +} -newtype!( +newtype! { #[derive(Debug)] - struct Conflicts(BTreeSet) -); + struct Conflicts(BTreeSet); +} #[derive(Debug, PartialEq, Eq, Clone)] struct TxMempoolEntry { @@ -234,10 +234,10 @@ impl Ord for TxMempoolEntry { } } -newtype!( +newtype! { #[derive(Debug, PartialEq, Eq, Ord, PartialOrd)] - struct DescendantScore(Amount) -); + struct DescendantScore(Amount); +} #[derive(Clone, Copy, Debug)] struct RollingFeeRate { diff --git a/utils/src/newtype.rs b/utils/src/newtype.rs index 59b2962b7b..46a87a4302 100644 --- a/utils/src/newtype.rs +++ b/utils/src/newtype.rs @@ -15,7 +15,7 @@ #[macro_export] macro_rules! newtype { - ($(#[$meta:meta])* $vis:vis struct $name:ident($wrapped:ty)) => { + ($(#[$meta:meta])* $vis:vis struct $name:ident($wrapped:ty);) => { $(#[$meta])* $vis struct $name($wrapped); @@ -65,9 +65,10 @@ mod tests { } } - newtype!( - #[derive(Clone, Debug)] - struct NewInt(OldInt)); + newtype! { + #[derive(Clone, Debug)] + struct NewInt(OldInt); + } #[test] fn test_new_type() { From 3eba52932b842760c6634580f3ff85294132fa45 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Tue, 26 Jul 2022 18:29:01 +0300 Subject: [PATCH 121/153] Use exp2 method in rolling fee rate calculation --- mempool/src/pool.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 4f9a566aa5..a436340899 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -256,10 +256,9 @@ impl RollingFeeRate { halflife, ); - let divisor = 2f64.powf( - (current_time.as_secs() - self.last_rolling_fee_update.as_secs()) as f64 - / (halflife.as_secs() as f64), - ); + let divisor = ((current_time.as_secs() - self.last_rolling_fee_update.as_secs()) as f64 + / (halflife.as_secs() as f64)) + .exp2(); self.rolling_minimum_fee_rate = FeeRate::new(Amount::from_atoms( (self.rolling_minimum_fee_rate.tokens_per_kb() as f64 / divisor) as u128, )); From 0dbcac4b76990b4858113ea66580acd1f979d92f Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Tue, 26 Jul 2022 18:46:11 +0300 Subject: [PATCH 122/153] Use into_atoms to convert Amount to u128 --- common/src/primitives/amount.rs | 6 ------ mempool/src/feerate.rs | 4 ++-- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/common/src/primitives/amount.rs b/common/src/primitives/amount.rs index b1af89c6d2..9fa2c99e42 100644 --- a/common/src/primitives/amount.rs +++ b/common/src/primitives/amount.rs @@ -131,12 +131,6 @@ impl Amount { } } -impl From for u128 { - fn from(amount: Amount) -> Self { - amount.val - } -} - impl std::ops::Add for Amount { type Output = Option; diff --git a/mempool/src/feerate.rs b/mempool/src/feerate.rs index 1fbac90cbc..6b7a41c701 100644 --- a/mempool/src/feerate.rs +++ b/mempool/src/feerate.rs @@ -13,13 +13,13 @@ pub(crate) struct FeeRate { impl FeeRate { pub(crate) fn new(tokens_per_kb: Amount) -> Self { Self { - atoms_per_kb: tokens_per_kb.into(), + atoms_per_kb: tokens_per_kb.into_atoms(), } } pub(crate) fn of_tx(fee: Amount, tx_size: usize) -> Self { Self { - atoms_per_kb: Self::div_up(1000 * u128::try_from(fee).expect("of_tx"), tx_size), + atoms_per_kb: Self::div_up(1000 * fee.into_atoms(), tx_size), } } From 8fb0b6c67c908fbcee2ac3d22216feaca404df20 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Tue, 26 Jul 2022 18:47:35 +0300 Subject: [PATCH 123/153] Update scale codec to 3.1 --- Cargo.lock | 84 ++++++++-------------------------------------- mempool/Cargo.toml | 2 +- 2 files changed, 15 insertions(+), 71 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 301ed0571f..bd192215ae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -329,28 +329,16 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" -[[package]] -name = "bitvec" -version = "0.20.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7774144344a4faa177370406a7ff5f1da24303817368584c6206c8303eb07848" -dependencies = [ - "funty 1.1.0", - "radium 0.6.2", - "tap", - "wyz 0.2.0", -] - [[package]] name = "bitvec" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" dependencies = [ - "funty 2.0.0", - "radium 0.7.0", + "funty", + "radium", "tap", - "wyz 0.5.0", + "wyz", ] [[package]] @@ -573,7 +561,7 @@ name = "chainstate-types" version = "0.1.0" dependencies = [ "common", - "parity-scale-codec 3.1.5", + "parity-scale-codec", "serialization", "storage", "thiserror", @@ -692,7 +680,7 @@ dependencies = [ "logging", "loom", "merkletree", - "parity-scale-codec 3.1.5", + "parity-scale-codec", "proptest", "rand 0.8.5", "rustc-hex", @@ -836,7 +824,7 @@ dependencies = [ "num", "num-derive", "num-traits", - "parity-scale-codec 3.1.5", + "parity-scale-codec", "rand 0.8.5", "rand_chacha 0.3.1", "ripemd", @@ -1124,12 +1112,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" -[[package]] -name = "funty" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed34cd105917e91daa4da6b3728c47b068749d6a62c59811f06ed2ac71d9da7" - [[package]] name = "funty" version = "2.0.0" @@ -2195,7 +2177,7 @@ dependencies = [ "lazy_static", "logging", "mockall", - "parity-scale-codec 2.3.1", + "parity-scale-codec", "serialization", "thiserror", "utils", @@ -2639,7 +2621,7 @@ dependencies = [ "libp2p", "logging", "p2p-test-utils", - "parity-scale-codec 3.1.5", + "parity-scale-codec", "portpicker", "rpc", "serde", @@ -2667,20 +2649,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "parity-scale-codec" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "373b1a4c1338d9cd3d1fa53b3a11bdab5ab6bd80a20f7f7becd76953ae2be909" -dependencies = [ - "arrayvec", - "bitvec 0.20.4", - "byte-slice-cast", - "impl-trait-for-tuples", - "parity-scale-codec-derive 2.3.1", - "serde", -] - [[package]] name = "parity-scale-codec" version = "3.1.5" @@ -2688,25 +2656,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9182e4a71cae089267ab03e67c99368db7cd877baf50f931e5d6d4b71e195ac0" dependencies = [ "arrayvec", - "bitvec 1.0.1", + "bitvec", "byte-slice-cast", "impl-trait-for-tuples", - "parity-scale-codec-derive 3.1.3", + "parity-scale-codec-derive", "serde", ] -[[package]] -name = "parity-scale-codec-derive" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1557010476e0595c9b568d16dcfb81b93cdeb157612726f5170d31aa707bed27" -dependencies = [ - "proc-macro-crate", - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "parity-scale-codec-derive" version = "3.1.3" @@ -3104,12 +3060,6 @@ dependencies = [ "proc-macro2", ] -[[package]] -name = "radium" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "643f8f41a8ebc4c5dc4515c82bb8abd397b527fc20fd681b7c011c2aee5d44fb" - [[package]] name = "radium" version = "0.7.0" @@ -3563,7 +3513,7 @@ dependencies = [ "hex", "hex-literal", "logging", - "parity-scale-codec 3.1.5", + "parity-scale-codec", "proptest", "serialization", "thiserror", @@ -3722,7 +3672,7 @@ version = "0.1.0" dependencies = [ "arraytools", "hex-literal", - "parity-scale-codec 3.1.5", + "parity-scale-codec", "rand 0.8.5", ] @@ -3730,7 +3680,7 @@ dependencies = [ name = "serialization-tagged" version = "0.1.0" dependencies = [ - "parity-scale-codec 3.1.5", + "parity-scale-codec", "proptest", "serialization", "serialization-core", @@ -4522,7 +4472,7 @@ dependencies = [ "crypto", "itertools", "logging", - "parity-scale-codec 3.1.5", + "parity-scale-codec", "serialization", "thiserror", ] @@ -4905,12 +4855,6 @@ version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" -[[package]] -name = "wyz" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85e60b0d1b5f99db2556934e21937020776a5d31520bf169e851ac44e6420214" - [[package]] name = "wyz" version = "0.5.0" diff --git a/mempool/Cargo.toml b/mempool/Cargo.toml index 3fb2cd2738..ffca866f1f 100644 --- a/mempool/Cargo.toml +++ b/mempool/Cargo.toml @@ -7,7 +7,7 @@ license = "MIT" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -parity-scale-codec = {version = "2.3.1", features = ["derive", "chain-error"]} +parity-scale-codec = {version = "3.1", features = ["derive", "chain-error"]} serialization = { path = '../serialization' } common = { path = '../common' } utils = { path = '../utils' } From ac5e23c65fbe746a3fdaeafd64037f76b16159cf Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Tue, 26 Jul 2022 19:08:15 +0300 Subject: [PATCH 124/153] Test sum of empty Amount vector --- common/src/primitives/amount.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/common/src/primitives/amount.rs b/common/src/primitives/amount.rs index 9fa2c99e42..e2f5969ec1 100644 --- a/common/src/primitives/amount.rs +++ b/common/src/primitives/amount.rs @@ -338,6 +338,14 @@ mod tests { assert_eq!(amounts.into_iter().sum::>(), None); } + #[test] + fn sum_empty() { + assert_eq!( + vec![].into_iter().sum::>(), + Some(Amount::from_atoms(0)) + ) + } + #[test] fn sub_underflow() { assert_eq!(Amount { val: IntType::MIN } - Amount { val: 1 }, None); From 7170066b85b64f5cb434bf6e9b4f574d6a12b7e7 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Tue, 26 Jul 2022 19:18:02 +0300 Subject: [PATCH 125/153] Clean up compute_fee --- mempool/src/feerate.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mempool/src/feerate.rs b/mempool/src/feerate.rs index 6b7a41c701..dc62de7ab7 100644 --- a/mempool/src/feerate.rs +++ b/mempool/src/feerate.rs @@ -24,9 +24,8 @@ impl FeeRate { } pub(crate) fn compute_fee(&self, size: usize) -> Amount { - Amount::from_atoms( - self.atoms_per_kb * u128::try_from(size).expect("compute_fee conversion") / 1000, - ) + let size = u128::try_from(size).expect("compute_fee conversion"); + Amount::from_atoms(self.atoms_per_kb * size / 1000) } pub(crate) fn tokens_per_kb(&self) -> u128 { From c5fa3a224d13462ff7bf79cf0639765e58d62b4f Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Tue, 26 Jul 2022 19:26:11 +0300 Subject: [PATCH 126/153] Use saturating_sub when computing expired entries --- mempool/src/pool.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index a436340899..49ee3aae8a 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -939,7 +939,8 @@ where .map(|entry_id| self.store.txs_by_id.get(entry_id).expect("entry should exist")) .filter(|entry| { let now = self.clock.get_time(); - if now - entry.creation_time > self.max_tx_age { + let expired = now.saturating_sub(entry.creation_time) > self.max_tx_age; + if expired { log::trace!( "Evicting tx {} which was created at {:?}. It is now {:?}", entry.tx_id(), From 07019c9e38a412e012671e6d6545ebb07141e3ce Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Wed, 27 Jul 2022 15:04:19 +0300 Subject: [PATCH 127/153] Add test for FeeRate::div_up --- mempool/src/feerate.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/mempool/src/feerate.rs b/mempool/src/feerate.rs index dc62de7ab7..d400b2fdb6 100644 --- a/mempool/src/feerate.rs +++ b/mempool/src/feerate.rs @@ -56,3 +56,16 @@ impl std::ops::Div for FeeRate { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_div_up() { + let fee = 7; + let tx_size = usize::MAX; + let rate = FeeRate::div_up(fee, tx_size); + assert_eq!(rate, 1); + } +} From cd04ceb72204e8d6f7e9ae6d3aff58db729b1cc4 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Wed, 27 Jul 2022 15:08:25 +0300 Subject: [PATCH 128/153] Add comment to time_lock_notes.txt --- notes/time_lock_notes.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/notes/time_lock_notes.txt b/notes/time_lock_notes.txt index 103161ec9b..f37fcc9560 100644 --- a/notes/time_lock_notes.txt +++ b/notes/time_lock_notes.txt @@ -1,3 +1,5 @@ +NOTE: This file was written before we had any implementation of timelocks. We should handle timelocks in the mempool, and then also remove this file. + TIMELOCK NOTES 1. There are two kinds of timelocks, absolute and relative. Absolute timelocks prevent transmitting a tx over the network or spending a utxo before a specific point in time (measured either in Unix Epoch Time or block height). Relative timelocks prevent spending an outputs before it has aged by a certain amount of time. From 40c45b7fe68b0fce05e4d4ffe8a767173588d256 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Wed, 27 Jul 2022 15:21:30 +0300 Subject: [PATCH 129/153] Use NonZeroU128 as FeeRate divisor --- mempool/src/feerate.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mempool/src/feerate.rs b/mempool/src/feerate.rs index d400b2fdb6..96a10671cd 100644 --- a/mempool/src/feerate.rs +++ b/mempool/src/feerate.rs @@ -1,3 +1,5 @@ +use std::num::NonZeroU128; + use common::primitives::amount::Amount; lazy_static::lazy_static! { @@ -48,9 +50,9 @@ impl std::ops::Add for FeeRate { } } -impl std::ops::Div for FeeRate { +impl std::ops::Div for FeeRate { type Output = FeeRate; - fn div(self, rhs: u128) -> Self::Output { + fn div(self, rhs: NonZeroU128) -> Self::Output { FeeRate { atoms_per_kb: self.atoms_per_kb / rhs, } From 5eb09bc6860a23430b35c77a67964e31d63f8319 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Thu, 28 Jul 2022 15:12:30 +0300 Subject: [PATCH 130/153] Remove unnecessary static bounds --- mempool/src/pool.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 49ee3aae8a..91a51e705d 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -45,12 +45,12 @@ const ROLLING_FEE_DECAY_INTERVAL: Time = Duration::new(10, 0); pub(crate) type MemoryUsage = usize; #[automock] -pub trait GetMemoryUsage: 'static { +pub trait GetMemoryUsage { fn get_memory_usage(&self) -> MemoryUsage; } pub(crate) type Time = Duration; -pub trait GetTime: Clone + 'static { +pub trait GetTime { fn get_time(&self) -> Time; } @@ -101,7 +101,7 @@ pub trait Mempool { fn new_tip_set(&mut self, chain_state: C); } -pub trait ChainState: Debug + 'static { +pub trait ChainState: Debug { fn contains_outpoint(&self, outpoint: &OutPoint) -> bool; fn get_outpoint_value(&self, outpoint: &OutPoint) -> Result; } From 98e8e4afe6f99d53b1be90157ec1a866fe16970e Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Thu, 28 Jul 2022 15:15:33 +0300 Subject: [PATCH 131/153] Rename test one_ancestor_signal_is_enough --- mempool/src/pool/tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mempool/src/pool/tests.rs b/mempool/src/pool/tests.rs index 5277a40ad6..a44e0e780f 100644 --- a/mempool/src/pool/tests.rs +++ b/mempool/src/pool/tests.rs @@ -886,7 +886,7 @@ fn tx_spend_several_inputs( } #[test] -fn one_ancestor_signal_is_enough() -> anyhow::Result<()> { +fn one_ancestor_replaceability_signal_is_enough() -> anyhow::Result<()> { let mut mempool = setup(); let tx = TxGenerator::new() .with_num_outputs(2) From b3b0244a0eb789c27ad7ef08d7fa24aafa16113b Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Thu, 28 Jul 2022 15:17:06 +0300 Subject: [PATCH 132/153] Rename tokens_per_kb -> atoms_per_kb --- mempool/src/feerate.rs | 12 +++++------- mempool/src/pool.rs | 2 +- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/mempool/src/feerate.rs b/mempool/src/feerate.rs index 96a10671cd..4afc1b9ba8 100644 --- a/mempool/src/feerate.rs +++ b/mempool/src/feerate.rs @@ -13,9 +13,9 @@ pub(crate) struct FeeRate { } impl FeeRate { - pub(crate) fn new(tokens_per_kb: Amount) -> Self { + pub(crate) fn new(atoms_per_kb: Amount) -> Self { Self { - atoms_per_kb: tokens_per_kb.into_atoms(), + atoms_per_kb: atoms_per_kb.into_atoms(), } } @@ -30,7 +30,7 @@ impl FeeRate { Amount::from_atoms(self.atoms_per_kb * size / 1000) } - pub(crate) fn tokens_per_kb(&self) -> u128 { + pub(crate) fn atoms_per_kb(&self) -> u128 { self.atoms_per_kb } @@ -43,10 +43,8 @@ impl FeeRate { impl std::ops::Add for FeeRate { type Output = FeeRate; fn add(self, other: Self) -> Self::Output { - let tokens_per_kb = self.atoms_per_kb + other.atoms_per_kb; - FeeRate { - atoms_per_kb: tokens_per_kb, - } + let atoms_per_kb = self.atoms_per_kb + other.atoms_per_kb; + FeeRate { atoms_per_kb } } } diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 91a51e705d..224e6bbffc 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -260,7 +260,7 @@ impl RollingFeeRate { / (halflife.as_secs() as f64)) .exp2(); self.rolling_minimum_fee_rate = FeeRate::new(Amount::from_atoms( - (self.rolling_minimum_fee_rate.tokens_per_kb() as f64 / divisor) as u128, + (self.rolling_minimum_fee_rate.atoms_per_kb() as f64 / divisor) as u128, )); log::trace!( From 358577315654e90fa718babd10f5509a4f2bf7dd Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Thu, 28 Jul 2022 15:18:58 +0300 Subject: [PATCH 133/153] Rename FeeRate::of_tx -> FeeRate::from_total_tx_fees --- mempool/src/feerate.rs | 4 ++-- mempool/src/pool.rs | 5 ++++- mempool/src/pool/tests.rs | 5 +++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/mempool/src/feerate.rs b/mempool/src/feerate.rs index 4afc1b9ba8..1ff4df7cc5 100644 --- a/mempool/src/feerate.rs +++ b/mempool/src/feerate.rs @@ -19,9 +19,9 @@ impl FeeRate { } } - pub(crate) fn of_tx(fee: Amount, tx_size: usize) -> Self { + pub(crate) fn from_total_tx_fee(total_tx_fee: Amount, tx_size: usize) -> Self { Self { - atoms_per_kb: Self::div_up(1000 * fee.into_atoms(), tx_size), + atoms_per_kb: Self::div_up(1000 * total_tx_fee.into_atoms(), tx_size), } } diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 224e6bbffc..f31a1a789d 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -980,7 +980,10 @@ where removed.fees_with_descendants, removed.tx.encoded_size() ); - removed_fees.push(FeeRate::of_tx(removed.fee, removed.tx.encoded_size())); + removed_fees.push(FeeRate::from_total_tx_fee( + removed.fee, + removed.tx.encoded_size(), + )); self.store.drop_tx_and_descendants(removed.tx.get_id()); } removed_fees diff --git a/mempool/src/pool/tests.rs b/mempool/src/pool/tests.rs index a44e0e780f..9e63b16f58 100644 --- a/mempool/src/pool/tests.rs +++ b/mempool/src/pool/tests.rs @@ -1409,7 +1409,8 @@ fn rolling_fee() -> anyhow::Result<()> { log::debug!("FeeRate of child_0 {:?}", child_0_fee); assert_eq!( rolling_fee, - *INCREMENTAL_RELAY_FEE_RATE + FeeRate::of_tx(child_0_fee, child_0.encoded_size()) + *INCREMENTAL_RELAY_FEE_RATE + + FeeRate::from_total_tx_fee(child_0_fee, child_0.encoded_size()) ); assert_eq!(rolling_fee, FeeRate::new(Amount::from_atoms(3582))); log::debug!( @@ -1418,7 +1419,7 @@ fn rolling_fee() -> anyhow::Result<()> { ); assert_eq!( rolling_fee, - FeeRate::of_tx(mempool.try_get_fee(&child_0)?, child_0.encoded_size()) + FeeRate::from_total_tx_fee(mempool.try_get_fee(&child_0)?, child_0.encoded_size()) + *INCREMENTAL_RELAY_FEE_RATE ); From 122479fc1d0869d7dbd02cea8ace98d466a4f633 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Thu, 28 Jul 2022 15:26:18 +0300 Subject: [PATCH 134/153] Remove eprintln calls --- mempool/src/pool.rs | 12 ++++++------ mempool/src/pool/tests.rs | 9 +++++---- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index f31a1a789d..663a537015 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -88,7 +88,6 @@ where fn get_relay_fee(tx: &Transaction) -> Amount { // TODO we should never reach the expect, but should this be an error anyway? - eprintln!("get_relay_fee - encoded_size {:?}", tx.encoded_size()); Amount::from_atoms(u128::try_from(tx.encoded_size() * RELAY_FEE_PER_BYTE).expect("Overflow")) } @@ -747,14 +746,13 @@ where fn pays_minimum_relay_fees(&self, tx: &Transaction) -> Result<(), TxValidationError> { let tx_fee = self.try_get_fee(tx)?; let relay_fee = get_relay_fee(tx); - eprintln!("tx_fee: {:?}, relay_fee: {:?}", tx_fee, relay_fee); + log::debug!("tx_fee: {:?}, relay_fee: {:?}", tx_fee, relay_fee); (tx_fee >= relay_fee) .then(|| ()) .ok_or(TxValidationError::InsufficientFeesToRelay { tx_fee, relay_fee }) } fn rbf_checks(&self, tx: &Transaction) -> Result { - eprintln!("rbf_checks"); let conflicts = tx .inputs() .iter() @@ -805,13 +803,15 @@ where tx: &Transaction, total_conflict_fees: Amount, ) -> Result<(), TxValidationError> { - eprintln!("tx fee is {:?}", self.try_get_fee(tx)?); + log::debug!("pays_for_bandwidth: tx fee is {:?}", self.try_get_fee(tx)?); let additional_fees = (self.try_get_fee(tx)? - total_conflict_fees) .ok_or(TxValidationError::AdditionalFeesUnderflow)?; let relay_fee = get_relay_fee(tx); - eprintln!( + log::debug!( "conflict fees: {:?}, additional fee: {:?}, relay_fee {:?}", - total_conflict_fees, additional_fees, relay_fee + total_conflict_fees, + additional_fees, + relay_fee ); (additional_fees >= relay_fee) .then(|| ()) diff --git a/mempool/src/pool/tests.rs b/mempool/src/pool/tests.rs index 9e63b16f58..ff127bb3d2 100644 --- a/mempool/src/pool/tests.rs +++ b/mempool/src/pool/tests.rs @@ -671,9 +671,10 @@ fn tx_too_big() -> anyhow::Result<()> { } fn test_replace_tx(original_fee: Amount, replacement_fee: Amount) -> Result<(), Error> { - eprintln!( + log::debug!( "tx_replace_tx: original_fee: {:?}, replacement_fee {:?}", - original_fee, replacement_fee + original_fee, + replacement_fee ); let mut mempool = setup(); let outpoint = mempool @@ -697,13 +698,13 @@ fn test_replace_tx(original_fee: Amount, replacement_fee: Amount) -> Result<(), let original = tx_spend_input(&mempool, input.clone(), original_fee, flags, locktime) .expect("should be able to spend here"); let original_id = original.get_id(); - eprintln!("created a tx with fee {:?}", mempool.try_get_fee(&original)); + log::debug!("created a tx with fee {:?}", mempool.try_get_fee(&original)); mempool.add_transaction(original)?; let flags = 0; let replacement = tx_spend_input(&mempool, input, replacement_fee, flags, locktime) .expect("should be able to spend here"); - eprintln!( + log::debug!( "created a replacement with fee {:?}", mempool.try_get_fee(&replacement) ); From f99611e76c2bac286ec69c9cbc1a035d05b1b0dc Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Thu, 28 Jul 2022 15:31:53 +0300 Subject: [PATCH 135/153] Use ensure macro --- mempool/src/pool.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 663a537015..a683cb4a88 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -19,6 +19,7 @@ use common::primitives::H256; use logging::log; +use utils::ensure; use utils::newtype; use crate::error::Error; @@ -774,10 +775,11 @@ where ) -> Result { for entry in conflicts { // Enforce BIP125 Rule #1. - entry - .is_replaceable(&self.store) - .then(|| ()) - .ok_or(TxValidationError::ConflictWithIrreplaceableTransaction)?; + + ensure!( + entry.is_replaceable(&self.store), + TxValidationError::ConflictWithIrreplaceableTransaction + ); } // It's possible that the replacement pays more fees than its direct conflicts but not more // than all conflicts (i.e. the direct conflicts have high-fee descendants). However, if the From 205b79e246c4eaf0c861698ff0e14f05787b51db Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Thu, 28 Jul 2022 15:32:15 +0300 Subject: [PATCH 136/153] Fix indentation in newtype example --- utils/src/newtype.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/src/newtype.rs b/utils/src/newtype.rs index 46a87a4302..a3515dd741 100644 --- a/utils/src/newtype.rs +++ b/utils/src/newtype.rs @@ -66,8 +66,8 @@ mod tests { } newtype! { - #[derive(Clone, Debug)] - struct NewInt(OldInt); + #[derive(Clone, Debug)] + struct NewInt(OldInt); } #[test] From 491585d69958abe5e725de501a2c72e29993f5e9 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Thu, 28 Jul 2022 16:17:18 +0300 Subject: [PATCH 137/153] Friendlier human-readable errors for mempool --- mempool/src/error.rs | 48 ++++++++++++++++++++------------------------ 1 file changed, 22 insertions(+), 26 deletions(-) diff --git a/mempool/src/error.rs b/mempool/src/error.rs index a1e569d052..6feabee692 100644 --- a/mempool/src/error.rs +++ b/mempool/src/error.rs @@ -16,61 +16,57 @@ pub enum Error { #[derive(Debug, Error)] pub enum TxValidationError { - #[error("No Inputs")] + #[error("Transaction has no inputs.")] NoInputs, - #[error("No Ouputs")] + #[error("Transaction has no outputs.")] NoOutputs, - #[error("DuplicateInputs")] + #[error("Transaction has duplicate inputs.")] DuplicateInputs, - #[error("LooseCoinbase")] - LooseCoinbase, - #[error("OutPointNotFound {outpoint:?}")] + #[error("Outpoint not found: {outpoint:?}")] OutPointNotFound { outpoint: OutPoint, tx_id: Id, }, - #[error("ExceedsMaxBlockSize")] + #[error("Transaction exceeds the maximum block size.")] ExceedsMaxBlockSize, - #[error("TransactionAlreadyInMempool")] + #[error("Transaction already exists in the mempool.")] TransactionAlreadyInMempool, - #[error("ConflictWithIrreplaceableTransaction")] + #[error("Transaction conflicts with another, irreplaceable transaction.")] ConflictWithIrreplaceableTransaction, - #[error("InputValuesOverflow")] + #[error("The sum of the transaction's inputs' values overflows.")] InputValuesOverflow, - #[error("OutputValuesOverflow")] + #[error("The sum of the transaction's outputs' values overflows.")] OutputValuesOverflow, - #[error("InputsBelowOutputs")] + #[error("The sum of the transaction's inputs is smaller than the sum of its outputs.")] InputsBelowOutputs, - #[error("ReplacementFeeLowerThanOriginal: The replacement transaction has fee {replacement_fee:?}, the original transaction has fee {original_fee:?}")] + #[error("Replacement transaction has fee lower than the original. Replacement fee is {replacement_fee:?}, original fee {original_fee:?}")] ReplacementFeeLowerThanOriginal { replacement_tx: H256, replacement_fee: Amount, original_tx: H256, original_fee: Amount, }, - #[error("TooManyPotentialReplacements")] + #[error("Transaction would require too many replacements.")] TooManyPotentialReplacements, - #[error("SpendsNewUnconfirmedInput")] + #[error("Replacement transaction spends an unconfirmed input which was not spent by any of the original transactions.")] SpendsNewUnconfirmedOutput, - #[error("ConflictsFeeOverflow")] + #[error("The sum of the fees of this transaction's conflicts overflows.")] ConflictsFeeOverflow, - #[error("TransactionFeeLowerThanConflictsWithDescendants")] + #[error("Transaction pays a fee that is lower than the fee of its conflicts with their descendants.")] TransactionFeeLowerThanConflictsWithDescendants, - #[error("AdditionalFeesUnderflow")] + #[error("Underflow in computing transaction's additional fees.")] AdditionalFeesUnderflow, - #[error("InsufficientFeesToRelay")] + #[error("Transaction does not pay sufficient fees to be relayed.")] InsufficientFeesToRelay { tx_fee: Amount, relay_fee: Amount }, - #[error("InsufficientFeesToRelayRBF")] + #[error("Replacement transaction does not pay enough for its bandwidth.")] InsufficientFeesToRelayRBF, - #[error("RollingFeeThresholdNotMet")] + #[error("Rolling fee threshold not met.")] RollingFeeThresholdNotMet { minimum_fee: Amount, tx_fee: Amount }, - #[error("FeeRate error")] - FeeRateError, - #[error("AncestorFeeUpdateOverflow")] + #[error("Overflow encountered while updating ancestor fee.")] AncestorFeeUpdateOverflow, - #[error("Descendant of expired transaction")] + #[error("Transaction is a descendant of expired transaction.")] DescendantOfExpiredTransaction, - #[error("Internal Error")] + #[error("Internal Error.")] InternalError, } From 01f766c4fee795d84f7d06ff3f85fb391e0fd726 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Thu, 28 Jul 2022 16:40:03 +0300 Subject: [PATCH 138/153] Use Id instead of H256 where possible --- mempool/src/pool.rs | 86 +++++++++++++++++++++------------------ mempool/src/pool/tests.rs | 4 +- 2 files changed, 49 insertions(+), 41 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index a683cb4a88..806efc820b 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -112,25 +112,25 @@ trait TryGetFee { newtype! { #[derive(Debug)] - struct Ancestors(BTreeSet); + struct Ancestors(BTreeSet>); } newtype! { #[derive(Debug)] - struct Descendants(BTreeSet); + struct Descendants(BTreeSet>); } newtype! { #[derive(Debug)] - struct Conflicts(BTreeSet); + struct Conflicts(BTreeSet>); } #[derive(Debug, PartialEq, Eq, Clone)] struct TxMempoolEntry { tx: Transaction, fee: Amount, - parents: BTreeSet, - children: BTreeSet, + parents: BTreeSet>, + children: BTreeSet>, count_with_descendants: usize, fees_with_descendants: Amount, creation_time: Time, @@ -140,7 +140,7 @@ impl TxMempoolEntry { fn new( tx: Transaction, fee: Amount, - parents: BTreeSet, + parents: BTreeSet>, creation_time: Time, ) -> TxMempoolEntry { Self { @@ -158,23 +158,23 @@ impl TxMempoolEntry { self.count_with_descendants } - fn tx_id(&self) -> H256 { - self.tx.get_id().get() + fn tx_id(&self) -> Id { + self.tx.get_id() } - fn unconfirmed_parents(&self) -> impl Iterator { + fn unconfirmed_parents(&self) -> impl Iterator> { self.parents.iter() } - fn unconfirmed_children(&self) -> impl Iterator { + fn unconfirmed_children(&self) -> impl Iterator> { self.children.iter() } - fn get_children_mut(&mut self) -> &mut BTreeSet { + fn get_children_mut(&mut self) -> &mut BTreeSet> { &mut self.children } - fn get_parents_mut(&mut self) -> &mut BTreeSet { + fn get_parents_mut(&mut self) -> &mut BTreeSet> { &mut self.parents } @@ -312,19 +312,19 @@ struct MempoolStore { // Where feerate is computed as fee(tx)/size(tx) // Note that if we wish to follow Bitcoin Bore, "size" is not simply the encoded size, but // rather a value that takes into account witdess and sigop data (see CTxMemPoolEntry::GetTxSize). - txs_by_descendant_score: BTreeMap>, + txs_by_descendant_score: BTreeMap>>, // Entries that have remained in the mempool for a long time (see DEFAULT_MEMPOOL_EXPIRY) are // evicted. To efficiently know which entries to evict, we store the mempool entries sorted by // their creation time, from earliest to latest. - txs_by_creation_time: BTreeMap>, + txs_by_creation_time: BTreeMap>>, // TODO add txs_by_ancestor_score index, which will be used by the block production subsystem // to select the best transactions for the next block // // We keep the information of which outpoints are spent by entries currently in the mempool. // This allows us to recognize conflicts (double-spends) and handle them - spender_txs: BTreeMap, + spender_txs: BTreeMap>, } impl MempoolStore { @@ -366,14 +366,14 @@ impl MempoolStore { .map(|output| output.value()) } - fn get_entry(&self, id: &H256) -> Option<&TxMempoolEntry> { - self.txs_by_id.get(id) + fn get_entry(&self, id: &Id) -> Option<&TxMempoolEntry> { + self.txs_by_id.get(&id.get()) } fn append_to_parents(&mut self, entry: &TxMempoolEntry) { for parent in entry.unconfirmed_parents() { self.txs_by_id - .get_mut(parent) + .get_mut(&parent.get()) .expect("append_to_parents") .get_children_mut() .insert(entry.tx_id()); @@ -383,7 +383,7 @@ impl MempoolStore { fn remove_from_parents(&mut self, entry: &TxMempoolEntry) { for parent in entry.unconfirmed_parents() { self.txs_by_id - .get_mut(parent) + .get_mut(&parent.get()) .expect("remove_from_parents") .get_children_mut() .remove(&entry.tx_id()); @@ -393,7 +393,7 @@ impl MempoolStore { fn remove_from_children(&mut self, entry: &TxMempoolEntry) { for child in entry.unconfirmed_children() { self.txs_by_id - .get_mut(child) + .get_mut(&child.get()) .expect("remove_from_children") .get_parents_mut() .remove(&entry.tx_id()); @@ -402,7 +402,7 @@ impl MempoolStore { fn update_ancestor_state_for_add(&mut self, entry: &TxMempoolEntry) -> Result<(), Error> { for ancestor in entry.unconfirmed_ancestors(self).0 { - let ancestor = self.txs_by_id.get_mut(&ancestor).expect("ancestor"); + let ancestor = self.txs_by_id.get_mut(&ancestor.get()).expect("ancestor"); ancestor.fees_with_descendants = (ancestor.fees_with_descendants + entry.fee) .ok_or(TxValidationError::AncestorFeeUpdateOverflow)?; ancestor.count_with_descendants += 1; @@ -412,7 +412,7 @@ impl MempoolStore { fn update_ancestor_state_for_drop(&mut self, entry: &TxMempoolEntry) { for ancestor in entry.unconfirmed_ancestors(self).0 { - let ancestor = self.txs_by_id.get_mut(&ancestor).expect("ancestor"); + let ancestor = self.txs_by_id.get_mut(&ancestor.get()).expect("ancestor"); ancestor.fees_with_descendants = (ancestor.fees_with_descendants - entry.fee).expect("fee with descendants"); ancestor.count_with_descendants -= 1; @@ -438,11 +438,11 @@ impl MempoolStore { let creation_time = entry.creation_time; let tx_id = entry.tx_id(); - self.txs_by_id.insert(tx_id, entry.clone()); + self.txs_by_id.insert(tx_id.get(), entry.clone()); self.add_to_descendant_score_index(&entry); self.txs_by_creation_time.entry(creation_time).or_default().insert(tx_id); - assert!(self.txs_by_id.get(&tx_id).is_some()); + assert!(self.txs_by_id.get(&tx_id.get()).is_some()); Ok(()) } @@ -463,7 +463,8 @@ impl MempoolStore { entries.retain(|id| !ancestors.contains(id)) } for ancestor_id in ancestors.0 { - let ancestor = self.txs_by_id.get(&ancestor_id).expect("Inconsistent mempool state"); + let ancestor = + self.txs_by_id.get(&ancestor_id.get()).expect("Inconsistent mempool state"); self.txs_by_descendant_score .entry(ancestor.fees_with_descendants.into()) .or_default() @@ -480,8 +481,8 @@ impl MempoolStore { self.update_ancestor_state_for_drop(&entry); self.drop_tx(&entry); } else { - assert!(!self.txs_by_descendant_score.values().flatten().any(|id| *id == tx_id.get())); - assert!(!self.spender_txs.iter().any(|(_, id)| *id == tx_id.get())); + assert!(!self.txs_by_descendant_score.values().flatten().any(|id| *id == *tx_id)); + assert!(!self.spender_txs.iter().any(|(_, id)| *id == *tx_id)); } } @@ -516,7 +517,7 @@ impl MempoolStore { fn drop_conflicts(&mut self, conflicts: Conflicts) { for conflict in conflicts.0 { - self.remove_tx(&Id::new(conflict)) + self.remove_tx(&conflict) } } @@ -531,14 +532,14 @@ impl MempoolStore { self.remove_tx(&entry.tx.get_id()); for descendant_id in descendants.0 { // It may be that this descendant has several ancestors and has already been removed - if let Some(descendant) = self.txs_by_id.get(&descendant_id).cloned() { + if let Some(descendant) = self.txs_by_id.get(&descendant_id.get()).cloned() { self.remove_tx(&descendant.tx.get_id()) } } } } - fn find_conflicting_tx(&self, outpoint: &OutPoint) -> Option { + fn find_conflicting_tx(&self, outpoint: &OutPoint) -> Option> { self.spender_txs.get(outpoint).cloned() } } @@ -636,8 +637,8 @@ where let parents = tx .inputs() .iter() - .map(|input| input.outpoint().tx_id().get_tx_id().expect("Not coinbase").get()) - .filter_map(|id| self.store.txs_by_id.contains_key(&id).then(|| id)) + .map(|input| *input.outpoint().tx_id().get_tx_id().expect("Not coinbase")) + .filter_map(|id| self.store.txs_by_id.contains_key(&id.get()).then(|| id)) .collect::>(); let fee = self.try_get_fee(&tx)?; @@ -823,10 +824,13 @@ where fn pays_more_than_conflicts_with_descendants( &self, tx: &Transaction, - conflicts_with_descendants: &BTreeSet, + conflicts_with_descendants: &BTreeSet>, ) -> Result { let conflicts_with_descendants = conflicts_with_descendants.iter().map(|conflict_id| { - self.store.txs_by_id.get(conflict_id).expect("tx should exist in mempool") + self.store + .txs_by_id + .get(&conflict_id.get()) + .expect("tx should exist in mempool") }); let total_conflict_fees = conflicts_with_descendants @@ -877,7 +881,7 @@ where replacement_tx: tx.get_id().get(), replacement_fee, original_fee: conflict.fee, - original_tx: conflict.tx_id(), + original_tx: conflict.tx_id().get(), }) }, ) @@ -886,7 +890,7 @@ where fn potential_replacements_within_limit( &self, conflicts: &[&TxMempoolEntry], - ) -> Result, TxValidationError> { + ) -> Result>, TxValidationError> { let mut num_potential_replacements = 0; for conflict in conflicts { num_potential_replacements += conflict.count_with_descendants(); @@ -938,7 +942,7 @@ where .txs_by_creation_time .values() .flatten() - .map(|entry_id| self.store.txs_by_id.get(entry_id).expect("entry should exist")) + .map(|entry_id| self.store.txs_by_id.get(&entry_id.get()).expect("entry should exist")) .filter(|entry| { let now = self.clock.get_time(); let expired = now.saturating_sub(entry.creation_time) > self.max_tx_age; @@ -973,8 +977,12 @@ where .flatten() .next() .expect("pool not empty"); - let removed = - self.store.txs_by_id.get(removed_id).expect("tx with id should exist").clone(); + let removed = self + .store + .txs_by_id + .get(&removed_id.get()) + .expect("tx with id should exist") + .clone(); log::debug!( "Mempool trim: Evicting tx {} which has a descendant score of {:?} and has size {}", diff --git a/mempool/src/pool/tests.rs b/mempool/src/pool/tests.rs index ff127bb3d2..b6cd1f2110 100644 --- a/mempool/src/pool/tests.rs +++ b/mempool/src/pool/tests.rs @@ -1658,9 +1658,9 @@ fn check_txs_sorted_by_descendant_sore( log::debug!("i = {}", i); let tx_id = txs_by_descendant_score.get(i).unwrap(); let next_tx_id = txs_by_descendant_score.get(i + 1).unwrap(); - let entry_score = mempool.store.txs_by_id.get(tx_id).unwrap().fees_with_descendants; + let entry_score = mempool.store.txs_by_id.get(&tx_id.get()).unwrap().fees_with_descendants; let next_entry_score = - mempool.store.txs_by_id.get(next_tx_id).unwrap().fees_with_descendants; + mempool.store.txs_by_id.get(&next_tx_id.get()).unwrap().fees_with_descendants; log::debug!("entry_score: {:?}", entry_score); log::debug!("next_entry_score: {:?}", next_entry_score); assert!(entry_score <= next_entry_score) From 168509f6c0486a2ee0e4d9bb0438d83fe394588d Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Mon, 1 Aug 2022 15:52:54 +0300 Subject: [PATCH 139/153] Use ensure in mempool --- mempool/src/pool.rs | 45 ++++++++++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 806efc820b..209802766d 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -737,21 +737,25 @@ where fn pays_minimum_mempool_fee(&self, tx: &Transaction) -> Result<(), TxValidationError> { let tx_fee = self.try_get_fee(tx)?; let minimum_fee = self.get_update_minimum_mempool_fee(tx); - (tx_fee >= minimum_fee) - .then(|| ()) - .ok_or(TxValidationError::RollingFeeThresholdNotMet { + ensure!( + tx_fee >= minimum_fee, + TxValidationError::RollingFeeThresholdNotMet { minimum_fee, tx_fee, - }) + } + ); + Ok(()) } fn pays_minimum_relay_fees(&self, tx: &Transaction) -> Result<(), TxValidationError> { let tx_fee = self.try_get_fee(tx)?; let relay_fee = get_relay_fee(tx); log::debug!("tx_fee: {:?}, relay_fee: {:?}", tx_fee, relay_fee); - (tx_fee >= relay_fee) - .then(|| ()) - .ok_or(TxValidationError::InsufficientFeesToRelay { tx_fee, relay_fee }) + ensure!( + tx_fee >= relay_fee, + TxValidationError::InsufficientFeesToRelay { tx_fee, relay_fee } + ); + Ok(()) } fn rbf_checks(&self, tx: &Transaction) -> Result { @@ -816,9 +820,11 @@ where additional_fees, relay_fee ); - (additional_fees >= relay_fee) - .then(|| ()) - .ok_or(TxValidationError::InsufficientFeesToRelayRBF) + ensure!( + additional_fees >= relay_fee, + TxValidationError::InsufficientFeesToRelayRBF + ); + Ok(()) } fn pays_more_than_conflicts_with_descendants( @@ -839,9 +845,10 @@ where .ok_or(TxValidationError::ConflictsFeeOverflow)?; let replacement_fee = self.try_get_fee(tx)?; - (replacement_fee > total_conflict_fees) - .then(|| ()) - .ok_or(TxValidationError::TransactionFeeLowerThanConflictsWithDescendants)?; + ensure!( + replacement_fee > total_conflict_fees, + TxValidationError::TransactionFeeLowerThanConflictsWithDescendants + ); Ok(total_conflict_fees) } @@ -912,14 +919,14 @@ where let id = entry.tx.get_id().get(); self.store.add_tx(entry)?; self.remove_expired_transactions(); - self.store - .txs_by_id - .contains_key(&id) - .then(|| ()) - .ok_or(TxValidationError::DescendantOfExpiredTransaction)?; + ensure!( + self.store.txs_by_id.contains_key(&id), + TxValidationError::DescendantOfExpiredTransaction + ); self.limit_mempool_size()?; - self.store.txs_by_id.contains_key(&id).then(|| ()).ok_or(Error::MempoolFull) + ensure!(self.store.txs_by_id.contains_key(&id), Error::MempoolFull); + Ok(()) } fn limit_mempool_size(&mut self) -> Result<(), Error> { From 56e4da27e406ff01407136c30098a44b6a5efee6 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Mon, 1 Aug 2022 16:01:45 +0300 Subject: [PATCH 140/153] Use from attribute to generate conversion TxValidationError->MempoolError --- mempool/src/error.rs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/mempool/src/error.rs b/mempool/src/error.rs index 6feabee692..4a2d4ebb67 100644 --- a/mempool/src/error.rs +++ b/mempool/src/error.rs @@ -11,7 +11,7 @@ pub enum Error { #[error("Mempool is full")] MempoolFull, #[error(transparent)] - TxValidationError(TxValidationError), + TxValidationError(#[from] TxValidationError), } #[derive(Debug, Error)] @@ -69,9 +69,3 @@ pub enum TxValidationError { #[error("Internal Error.")] InternalError, } - -impl From for Error { - fn from(e: TxValidationError) -> Self { - Error::TxValidationError(e) - } -} From b98edf81177bf0973ac3a1b120e773c2d9ccba89 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Mon, 1 Aug 2022 16:04:45 +0300 Subject: [PATCH 141/153] Suppress clippy float arithmetic warning --- mempool/src/pool.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 209802766d..40936b3f73 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -247,6 +247,7 @@ struct RollingFeeRate { } impl RollingFeeRate { + #[allow(clippy::float_arithmetic)] fn decay_fee(mut self, halflife: Time, current_time: Time) -> Self { log::trace!( "decay_fee: old fee rate: {:?}\nCurrent time: {:?}\nLast Rolling Fee Update: {:?}\nHalflife: {:?}", From 48d61d03f69f5ad165ba715ff3e170281b249dd4 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Mon, 1 Aug 2022 16:11:30 +0300 Subject: [PATCH 142/153] Make PartialEq, Ord implementations consistent for TxMempoolEntry --- mempool/src/pool.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 40936b3f73..b2b36ce94f 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -125,7 +125,7 @@ newtype! { struct Conflicts(BTreeSet>); } -#[derive(Debug, PartialEq, Eq, Clone)] +#[derive(Debug, Eq, Clone)] struct TxMempoolEntry { tx: Transaction, fee: Amount, @@ -228,6 +228,12 @@ impl PartialOrd for TxMempoolEntry { } } +impl PartialEq for TxMempoolEntry { + fn eq(&self, other: &Self) -> bool { + self.tx_id() == other.tx_id() + } +} + impl Ord for TxMempoolEntry { fn cmp(&self, other: &Self) -> Ordering { other.tx_id().cmp(&self.tx_id()) From 91947ce5fa6436f2159ec6f8c5005a4e21cba7a6 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Mon, 1 Aug 2022 17:54:07 +0300 Subject: [PATCH 143/153] Correct order of operations in feerate calculation --- mempool/src/feerate.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mempool/src/feerate.rs b/mempool/src/feerate.rs index 1ff4df7cc5..ac056531cb 100644 --- a/mempool/src/feerate.rs +++ b/mempool/src/feerate.rs @@ -27,7 +27,7 @@ impl FeeRate { pub(crate) fn compute_fee(&self, size: usize) -> Amount { let size = u128::try_from(size).expect("compute_fee conversion"); - Amount::from_atoms(self.atoms_per_kb * size / 1000) + Amount::from_atoms((self.atoms_per_kb * size) / 1000) } pub(crate) fn atoms_per_kb(&self) -> u128 { From 51fc129ff6983450d6c7db6ed85eb3a87f4cd9f6 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Mon, 1 Aug 2022 18:09:37 +0300 Subject: [PATCH 144/153] Round up in fee computation --- mempool/src/feerate.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mempool/src/feerate.rs b/mempool/src/feerate.rs index ac056531cb..0e881ce1b6 100644 --- a/mempool/src/feerate.rs +++ b/mempool/src/feerate.rs @@ -27,7 +27,8 @@ impl FeeRate { pub(crate) fn compute_fee(&self, size: usize) -> Amount { let size = u128::try_from(size).expect("compute_fee conversion"); - Amount::from_atoms((self.atoms_per_kb * size) / 1000) + // +999 for ceil operation + Amount::from_atoms((self.atoms_per_kb * size + 999) / 1000) } pub(crate) fn atoms_per_kb(&self) -> u128 { From 78653366384f584b395820f2204b47b07fc8b78c Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Mon, 1 Aug 2022 18:15:38 +0300 Subject: [PATCH 145/153] Add documentation --- mempool/src/pool.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index b2b36ce94f..849f8cf43a 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -96,8 +96,17 @@ pub trait Mempool { fn create(chain_state: C, clock: T, memory_usage_estimator: M) -> Self; fn add_transaction(&mut self, tx: Transaction) -> Result<(), Error>; fn get_all(&self) -> Vec<&Transaction>; + + // Returns `true` if the mempool contains a transaction with the given id, `false` otherwise. fn contains_transaction(&self, tx: &Id) -> bool; + + // Drops a transaction from the mempool, updating its in-mempool parents and children. This + // operation removes the transaction from all indices, as well as updating the state (fee, + // count with descendants) of the transaction's ancestors. In addition, outpoints spent by this + // transaction are no longer marked as spent fn drop_transaction(&mut self, tx: &Id); + + // Add/remove transactions to/from the mempool according to a new tip fn new_tip_set(&mut self, chain_state: C); } From b15c8688c3b29739957d7491b10f1597a83acd33 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Mon, 1 Aug 2022 18:38:30 +0300 Subject: [PATCH 146/153] Fix test tx_too_big --- mempool/src/pool/tests.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/mempool/src/pool/tests.rs b/mempool/src/pool/tests.rs index b6cd1f2110..b32306adf4 100644 --- a/mempool/src/pool/tests.rs +++ b/mempool/src/pool/tests.rs @@ -657,8 +657,14 @@ fn outpoint_not_found() -> anyhow::Result<()> { #[test] fn tx_too_big() -> anyhow::Result<()> { let mut mempool = setup(); + let single_output_size = TxOutput::new( + Amount::from_atoms(100), + OutputPurpose::Transfer(Destination::AnyoneCanSpend), + ) + .encoded_size(); + let too_many_outputs = MAX_BLOCK_SIZE_BYTES / single_output_size; let tx = TxGenerator::new() - .with_num_outputs(400_000) + .with_num_outputs(too_many_outputs) .generate_tx(&mempool) .expect("generate_tx failed"); assert!(matches!( From c7fea8bcfe0c821df0624cc947c705d4021e8003 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Mon, 1 Aug 2022 18:53:50 +0300 Subject: [PATCH 147/153] Move test machinery to different submodule --- mempool/src/pool/tests.rs | 56 ++------------------------------- mempool/src/pool/tests/utils.rs | 55 ++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 53 deletions(-) create mode 100644 mempool/src/pool/tests/utils.rs diff --git a/mempool/src/pool/tests.rs b/mempool/src/pool/tests.rs index b32306adf4..584d49b4fd 100644 --- a/mempool/src/pool/tests.rs +++ b/mempool/src/pool/tests.rs @@ -7,61 +7,11 @@ use core::panic; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; -const DUMMY_WITNESS_MSG: &[u8] = b"dummy_witness_msg"; - -#[derive(Debug, PartialEq, Eq, Clone)] -struct ValuedOutPoint { - outpoint: OutPoint, - value: Amount, -} - -impl std::cmp::PartialOrd for ValuedOutPoint { - fn partial_cmp(&self, other: &Self) -> Option { - other.value.partial_cmp(&self.value) - } -} - -impl std::cmp::Ord for ValuedOutPoint { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - other.value.cmp(&self.value) - } -} - -fn dummy_input() -> TxInput { - let outpoint_source_id = OutPointSourceId::Transaction(Id::new(H256::zero())); - let output_index = 0; - let witness = DUMMY_WITNESS_MSG.to_vec(); - TxInput::new( - outpoint_source_id, - output_index, - InputWitness::NoSignature(Some(witness)), - ) -} +mod utils; -fn dummy_output() -> TxOutput { - let value = Amount::from_atoms(0); - let purpose = OutputPurpose::Transfer(Destination::AnyoneCanSpend); - TxOutput::new(value, purpose) -} +use self::utils::*; -fn estimate_tx_size(num_inputs: usize, num_outputs: usize) -> usize { - let inputs = (0..num_inputs).into_iter().map(|_| dummy_input()).collect(); - let outputs = (0..num_outputs).into_iter().map(|_| dummy_output()).collect(); - let flags = 0; - let locktime = 0; - let size = Transaction::new(flags, inputs, outputs, locktime).unwrap().encoded_size(); - // Take twice the encoded size of the dummy tx.Real Txs are larger than these dummy ones, - // but taking 3 times the size seems to ensure our txs won't fail the minimum relay fee - // validation (see the function `pays_minimum_relay_fees`) - let result = 3 * size; - log::debug!( - "estimated size for tx with {} inputs and {} outputs: {}", - num_inputs, - num_outputs, - result - ); - result -} +const DUMMY_WITNESS_MSG: &[u8] = b"dummy_witness_msg"; #[test] fn dummy_size() { diff --git a/mempool/src/pool/tests/utils.rs b/mempool/src/pool/tests/utils.rs new file mode 100644 index 0000000000..47bcdac2e1 --- /dev/null +++ b/mempool/src/pool/tests/utils.rs @@ -0,0 +1,55 @@ +use super::*; + +#[derive(Debug, PartialEq, Eq, Clone)] +pub(in crate::pool::tests) struct ValuedOutPoint { + pub(in crate::pool::tests) outpoint: OutPoint, + pub(in crate::pool::tests) value: Amount, +} + +impl std::cmp::PartialOrd for ValuedOutPoint { + fn partial_cmp(&self, other: &Self) -> Option { + other.value.partial_cmp(&self.value) + } +} + +impl std::cmp::Ord for ValuedOutPoint { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + other.value.cmp(&self.value) + } +} + +fn dummy_input() -> TxInput { + let outpoint_source_id = OutPointSourceId::Transaction(Id::new(H256::zero())); + let output_index = 0; + let witness = DUMMY_WITNESS_MSG.to_vec(); + TxInput::new( + outpoint_source_id, + output_index, + InputWitness::NoSignature(Some(witness)), + ) +} + +fn dummy_output() -> TxOutput { + let value = Amount::from_atoms(0); + let purpose = OutputPurpose::Transfer(Destination::AnyoneCanSpend); + TxOutput::new(value, purpose) +} + +pub(in crate::pool::tests) fn estimate_tx_size(num_inputs: usize, num_outputs: usize) -> usize { + let inputs = (0..num_inputs).into_iter().map(|_| dummy_input()).collect(); + let outputs = (0..num_outputs).into_iter().map(|_| dummy_output()).collect(); + let flags = 0; + let locktime = 0; + let size = Transaction::new(flags, inputs, outputs, locktime).unwrap().encoded_size(); + // Take twice the encoded size of the dummy tx.Real Txs are larger than these dummy ones, + // but taking 3 times the size seems to ensure our txs won't fail the minimum relay fee + // validation (see the function `pays_minimum_relay_fees`) + let result = 3 * size; + log::debug!( + "estimated size for tx with {} inputs and {} outputs: {}", + num_inputs, + num_outputs, + result + ); + result +} From 895615e09bdaa6c972070c148281a1ef693ad2ae Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Tue, 2 Aug 2022 21:23:44 +0300 Subject: [PATCH 148/153] Refactor drop_tx --- mempool/src/pool.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 849f8cf43a..91e0c76825 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -493,7 +493,6 @@ impl MempoolStore { fn remove_tx(&mut self, tx_id: &Id) { log::info!("remove_tx: {}", tx_id.get()); if let Some(entry) = self.txs_by_id.remove(&tx_id.get()) { - self.update_for_drop(&entry); self.update_ancestor_state_for_drop(&entry); self.drop_tx(&entry); } else { @@ -508,6 +507,7 @@ impl MempoolStore { } fn drop_tx(&mut self, entry: &TxMempoolEntry) { + self.update_for_drop(entry); self.remove_from_descendant_score_index(entry); self.txs_by_creation_time.entry(entry.creation_time).and_modify(|entries| { entries.remove(&entry.tx_id()).then(|| ()).expect("Inconsistent mempool store") From bf2fae094ac2b37ba289549f6b032e9a62ed752e Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Tue, 2 Aug 2022 21:25:42 +0300 Subject: [PATCH 149/153] Improvements to FeeRate::div_up --- mempool/src/feerate.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/mempool/src/feerate.rs b/mempool/src/feerate.rs index 0e881ce1b6..a3a49fb3bd 100644 --- a/mempool/src/feerate.rs +++ b/mempool/src/feerate.rs @@ -35,9 +35,11 @@ impl FeeRate { self.atoms_per_kb } - fn div_up(fee: u128, tx_size: usize) -> u128 { - let tx_size = u128::try_from(tx_size).expect("div_up conversion"); - (fee + tx_size - 1) / tx_size + // TODO Use NonZeroUsize for divisor + fn div_up(dividend: u128, divisor: usize) -> u128 { + debug_assert!(divisor != 0); + let divisor = u128::try_from(divisor).expect("div_up conversion"); + (dividend + divisor - 1) / divisor } } From 7de1ead86b9dc1f0e01a176aeca30e01e00862fb Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Tue, 2 Aug 2022 21:27:11 +0300 Subject: [PATCH 150/153] Remove no longer needed assert --- mempool/src/pool.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 91e0c76825..2a728406ab 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -458,7 +458,6 @@ impl MempoolStore { self.add_to_descendant_score_index(&entry); self.txs_by_creation_time.entry(creation_time).or_default().insert(tx_id); - assert!(self.txs_by_id.get(&tx_id.get()).is_some()); Ok(()) } From 919df10e618121bfdaecf2c716e45188b3671a17 Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Tue, 2 Aug 2022 21:36:15 +0300 Subject: [PATCH 151/153] FeeRate division used only in tests --- mempool/src/feerate.rs | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/mempool/src/feerate.rs b/mempool/src/feerate.rs index a3a49fb3bd..2e7eb456af 100644 --- a/mempool/src/feerate.rs +++ b/mempool/src/feerate.rs @@ -1,5 +1,3 @@ -use std::num::NonZeroU128; - use common::primitives::amount::Amount; lazy_static::lazy_static! { @@ -51,18 +49,19 @@ impl std::ops::Add for FeeRate { } } -impl std::ops::Div for FeeRate { - type Output = FeeRate; - fn div(self, rhs: NonZeroU128) -> Self::Output { - FeeRate { - atoms_per_kb: self.atoms_per_kb / rhs, - } - } -} - #[cfg(test)] mod tests { use super::*; + use std::num::NonZeroU128; + + impl std::ops::Div for FeeRate { + type Output = FeeRate; + fn div(self, rhs: NonZeroU128) -> Self::Output { + FeeRate { + atoms_per_kb: self.atoms_per_kb / rhs, + } + } + } #[test] fn test_div_up() { From bfa93250141c528b0be31c9d70d303bc49c927fc Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Tue, 2 Aug 2022 21:37:10 +0300 Subject: [PATCH 152/153] Mempool TODOs --- mempool/src/feerate.rs | 3 +++ mempool/src/pool.rs | 3 +++ 2 files changed, 6 insertions(+) diff --git a/mempool/src/feerate.rs b/mempool/src/feerate.rs index 2e7eb456af..65faf42d69 100644 --- a/mempool/src/feerate.rs +++ b/mempool/src/feerate.rs @@ -5,6 +5,9 @@ lazy_static::lazy_static! { pub(crate) static ref INCREMENTAL_RELAY_THRESHOLD: FeeRate = FeeRate::new(Amount::from_atoms(500)); } +// TODO we should reconsider using Amount and define functions for the specific operations required +// on FeeRate. Previous attempts to do this did not pan out, but this should be revisited to avoid +// dangerous arithmetic operations #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub(crate) struct FeeRate { atoms_per_kb: u128, diff --git a/mempool/src/pool.rs b/mempool/src/pool.rs index 2a728406ab..abb88b6fc8 100644 --- a/mempool/src/pool.rs +++ b/mempool/src/pool.rs @@ -61,6 +61,7 @@ where T: GetTime, M: GetMemoryUsage, { + // TODO this calculation is already done in ChainState, reuse it fn try_get_fee(&self, tx: &Transaction) -> Result { let inputs = tx .inputs() @@ -730,6 +731,8 @@ where return Err(TxValidationError::DuplicateInputs); } + // TODO see this issue: + // https://github.com/mintlayer/mintlayer-core/issues/331 if tx.encoded_size() > MAX_BLOCK_SIZE_BYTES { return Err(TxValidationError::ExceedsMaxBlockSize); } From a78b0cf6285bfab63007f5d620157497b008f81e Mon Sep 17 00:00:00 2001 From: Roy Ben Abraham Date: Wed, 3 Aug 2022 14:36:33 +0300 Subject: [PATCH 153/153] Update deny.toml --- deny.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deny.toml b/deny.toml index 5b73398da9..87d0df7700 100644 --- a/deny.toml +++ b/deny.toml @@ -44,6 +44,7 @@ skip = [ {name = "textwrap"}, {name = "wasi"}, {name = "winapi"}, + {name = "windows"}, {name = "windows_aarch64_msvc" }, {name = "windows_i686_gnu" }, {name = "windows_i686_msvc" }, @@ -70,6 +71,7 @@ allow = [ "LicenseRef-webpki", "WTFPL", "BSL-1.0", + "Unicode-DFS-2016", "Unlicense",#this is a specific license rather than no license at all ] #deny a license not in this set of licenses