Allow using spent unconfirmed UTXOs in add_utxo

UTXOs will be deleted from the database only when the
transaction spending them is confirmed, this way you can
build a tx double-spending one in mempool using `add_utxo`.
listunspent won't return spent in mempool utxos, effectively excluding them from the
coin selection and balance calculation.
Closes #414

Author: danielabrozzoni

CHANGELOG.md

@@ -34,6 +34,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Add new method on the `TxBuilder` to embed data in the transaction via `OP_RETURN`. To allow that a fix to check the dust only on spendable output has been introduced.
 - Update the `Database` trait to store the last sync timestamp and block height
 - Rename `ConfirmationTime` to `BlockTime`
+- Add `is_spent_unconfirmed` field in `LocalUtxo`, which marks an output that is currently being used in an unconfirmed transaction. This UTXO won't be selected in an automatic coin selection and its value won't be added to the balance, but it can be added manually to a transaction using `add_utxo`.
 
 ## [v0.13.0] - [v0.12.0]
 

src/blockchain/compact_filters/mod.rs

@@ -163,11 +163,28 @@ impl CompactFiltersBlockchain {
             if let Some(previous_output) = database.get_previous_output(&input.previous_output)? {
                 inputs_sum += previous_output.value;
 
-                if database.is_mine(&previous_output.script_pubkey)? {
+                // this output is ours, we have a path to derive it
+                if let Some((keychain, _)) =
+                    database.get_path_from_script_pubkey(&previous_output.script_pubkey)?
+                {
                     outgoing += previous_output.value;
 
-                    debug!("{} input #{} is mine, removing from utxo", tx.txid(), i);
-                    updates.del_utxo(&input.previous_output)?;
+                    if height.is_none() {
+                        debug!(
+                            "{} input #{} is mine, setting utxo as spent_unconfirmed",
+                            tx.txid(),
+                            i
+                        );
+                        updates.set_utxo(&LocalUtxo {
+                            outpoint: input.previous_output,
+                            txout: previous_output.clone(),
+                            keychain,
+                            is_spent_unconfirmed: true,
+                        })?;
+                    } else {
+                        debug!("{} input #{} is mine, removing from utxo", tx.txid(), i);
+                        updates.del_utxo(&input.previous_output)?;
+                    }
                 }
             }
         }
@@ -185,6 +202,7 @@ impl CompactFiltersBlockchain {
                     outpoint: OutPoint::new(tx.txid(), i as u32),
                     txout: output.clone(),
                     keychain,
+                    is_spent_unconfirmed: false,
                 })?;
                 incoming += output.value;
 

src/blockchain/rpc.rs

@@ -213,19 +213,44 @@ impl Blockchain for RpcBlockchain {
             .list_unspent(Some(0), None, None, Some(true), None)?;
         debug!("current_utxo len {}", current_utxo.len());
 
+        let mut spent_in_mempool = Vec::new();
         //TODO supported up to 1_000 txs, should use since_blocks or do paging
         let list_txs = self
             .client
             .list_transactions(None, Some(1_000), None, Some(true))?;
         let mut list_txs_ids = HashSet::new();
 
         for tx_result in list_txs.iter().filter(|t| {
-            // list_txs returns all conflicting tx we want to
+            // list_txs returns all conflicting txs, we want to
             // filter out replaced tx => unconfirmed and not in the mempool
             t.info.confirmations > 0 || self.client.get_mempool_entry(&t.info.txid).is_ok()
         }) {
             let txid = tx_result.info.txid;
             list_txs_ids.insert(txid);
+
+            let tx_result = self.client.get_transaction(&txid, Some(true))?;
+            let tx: Transaction = deserialize(&tx_result.hex)?;
+
+            // If it's unconfirmed, we may want to add its inputs to the currently spent in mempool
+            if tx_result.info.blockhash.is_none() {
+                for input in tx.input.iter() {
+                    // This input is mine if I have both the previous tx and the script_pubkey in
+                    // the db
+                    if let Some(txout) = db.get_previous_output(&input.previous_output)? {
+                        if let Some(keychain) =
+                            db.get_path_from_script_pubkey(&txout.script_pubkey)?
+                        {
+                            spent_in_mempool.push(LocalUtxo {
+                                outpoint: input.previous_output,
+                                keychain: keychain.0,
+                                txout,
+                                is_spent_unconfirmed: true,
+                            });
+                        }
+                    }
+                }
+            }
+
             if let Some(mut known_tx) = known_txs.get_mut(&txid) {
                 let confirmation_time =
                     BlockTime::new(tx_result.info.blockheight, tx_result.info.blocktime);
@@ -239,9 +264,6 @@ impl Blockchain for RpcBlockchain {
                     db.set_tx(known_tx)?;
                 }
             } else {
-                //TODO check there is already the raw tx in db?
-                let tx_result = self.client.get_transaction(&txid, Some(true))?;
-                let tx: Transaction = deserialize(&tx_result.hex)?;
                 let mut received = 0u64;
                 let mut sent = 0u64;
                 for output in tx.output.iter() {
@@ -302,19 +324,27 @@ impl Blockchain for RpcBlockchain {
                         value: u.amount.as_sat(),
                         script_pubkey: u.script_pub_key,
                     },
+                    // Here we just claim that this utxo is not spent in mempool; if
+                    // instead it is, the `spent_in_mempool` loop below will
+                    // take care of replacing it
+                    is_spent_unconfirmed: false,
                 })
             })
             .collect::<Result<_, Error>>()?;
 
         let spent: HashSet<_> = known_utxos.difference(&current_utxos).collect();
-        for s in spent {
-            debug!("removing utxo: {:?}", s);
-            db.del_utxo(&s.outpoint)?;
+        for utxo in spent {
+            debug!("removing utxo: {:?}", utxo);
+            db.del_utxo(&utxo.outpoint)?;
         }
         let received: HashSet<_> = current_utxos.difference(&known_utxos).collect();
-        for s in received {
-            debug!("adding utxo: {:?}", s);
-            db.set_utxo(s)?;
+        for utxo in received {
+            debug!("adding utxo: {:?}", utxo);
+            db.set_utxo(utxo)?;
+        }
+        for utxo in spent_in_mempool {
+            debug!("adding utxo: {:?}", utxo);
+            db.set_utxo(&utxo)?;
         }
 
         for (keykind, index) in indexes {

src/blockchain/script_sync.rs

@@ -332,41 +332,62 @@ impl<'a, D: BatchDatabase> State<'a, D> {
             batch.del_tx(txid, true)?;
         }
 
-        // Set every tx we observed
+        let mut spent_utxos = HashSet::new();
+        let mut spent_unconfirmed_utxos = HashSet::new();
+
+        // track all the spent and spent_unconfirmed utxos
         for finished_tx in &finished_txs {
             let tx = finished_tx
                 .transaction
                 .as_ref()
                 .expect("transaction will always be present here");
-            for (i, output) in tx.output.iter().enumerate() {
-                if let Some((keychain, _)) =
-                    self.db.get_path_from_script_pubkey(&output.script_pubkey)?
-                {
-                    // add utxos we own from the new transactions we've seen.
-                    batch.set_utxo(&LocalUtxo {
-                        outpoint: OutPoint {
-                            txid: finished_tx.txid,
-                            vout: i as u32,
-                        },
-                        txout: output.clone(),
-                        keychain,
-                    })?;
+            for input in &tx.input {
+                if finished_tx.confirmation_time.is_some() {
+                    spent_utxos.insert(&input.previous_output);
+                } else {
+                    spent_unconfirmed_utxos.insert(&input.previous_output);
                 }
             }
-            batch.set_tx(finished_tx)?;
         }
 
-        // we don't do this in the loop above since we may want to delete some of the utxos we
-        // just added in case there are new tranasactions that spend form each other.
+        // set every utxo we observed, unless it's already spent
+        // we don't do this in the loop above as we want to know all the spent outputs before
+        // adding the non-spent to the batch in case there are new tranasactions
+        // that spend form each other.
         for finished_tx in &finished_txs {
             let tx = finished_tx
                 .transaction
                 .as_ref()
                 .expect("transaction will always be present here");
-            for input in &tx.input {
-                // Delete any spent utxos
-                batch.del_utxo(&input.previous_output)?;
+            for (i, output) in tx.output.iter().enumerate() {
+                if let Some((keychain, _)) =
+                    self.db.get_path_from_script_pubkey(&output.script_pubkey)?
+                {
+                    // add utxos we own from the new transactions we've seen.
+                    let outpoint = OutPoint {
+                        txid: finished_tx.txid,
+                        vout: i as u32,
+                    };
+
+                    // If it's not spent, we add it to the batch
+                    if spent_utxos.get(&outpoint).is_none() {
+                        batch.set_utxo(&LocalUtxo {
+                            outpoint,
+                            txout: output.clone(),
+                            keychain,
+                            // Is this UTXO spent in a tx still in mempool?
+                            is_spent_unconfirmed: spent_unconfirmed_utxos.contains(&outpoint),
+                        })?;
+                    }
+                }
             }
+
+            batch.set_tx(finished_tx)?;
+        }
+
+        // delete all the spent utxos
+        for spent_utxo in spent_utxos {
+            batch.del_utxo(spent_utxo)?;
         }
 
         for (keychain, last_active_index) in self.last_active_index {

src/database/keyvalue.rs

@@ -43,6 +43,7 @@ macro_rules! impl_batch_operations {
             let value = json!({
                 "t": utxo.txout,
                 "i": utxo.keychain,
+                "s": utxo.is_spent_unconfirmed,
             });
             self.insert(key, serde_json::to_vec(&value)?)$($after_insert)*;
 
@@ -125,8 +126,9 @@ macro_rules! impl_batch_operations {
                     let mut val: serde_json::Value = serde_json::from_slice(&b)?;
                     let txout = serde_json::from_value(val["t"].take())?;
                     let keychain = serde_json::from_value(val["i"].take())?;
+                    let is_spent_unconfirmed = val.get_mut("s").and_then(|s| s.take().as_bool()).unwrap_or(false);
 
-                    Ok(Some(LocalUtxo { outpoint: outpoint.clone(), txout, keychain }))
+                    Ok(Some(LocalUtxo { outpoint: outpoint.clone(), txout, keychain, is_spent_unconfirmed, }))
                 }
             }
         }
@@ -246,11 +248,16 @@ impl Database for Tree {
                 let mut val: serde_json::Value = serde_json::from_slice(&v)?;
                 let txout = serde_json::from_value(val["t"].take())?;
                 let keychain = serde_json::from_value(val["i"].take())?;
+                let is_spent_unconfirmed = val
+                    .get_mut("s")
+                    .and_then(|s| s.take().as_bool())
+                    .unwrap_or(false);
 
                 Ok(LocalUtxo {
                     outpoint,
                     txout,
                     keychain,
+                    is_spent_unconfirmed,
                 })
             })
             .collect()
@@ -314,11 +321,16 @@ impl Database for Tree {
                 let mut val: serde_json::Value = serde_json::from_slice(&b)?;
                 let txout = serde_json::from_value(val["t"].take())?;
                 let keychain = serde_json::from_value(val["i"].take())?;
+                let is_spent_unconfirmed = val
+                    .get_mut("s")
+                    .and_then(|s| s.take().as_bool())
+                    .unwrap_or(false);
 
                 Ok(LocalUtxo {
                     outpoint: *outpoint,
                     txout,
                     keychain,
+                    is_spent_unconfirmed,
                 })
             })
             .transpose()

src/database/memory.rs

@@ -150,8 +150,10 @@ impl BatchOperations for MemoryDatabase {
 
     fn set_utxo(&mut self, utxo: &LocalUtxo) -> Result<(), Error> {
         let key = MapKey::Utxo(Some(&utxo.outpoint)).as_map_key();
-        self.map
-            .insert(key, Box::new((utxo.txout.clone(), utxo.keychain)));
+        self.map.insert(
+            key,
+            Box::new((utxo.txout.clone(), utxo.keychain, utxo.is_spent_unconfirmed)),
+        );
 
         Ok(())
     }
@@ -228,11 +230,12 @@ impl BatchOperations for MemoryDatabase {
         match res {
             None => Ok(None),
             Some(b) => {
-                let (txout, keychain) = b.downcast_ref().cloned().unwrap();
+                let (txout, keychain, is_spent_unconfirmed) = b.downcast_ref().cloned().unwrap();
                 Ok(Some(LocalUtxo {
                     outpoint: *outpoint,
                     txout,
                     keychain,
+                    is_spent_unconfirmed,
                 }))
             }
         }
@@ -326,11 +329,12 @@ impl Database for MemoryDatabase {
             .range::<Vec<u8>, _>((Included(&key), Excluded(&after(&key))))
             .map(|(k, v)| {
                 let outpoint = deserialize(&k[1..]).unwrap();
-                let (txout, keychain) = v.downcast_ref().cloned().unwrap();
+                let (txout, keychain, is_spent_unconfirmed) = v.downcast_ref().cloned().unwrap();
                 Ok(LocalUtxo {
                     outpoint,
                     txout,
                     keychain,
+                    is_spent_unconfirmed,
                 })
             })
             .collect()
@@ -389,11 +393,12 @@ impl Database for MemoryDatabase {
     fn get_utxo(&self, outpoint: &OutPoint) -> Result<Option<LocalUtxo>, Error> {
         let key = MapKey::Utxo(Some(outpoint)).as_map_key();
         Ok(self.map.get(&key).map(|b| {
-            let (txout, keychain) = b.downcast_ref().cloned().unwrap();
+            let (txout, keychain, is_spent_unconfirmed) = b.downcast_ref().cloned().unwrap();
             LocalUtxo {
                 outpoint: *outpoint,
                 txout,
                 keychain,
+                is_spent_unconfirmed,
             }
         }))
     }
@@ -526,6 +531,7 @@ macro_rules! populate_test_db {
                     vout: vout as u32,
                 },
                 keychain: $crate::KeychainKind::External,
+                is_spent_unconfirmed: false,
             })
             .unwrap();
         }

src/database/mod.rs

@@ -316,6 +316,7 @@ pub mod test {
             txout,
             outpoint,
             keychain: KeychainKind::External,
+            is_spent_unconfirmed: true,
         };
 
         tree.set_utxo(&utxo).unwrap();