Skip to content

Commit 227eede

Browse files
danielabrozzonidanielabrozzoni
committed
Never delete spent utxos from the database
A `is_spent` field is added to LocalUtxo; when a txo is spent we set this field to true instead of deleting the entire utxo from the database. This allows us to create txs double-spending txs already in blockchain.
1 parent adf7d0c commit 227eede

File tree

13 files changed

+155
-66
lines changed

13 files changed

+155
-66
lines changed

‎CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
1010
- `verify` flag removed from `TransactionDetails`.
1111
- Add `get_internal_address` to allow you to get internal addresses just as you get external addresses.
1212
- added `ensure_addresses_cached` to `Wallet` to let offline wallets load and cache addresses in their database
13+
- Add `is_spent` field to `LocalUtxo`; when we notice that a utxo has been spent we set `is_spent` field to true instead of deleting it from the db.
1314

1415
## [v0.16.1] - [v0.16.0]
1516

‎src/blockchain/compact_filters/mod.rs

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -163,11 +163,19 @@ impl CompactFiltersBlockchain {
163163
if let Some(previous_output) = database.get_previous_output(&input.previous_output)? {
164164
inputs_sum += previous_output.value;
165165

166-
if database.is_mine(&previous_output.script_pubkey)? {
166+
// this output is ours, we have a path to derive it
167+
if let Some((keychain, _)) =
168+
database.get_path_from_script_pubkey(&previous_output.script_pubkey)?
169+
{
167170
outgoing += previous_output.value;
168171

169-
debug!("{} input #{} is mine, removing from utxo", tx.txid(), i);
170-
updates.del_utxo(&input.previous_output)?;
172+
debug!("{} input #{} is mine, setting utxo as spent", tx.txid(), i);
173+
updates.set_utxo(&LocalUtxo {
174+
outpoint: input.previous_output,
175+
txout: previous_output.clone(),
176+
keychain,
177+
is_spent: true,
178+
})?;
171179
}
172180
}
173181
}
@@ -185,6 +193,7 @@ impl CompactFiltersBlockchain {
185193
outpoint: OutPoint::new(tx.txid(), i as u32),
186194
txout: output.clone(),
187195
keychain,
196+
is_spent: false,
188197
})?;
189198
incoming += output.value;
190199

‎src/blockchain/rpc.rs

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -220,7 +220,7 @@ impl Blockchain for RpcBlockchain {
220220
let mut list_txs_ids = HashSet::new();
221221

222222
for tx_result in list_txs.iter().filter(|t| {
223-
// list_txs returns all conflicting tx we want to
223+
// list_txs returns all conflicting txs, we want to
224224
// filter out replaced tx => unconfirmed and not in the mempool
225225
t.info.confirmations > 0 || self.client.get_mempool_entry(&t.info.txid).is_ok()
226226
}) {
@@ -302,19 +302,22 @@ impl Blockchain for RpcBlockchain {
302302
value: u.amount.as_sat(),
303303
script_pubkey: u.script_pub_key,
304304
},
305+
is_spent: false,
305306
})
306307
})
307308
.collect::<Result<_, Error>>()?;
308309

309310
let spent: HashSet<_> = known_utxos.difference(&current_utxos).collect();
310-
for s in spent {
311-
debug!("removing utxo: {:?}", s);
312-
db.del_utxo(&s.outpoint)?;
311+
for utxo in spent {
312+
debug!("setting as spent utxo: {:?}", utxo);
313+
let mut spent_utxo = utxo.clone();
314+
spent_utxo.is_spent = true;
315+
db.set_utxo(&spent_utxo)?;
313316
}
314317
let received: HashSet<_> = current_utxos.difference(&known_utxos).collect();
315-
for s in received {
316-
debug!("adding utxo: {:?}", s);
317-
db.set_utxo(s)?;
318+
for utxo in received {
319+
debug!("adding utxo: {:?}", utxo);
320+
db.set_utxo(utxo)?;
318321
}
319322

320323
for (keykind, index) in indexes {

‎src/blockchain/script_sync.rs

Lines changed: 26 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -332,7 +332,23 @@ impl<'a, D: BatchDatabase> State<'a, D> {
332332
batch.del_tx(txid, true)?;
333333
}
334334

335-
// Set every tx we observed
335+
let mut spent_utxos = HashSet::new();
336+
337+
// track all the spent utxos
338+
for finished_tx in &finished_txs {
339+
let tx = finished_tx
340+
.transaction
341+
.as_ref()
342+
.expect("transaction will always be present here");
343+
for input in &tx.input {
344+
spent_utxos.insert(&input.previous_output);
345+
}
346+
}
347+
348+
// set every utxo we observed, unless it's already spent
349+
// we don't do this in the loop above as we want to know all the spent outputs before
350+
// adding the non-spent to the batch in case there are new tranasactions
351+
// that spend form each other.
336352
for finished_tx in &finished_txs {
337353
let tx = finished_tx
338354
.transaction
@@ -343,30 +359,22 @@ impl<'a, D: BatchDatabase> State<'a, D> {
343359
self.db.get_path_from_script_pubkey(&output.script_pubkey)?
344360
{
345361
// add utxos we own from the new transactions we've seen.
362+
let outpoint = OutPoint {
363+
txid: finished_tx.txid,
364+
vout: i as u32,
365+
};
366+
346367
batch.set_utxo(&LocalUtxo {
347-
outpoint: OutPoint {
348-
txid: finished_tx.txid,
349-
vout: i as u32,
350-
},
368+
outpoint,
351369
txout: output.clone(),
352370
keychain,
371+
// Is this UTXO in the spent_utxos set?
372+
is_spent: spent_utxos.get(&outpoint).is_some(),
353373
})?;
354374
}
355375
}
356-
batch.set_tx(finished_tx)?;
357-
}
358376

359-
// we don't do this in the loop above since we may want to delete some of the utxos we
360-
// just added in case there are new tranasactions that spend form each other.
361-
for finished_tx in &finished_txs {
362-
let tx = finished_tx
363-
.transaction
364-
.as_ref()
365-
.expect("transaction will always be present here");
366-
for input in &tx.input {
367-
// Delete any spent utxos
368-
batch.del_utxo(&input.previous_output)?;
369-
}
377+
batch.set_tx(finished_tx)?;
370378
}
371379

372380
for (keychain, last_active_index) in self.last_active_index {

‎src/database/keyvalue.rs

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ macro_rules! impl_batch_operations {
4343
let value = json!({
4444
"t": utxo.txout,
4545
"i": utxo.keychain,
46+
"s": utxo.is_spent,
4647
});
4748
self.insert(key, serde_json::to_vec(&value)?)$($after_insert)*;
4849

@@ -125,8 +126,9 @@ macro_rules! impl_batch_operations {
125126
let mut val: serde_json::Value = serde_json::from_slice(&b)?;
126127
let txout = serde_json::from_value(val["t"].take())?;
127128
let keychain = serde_json::from_value(val["i"].take())?;
129+
let is_spent = val.get_mut("s").and_then(|s| s.take().as_bool()).unwrap_or(false);
128130

129-
Ok(Some(LocalUtxo { outpoint: outpoint.clone(), txout, keychain }))
131+
Ok(Some(LocalUtxo { outpoint: outpoint.clone(), txout, keychain, is_spent, }))
130132
}
131133
}
132134
}
@@ -246,11 +248,16 @@ impl Database for Tree {
246248
let mut val: serde_json::Value = serde_json::from_slice(&v)?;
247249
let txout = serde_json::from_value(val["t"].take())?;
248250
let keychain = serde_json::from_value(val["i"].take())?;
251+
let is_spent = val
252+
.get_mut("s")
253+
.and_then(|s| s.take().as_bool())
254+
.unwrap_or(false);
249255

250256
Ok(LocalUtxo {
251257
outpoint,
252258
txout,
253259
keychain,
260+
is_spent,
254261
})
255262
})
256263
.collect()
@@ -314,11 +321,16 @@ impl Database for Tree {
314321
let mut val: serde_json::Value = serde_json::from_slice(&b)?;
315322
let txout = serde_json::from_value(val["t"].take())?;
316323
let keychain = serde_json::from_value(val["i"].take())?;
324+
let is_spent = val
325+
.get_mut("s")
326+
.and_then(|s| s.take().as_bool())
327+
.unwrap_or(false);
317328

318329
Ok(LocalUtxo {
319330
outpoint: *outpoint,
320331
txout,
321332
keychain,
333+
is_spent,
322334
})
323335
})
324336
.transpose()

‎src/database/memory.rs

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -150,8 +150,10 @@ impl BatchOperations for MemoryDatabase {
150150

151151
fn set_utxo(&mut self, utxo: &LocalUtxo) -> Result<(), Error> {
152152
let key = MapKey::Utxo(Some(&utxo.outpoint)).as_map_key();
153-
self.map
154-
.insert(key, Box::new((utxo.txout.clone(), utxo.keychain)));
153+
self.map.insert(
154+
key,
155+
Box::new((utxo.txout.clone(), utxo.keychain, utxo.is_spent)),
156+
);
155157

156158
Ok(())
157159
}
@@ -228,11 +230,12 @@ impl BatchOperations for MemoryDatabase {
228230
match res {
229231
None => Ok(None),
230232
Some(b) => {
231-
let (txout, keychain) = b.downcast_ref().cloned().unwrap();
233+
let (txout, keychain, is_spent) = b.downcast_ref().cloned().unwrap();
232234
Ok(Some(LocalUtxo {
233235
outpoint: *outpoint,
234236
txout,
235237
keychain,
238+
is_spent,
236239
}))
237240
}
238241
}
@@ -326,11 +329,12 @@ impl Database for MemoryDatabase {
326329
.range::<Vec<u8>, _>((Included(&key), Excluded(&after(&key))))
327330
.map(|(k, v)| {
328331
let outpoint = deserialize(&k[1..]).unwrap();
329-
let (txout, keychain) = v.downcast_ref().cloned().unwrap();
332+
let (txout, keychain, is_spent) = v.downcast_ref().cloned().unwrap();
330333
Ok(LocalUtxo {
331334
outpoint,
332335
txout,
333336
keychain,
337+
is_spent,
334338
})
335339
})
336340
.collect()
@@ -389,11 +393,12 @@ impl Database for MemoryDatabase {
389393
fn get_utxo(&self, outpoint: &OutPoint) -> Result<Option<LocalUtxo>, Error> {
390394
let key = MapKey::Utxo(Some(outpoint)).as_map_key();
391395
Ok(self.map.get(&key).map(|b| {
392-
let (txout, keychain) = b.downcast_ref().cloned().unwrap();
396+
let (txout, keychain, is_spent) = b.downcast_ref().cloned().unwrap();
393397
LocalUtxo {
394398
outpoint: *outpoint,
395399
txout,
396400
keychain,
401+
is_spent,
397402
}
398403
}))
399404
}
@@ -526,6 +531,7 @@ macro_rules! populate_test_db {
526531
vout: vout as u32,
527532
},
528533
keychain: $crate::KeychainKind::External,
534+
is_spent: false,
529535
})
530536
.unwrap();
531537
}

‎src/database/mod.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -316,6 +316,7 @@ pub mod test {
316316
txout,
317317
outpoint,
318318
keychain: KeychainKind::External,
319+
is_spent: true,
319320
};
320321

321322
tree.set_utxo(&utxo).unwrap();

‎src/database/sqlite.rs

Lines changed: 25 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ static MIGRATIONS: &[&str] = &[
4040
"CREATE TABLE transaction_details (txid BLOB, timestamp INTEGER, received INTEGER, sent INTEGER, fee INTEGER, height INTEGER);",
4141
"INSERT INTO transaction_details SELECT txid, timestamp, received, sent, fee, height FROM transaction_details_old;",
4242
"DROP TABLE transaction_details_old;",
43+
"ALTER TABLE utxos ADD COLUMN is_spent;",
4344
];
4445

4546
/// Sqlite database stored on filesystem
@@ -83,14 +84,16 @@ impl SqliteDatabase {
8384
vout: u32,
8485
txid: &[u8],
8586
script: &[u8],
87+
is_spent: bool,
8688
) -> Result<i64, Error> {
87-
let mut statement = self.connection.prepare_cached("INSERT INTO utxos (value, keychain, vout, txid, script) VALUES (:value, :keychain, :vout, :txid, :script)")?;
89+
let mut statement = self.connection.prepare_cached("INSERT INTO utxos (value, keychain, vout, txid, script, is_spent) VALUES (:value, :keychain, :vout, :txid, :script, :is_spent)")?;
8890
statement.execute(named_params! {
8991
":value": value,
9092
":keychain": keychain,
9193
":vout": vout,
9294
":txid": txid,
93-
":script": script
95+
":script": script,
96+
":is_spent": is_spent,
9497
})?;
9598

9699
Ok(self.connection.last_insert_rowid())
@@ -291,7 +294,7 @@ impl SqliteDatabase {
291294
fn select_utxos(&self) -> Result<Vec<LocalUtxo>, Error> {
292295
let mut statement = self
293296
.connection
294-
.prepare_cached("SELECT value, keychain, vout, txid, script FROM utxos")?;
297+
.prepare_cached("SELECT value, keychain, vout, txid, script, is_spent FROM utxos")?;
295298
let mut utxos: Vec<LocalUtxo> = vec![];
296299
let mut rows = statement.query([])?;
297300
while let Some(row) = rows.next()? {
@@ -300,6 +303,7 @@ impl SqliteDatabase {
300303
let vout = row.get(2)?;
301304
let txid: Vec<u8> = row.get(3)?;
302305
let script: Vec<u8> = row.get(4)?;
306+
let is_spent: bool = row.get(5)?;
303307

304308
let keychain: KeychainKind = serde_json::from_str(&keychain)?;
305309

@@ -310,19 +314,16 @@ impl SqliteDatabase {
310314
script_pubkey: script.into(),
311315
},
312316
keychain,
317+
is_spent,
313318
})
314319
}
315320

316321
Ok(utxos)
317322
}
318323

319-
fn select_utxo_by_outpoint(
320-
&self,
321-
txid: &[u8],
322-
vout: u32,
323-
) -> Result<Option<(u64, KeychainKind, Script)>, Error> {
324+
fn select_utxo_by_outpoint(&self, txid: &[u8], vout: u32) -> Result<Option<LocalUtxo>, Error> {
324325
let mut statement = self.connection.prepare_cached(
325-
"SELECT value, keychain, script FROM utxos WHERE txid=:txid AND vout=:vout",
326+
"SELECT value, keychain, script, is_spent FROM utxos WHERE txid=:txid AND vout=:vout",
326327
)?;
327328
let mut rows = statement.query(named_params! {":txid": txid,":vout": vout})?;
328329
match rows.next()? {
@@ -331,9 +332,18 @@ impl SqliteDatabase {
331332
let keychain: String = row.get(1)?;
332333
let keychain: KeychainKind = serde_json::from_str(&keychain)?;
333334
let script: Vec<u8> = row.get(2)?;
334-
let script: Script = script.into();
335+
let script_pubkey: Script = script.into();
336+
let is_spent: bool = row.get(3)?;
335337

336-
Ok(Some((value, keychain, script)))
338+
Ok(Some(LocalUtxo {
339+
outpoint: OutPoint::new(deserialize(txid)?, vout),
340+
txout: TxOut {
341+
value,
342+
script_pubkey,
343+
},
344+
keychain,
345+
is_spent,
346+
}))
337347
}
338348
None => Ok(None),
339349
}
@@ -620,6 +630,7 @@ impl BatchOperations for SqliteDatabase {
620630
utxo.outpoint.vout,
621631
&utxo.outpoint.txid,
622632
utxo.txout.script_pubkey.as_bytes(),
633+
utxo.is_spent,
623634
)?;
624635
Ok(())
625636
}
@@ -694,16 +705,9 @@ impl BatchOperations for SqliteDatabase {
694705

695706
fn del_utxo(&mut self, outpoint: &OutPoint) -> Result<Option<LocalUtxo>, Error> {
696707
match self.select_utxo_by_outpoint(&outpoint.txid, outpoint.vout)? {
697-
Some((value, keychain, script_pubkey)) => {
708+
Some(local_utxo) => {
698709
self.delete_utxo_by_outpoint(&outpoint.txid, outpoint.vout)?;
699-
Ok(Some(LocalUtxo {
700-
outpoint: *outpoint,
701-
txout: TxOut {
702-
value,
703-
script_pubkey,
704-
},
705-
keychain,
706-
}))
710+
Ok(Some(local_utxo))
707711
}
708712
None => Ok(None),
709713
}
@@ -832,17 +836,7 @@ impl Database for SqliteDatabase {
832836
}
833837

834838
fn get_utxo(&self, outpoint: &OutPoint) -> Result<Option<LocalUtxo>, Error> {
835-
match self.select_utxo_by_outpoint(&outpoint.txid, outpoint.vout)? {
836-
Some((value, keychain, script_pubkey)) => Ok(Some(LocalUtxo {
837-
outpoint: *outpoint,
838-
txout: TxOut {
839-
value,
840-
script_pubkey,
841-
},
842-
keychain,
843-
})),
844-
None => Ok(None),
845-
}
839+
self.select_utxo_by_outpoint(&outpoint.txid, outpoint.vout)
846840
}
847841

848842
fn get_raw_tx(&self, txid: &Txid) -> Result<Option<Transaction>, Error> {

0 commit comments

Comments
 (0)