assumeutxo draft

doc/assumeutxo.md

@@ -3,7 +3,7 @@
 Assumeutxo is a feature that allows fast bootstrapping of a validating bitcoind
 instance with a very similar security model to assumevalid.
 
-The RPC commands `dumptxoutset` and `loadtxoutset` (yet to be merged) are used to
+The RPC commands `dumptxoutset` and `loadtxoutset` are used to
 respectively generate and load UTXO snapshots. The utility script
 `./contrib/devtools/utxo_snapshot.sh` may be of use.
 

doc/release-notes-27596.md

@@ -0,0 +1,28 @@
+Pruning
+-------
+
+When using assumeutxo with `-prune`, the prune budget may be exceeded if it is set
+lower than 1100MB (i.e. `MIN_DISK_SPACE_FOR_BLOCK_FILES * 2`). Prune budget is normally
+split evenly across each chainstate, unless the resulting prune budget per chainstate
+is beneath `MIN_DISK_SPACE_FOR_BLOCK_FILES` in which case that value will be used.
+
+RPC
+---
+
+`loadtxoutset` has been added, which allows loading a UTXO snapshot of the format
+generated by `dumptxoutset`. Once this snapshot is loaded, its contents will be
+deserialized into a second chainstate data structure, which is then used to sync to
+the network's tip under a security model very much like `assumevalid`.
+
+Meanwhile, the original chainstate will complete the initial block download process in
+the background, eventually validating up to the block that the snapshot is based upon.
+
+The result is a usable bitcoind instance that is current with the network tip in a
+matter of minutes rather than hours. UTXO snapshot are typically obtained via
+third-party sources (HTTP, torrent, etc.) which is reasonable since their contents
+are always checked by hash.
+
+You can find more information on this process in the `assumeutxo` design
+document (<https://github.com/bitcoin/bitcoin/blob/master/doc/design/assumeutxo.md>).
+
+`getchainstates` has been added to aid in monitoring the assumeutxo sync process.

src/interfaces/chain.h

@@ -228,6 +228,9 @@ class Chain {
     //! Check if any block has been pruned.
     virtual bool havePruned() = 0;
 
+    //! Get the current prune height.
+    virtual std::optional<int> getPruneHeight() = 0;
+
     //! Check if the node is ready to broadcast transactions.
     virtual bool isReadyToBroadcast() = 0;
 

src/kernel/chainparams.cpp

@@ -213,7 +213,12 @@ class CMainParams : public CChainParams {
         checkpointData = CheckpointData(CBaseChainParams::MAIN);
 
         m_assumeutxo_data = {
-            // TODO to be specified in a future patch.
+            {.height = 888'000,
+             .hash_serialized =
+                 AssumeutxoHash{uint256S("0x50493f6218661a189654dbad816821a656b519454190c63daf376610e4fa0a7e")},
+             .nChainTx = 299'158'458,
+             .blockhash =
+                 BlockHash{uint256S("0x00000000000000002b218d995a292c34bc4c0244bb4bbdad18f3a97e88ccb567")}},
         };
 
         // Data as of block
@@ -485,6 +490,15 @@ class CRegTestParams : public CChainParams {
              .blockhash =
                  BlockHash{uint256S("0x47cfb2b77860d250060e78d3248bb05092876545"
                                     "3cbcbdbc121e3c48b99a376c")}},
+            {// For use by test/functional/feature_assumeutxo.py
+             .height = 299,
+             .hash_serialized =
+                 AssumeutxoHash{uint256S("0xa966794ed5a2f9debaefc7ca48dbc5d5e12"
+                                         "a89ff9fe45bd00ec5732d074580a9")},
+             .nChainTx = 334,
+             .blockhash =
+                 BlockHash{uint256S("0x118a7d5473bccce9b314789e14ce426fc65fb09d"
+                                    "feda0131032bb6d86ed2fd0b")}},
         };
 
         chainTxData = ChainTxData{0, 0, 0};

src/node/interfaces.cpp

@@ -27,6 +27,7 @@
 #include <policy/settings.h>
 #include <primitives/block.h>
 #include <primitives/transaction.h>
+#include <rpc/blockchain.h>
 #include <rpc/protocol.h>
 #include <rpc/server.h>
 #include <shutdown.h>
@@ -716,6 +717,10 @@ namespace {
             return !chainman().m_blockman.LoadingBlocks() &&
                    !isInitialBlockDownload();
         }
+        std::optional<int> getPruneHeight() override {
+            LOCK(chainman().GetMutex());
+            return GetPruneHeight(chainman().m_blockman, chainman().ActiveChain());
+        }
         bool isInitialBlockDownload() override {
             return chainman().IsInitialBlockDownload();
         }

src/rpc/blockchain.cpp

@@ -8,6 +8,7 @@
 #include <blockfilter.h>
 #include <chain.h>
 #include <chainparams.h>
+#include <clientversion.h>
 #include <coins.h>
 #include <common/args.h>
 #include <config.h>
@@ -2623,7 +2624,7 @@ static RPCHelpMan getblockfilter() {
 static RPCHelpMan dumptxoutset() {
     return RPCHelpMan{
         "dumptxoutset",
-        "Write the serialized UTXO set to disk.\n",
+        "Write the serialized UTXO set to a file.\n",
         {
             {"path", RPCArg::Type::STR, RPCArg::Optional::NO,
              "path to the output file. If relative, will be prefixed by "
@@ -2755,6 +2756,194 @@ UniValue CreateUTXOSnapshot(NodeContext &node, Chainstate &chainstate,
     return result;
 }
 
+static RPCHelpMan loadtxoutset() {
+    return RPCHelpMan{
+        "loadtxoutset",
+        "Load the serialized UTXO set from a file.\n"
+        "Once this snapshot is loaded, its contents will be deserialized into "
+        "a second chainstate data structure, which is then used to sync to the "
+        "network's tip. "
+        "Meanwhile, the original chainstate will complete the initial block "
+        "download process in the background, eventually validating up to the "
+        "block that the snapshot is based upon.\n\n"
+        "The result is a usable bitcoind instance that is current with the "
+        "network tip in a matter of minutes rather than hours. UTXO snapshot "
+        "are typically obtained from third-party sources (HTTP, torrent, etc.) "
+        "which is reasonable since their contents are always checked by "
+        "hash.\n\n"
+        "You can find more information on this process in the `assumeutxo` "
+        "design document "
+        "(<https://github.com/bitcoin/bitcoin/blob/master/doc/design/"
+        "assumeutxo.md>).",
+        {
+            {"path", RPCArg::Type::STR, RPCArg::Optional::NO,
+             "path to the snapshot file. If relative, will be prefixed by "
+             "datadir."},
+        },
+        RPCResult{RPCResult::Type::OBJ,
+                  "",
+                  "",
+                  {
+                      {RPCResult::Type::NUM, "coins_loaded",
+                       "the number of coins loaded from the snapshot"},
+                      {RPCResult::Type::STR_HEX, "tip_hash",
+                       "the hash of the base of the snapshot"},
+                      {RPCResult::Type::NUM, "base_height",
+                       "the height of the base of the snapshot"},
+                      {RPCResult::Type::STR, "path",
+                       "the absolute path that the snapshot was loaded from"},
+                  }},
+        RPCExamples{HelpExampleCli("loadtxoutset", "utxo.dat")},
+        [&](const RPCHelpMan &self, const Config &config,
+            const JSONRPCRequest &request) -> UniValue {
+            NodeContext &node = EnsureAnyNodeContext(request.context);
+            ChainstateManager &chainman = EnsureChainman(node);
+            fs::path path{AbsPathForConfigVal(
+                EnsureArgsman(node), fs::u8path(request.params[0].get_str()))};
+
+            FILE *file{fsbridge::fopen(path, "rb")};
+            AutoFile afile{file};
+            if (afile.IsNull()) {
+                throw JSONRPCError(RPC_INVALID_PARAMETER,
+                                   "Couldn't open file " + path.u8string() +
+                                       " for reading.");
+            }
+
+            SnapshotMetadata metadata;
+            afile >> metadata;
+
+            BlockHash base_blockhash = metadata.m_base_blockhash;
+            if (!chainman.GetParams()
+                     .AssumeutxoForBlockhash(base_blockhash)
+                     .has_value()) {
+                throw JSONRPCError(
+                    RPC_INTERNAL_ERROR,
+                    strprintf("Unable to load UTXO snapshot, "
+                              "assumeutxo block hash in snapshot metadata not "
+                              "recognized (%s)",
+                              base_blockhash.ToString()));
+            }
+            CBlockIndex *snapshot_start_block = WITH_LOCK(
+                ::cs_main,
+                return chainman.m_blockman.LookupBlockIndex(base_blockhash));
+
+            if (!snapshot_start_block) {
+                throw JSONRPCError(
+                    RPC_INTERNAL_ERROR,
+                    strprintf("The base block header (%s) must appear in the "
+                              "headers chain. Make sure all headers are "
+                              "syncing, and call this RPC again.",
+                              base_blockhash.ToString()));
+            }
+            if (!chainman.ActivateSnapshot(afile, metadata, false)) {
+                throw JSONRPCError(RPC_INTERNAL_ERROR,
+                                   "Unable to load UTXO snapshot " +
+                                       fs::PathToString(path));
+            }
+            CBlockIndex *new_tip{
+                WITH_LOCK(::cs_main, return chainman.ActiveTip())};
+
+            UniValue result(UniValue::VOBJ);
+            result.pushKV("coins_loaded", metadata.m_coins_count);
+            result.pushKV("tip_hash", new_tip->GetBlockHash().ToString());
+            result.pushKV("base_height", new_tip->nHeight);
+            result.pushKV("path", fs::PathToString(path));
+            return result;
+        },
+    };
+}
+
+const std::vector<RPCResult> RPCHelpForChainstate{
+    {RPCResult::Type::NUM, "blocks", "number of blocks in this chainstate"},
+    {RPCResult::Type::STR_HEX, "bestblockhash", "blockhash of the tip"},
+    {RPCResult::Type::NUM, "difficulty", "difficulty of the tip"},
+    {RPCResult::Type::NUM, "verificationprogress",
+     "progress towards the network tip"},
+    {RPCResult::Type::STR_HEX, "snapshot_blockhash", /*optional=*/true,
+     "the base block of the snapshot this chainstate is based on, if any"},
+    {RPCResult::Type::NUM, "coins_db_cache_bytes", "size of the coinsdb cache"},
+    {RPCResult::Type::NUM, "coins_tip_cache_bytes",
+     "size of the coinstip cache"},
+    {RPCResult::Type::BOOL, "validated",
+     "whether the chainstate is fully validated. True if all blocks in the "
+     "chainstate were validated, false if the chain is based on a snapshot and "
+     "the snapshot has not yet been validated."},
+
+};
+
+static RPCHelpMan getchainstates() {
+    return RPCHelpMan{
+        "getchainstates",
+        "\nReturn information about chainstates.\n",
+        {},
+        RPCResult{RPCResult::Type::OBJ,
+                  "",
+                  "",
+                  {
+                      {RPCResult::Type::NUM, "headers",
+                       "the number of headers seen so far"},
+                      {RPCResult::Type::ARR,
+                       "chainstates",
+                       "list of the chainstates ordered by work, with the "
+                       "most-work (active) chainstate last",
+                       {
+                           {RPCResult::Type::OBJ, "", "", RPCHelpForChainstate},
+                       }},
+                  }},
+        RPCExamples{HelpExampleCli("getchainstates", "") +
+                    HelpExampleRpc("getchainstates", "")},
+        [&](const RPCHelpMan &self, const Config &config,
+            const JSONRPCRequest &request) -> UniValue {
+            LOCK(cs_main);
+            UniValue obj(UniValue::VOBJ);
+
+            ChainstateManager &chainman = EnsureAnyChainman(request.context);
+
+            auto make_chain_data =
+                [&](const Chainstate &cs,
+                    bool validated) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) {
+                    AssertLockHeld(::cs_main);
+                    UniValue data(UniValue::VOBJ);
+                    if (!cs.m_chain.Tip()) {
+                        return data;
+                    }
+                    const CChain &chain = cs.m_chain;
+                    const CBlockIndex *tip = chain.Tip();
+
+                    data.pushKV("blocks", chain.Height());
+                    data.pushKV("bestblockhash", tip->GetBlockHash().GetHex());
+                    data.pushKV("difficulty", GetDifficulty(tip));
+                    data.pushKV(
+                        "verificationprogress",
+                        GuessVerificationProgress(Params().TxData(), tip));
+                    data.pushKV("coins_db_cache_bytes",
+                                cs.m_coinsdb_cache_size_bytes);
+                    data.pushKV("coins_tip_cache_bytes",
+                                cs.m_coinstip_cache_size_bytes);
+                    if (cs.m_from_snapshot_blockhash) {
+                        data.pushKV("snapshot_blockhash",
+                                    cs.m_from_snapshot_blockhash->ToString());
+                    }
+                    data.pushKV("validated", validated);
+                    return data;
+                };
+
+            obj.pushKV("headers", chainman.m_best_header
+                                      ? chainman.m_best_header->nHeight
+                                      : -1);
+
+            const auto &chainstates = chainman.GetAll();
+            UniValue obj_chainstates{UniValue::VARR};
+            for (Chainstate *cs : chainstates) {
+                obj_chainstates.push_back(
+                    make_chain_data(*cs, !cs->m_from_snapshot_blockhash ||
+                                             chainstates.size() == 1));
+            }
+            obj.pushKV("chainstates", std::move(obj_chainstates));
+            return obj;
+        }};
+}
+
 void RegisterBlockchainRPCCommands(CRPCTable &t) {
     // clang-format off
     static const CRPCCommand commands[] = {
@@ -2778,13 +2967,15 @@ void RegisterBlockchainRPCCommands(CRPCTable &t) {
         { "blockchain",         preciousblock,                     },
         { "blockchain",         scantxoutset,                      },
         { "blockchain",         getblockfilter,                    },
+        { "blockchain",         dumptxoutset,                      },
+        { "blockchain",         loadtxoutset,                      },
+        { "blockchain",         getchainstates,                    },
 
         /* Not shown in help */
         { "hidden",             invalidateblock,                   },
         { "hidden",             parkblock,                         },
         { "hidden",             reconsiderblock,                   },
         { "hidden",             syncwithvalidationinterfacequeue,  },
-        { "hidden",             dumptxoutset,                      },
         { "hidden",             unparkblock,                       },
         { "hidden",             waitfornewblock,                   },
         { "hidden",             waitforblock,                      },

src/validation.cpp

@@ -6702,6 +6702,12 @@ bool ChainstateManager::PopulateAndValidateSnapshot(
                 coins_count - coins_left);
             return false;
         }
+        if (!MoneyRange(coin.GetTxOut().nValue)) {
+            LogPrintf("[snapshot] bad snapshot data after deserializing %d "
+                      "coins - bad tx out value\n",
+                      coins_count - coins_left);
+            return false;
+        }
         coins_cache.EmplaceCoinInternalDANGER(std::move(outpoint),
                                               std::move(coin));
 

src/wallet/rpcwallet.cpp

@@ -3767,14 +3767,29 @@ RPCHelpMan rescanblockchain() {
                     }
                 }
 
-                // We can't rescan beyond non-pruned blocks, stop and throw an
-                // error
+                // We can't rescan unavailable blocks, stop and throw an error
                 if (!pwallet->chain().hasBlocks(pwallet->GetLastBlockHash(),
                                                 start_height, stop_height)) {
+                    if (pwallet->chain().havePruned() &&
+                        pwallet->chain().getPruneHeight() >= start_height) {
+                        throw JSONRPCError(RPC_MISC_ERROR,
+                                           "Can't rescan beyond pruned data. "
+                                           "Use RPC call getblockchaininfo to "
+                                           "determine your pruned height.");
+                    }
+                    if (pwallet->chain().hasAssumedValidChain()) {
+                        throw JSONRPCError(
+                            RPC_MISC_ERROR,
+                            "Failed to rescan unavailable blocks likely due to "
+                            "an in-progress assumeutxo background sync. Check "
+                            "logs or getchainstates RPC for assumeutxo "
+                            "background sync progress and try again later.");
+                    }
                     throw JSONRPCError(
                         RPC_MISC_ERROR,
-                        "Can't rescan beyond pruned data. Use RPC call "
-                        "getblockchaininfo to determine your pruned height.");
+                        "Failed to rescan unavailable blocks, potentially "
+                        "caused by data corruption. If the issue persists you "
+                        "may want to reindex (see -reindex option).");
                 }
 
                 CHECK_NONFATAL(pwallet->chain().findAncestorByHeight(