coinstatsindex.cpp
1 // Copyright (c) 2020-2022 The Bitcoin Core developers 2 // Distributed under the MIT software license, see the accompanying 3 // file COPYING or http://www.opensource.org/licenses/mit-license.php. 4 5 #include <index/coinstatsindex.h> 6 7 #include <arith_uint256.h> 8 #include <chain.h> 9 #include <chainparams.h> 10 #include <coins.h> 11 #include <common/args.h> 12 #include <consensus/amount.h> 13 #include <crypto/muhash.h> 14 #include <dbwrapper.h> 15 #include <index/base.h> 16 #include <interfaces/chain.h> 17 #include <interfaces/types.h> 18 #include <kernel/coinstats.h> 19 #include <logging.h> 20 #include <primitives/block.h> 21 #include <primitives/transaction.h> 22 #include <script/script.h> 23 #include <serialize.h> 24 #include <uint256.h> 25 #include <undo.h> 26 #include <util/check.h> 27 #include <util/fs.h> 28 #include <validation.h> 29 30 #include <compare> 31 #include <ios> 32 #include <limits> 33 #include <span> 34 #include <string> 35 #include <utility> 36 #include <vector> 37 38 using kernel::ApplyCoinHash; 39 using kernel::CCoinsStats; 40 using kernel::GetBogoSize; 41 using kernel::RemoveCoinHash; 42 43 static constexpr uint8_t DB_BLOCK_HASH{'s'}; 44 static constexpr uint8_t DB_BLOCK_HEIGHT{'t'}; 45 static constexpr uint8_t DB_MUHASH{'M'}; 46 47 namespace { 48 49 struct DBVal { 50 uint256 muhash{uint256::ZERO}; 51 uint64_t transaction_output_count{0}; 52 uint64_t bogo_size{0}; 53 CAmount total_amount{0}; 54 CAmount total_subsidy{0}; 55 arith_uint256 total_prevout_spent_amount{0}; 56 arith_uint256 total_new_outputs_ex_coinbase_amount{0}; 57 arith_uint256 total_coinbase_amount{0}; 58 CAmount total_unspendables_genesis_block{0}; 59 CAmount total_unspendables_bip30{0}; 60 CAmount total_unspendables_scripts{0}; 61 CAmount total_unspendables_unclaimed_rewards{0}; 62 63 SERIALIZE_METHODS(DBVal, obj) 64 { 65 uint256 prevout_spent, new_outputs, coinbase; 66 SER_WRITE(obj, prevout_spent = ArithToUint256(obj.total_prevout_spent_amount)); 67 SER_WRITE(obj, new_outputs = ArithToUint256(obj.total_new_outputs_ex_coinbase_amount)); 68 SER_WRITE(obj, coinbase = ArithToUint256(obj.total_coinbase_amount)); 69 70 READWRITE(obj.muhash); 71 READWRITE(obj.transaction_output_count); 72 READWRITE(obj.bogo_size); 73 READWRITE(obj.total_amount); 74 READWRITE(obj.total_subsidy); 75 READWRITE(prevout_spent); 76 READWRITE(new_outputs); 77 READWRITE(coinbase); 78 READWRITE(obj.total_unspendables_genesis_block); 79 READWRITE(obj.total_unspendables_bip30); 80 READWRITE(obj.total_unspendables_scripts); 81 READWRITE(obj.total_unspendables_unclaimed_rewards); 82 83 SER_READ(obj, obj.total_prevout_spent_amount = UintToArith256(prevout_spent)); 84 SER_READ(obj, obj.total_new_outputs_ex_coinbase_amount = UintToArith256(new_outputs)); 85 SER_READ(obj, obj.total_coinbase_amount = UintToArith256(coinbase)); 86 } 87 }; 88 89 struct DBHeightKey { 90 int height; 91 92 explicit DBHeightKey(int height_in) : height(height_in) {} 93 94 template <typename Stream> 95 void Serialize(Stream& s) const 96 { 97 ser_writedata8(s, DB_BLOCK_HEIGHT); 98 ser_writedata32be(s, height); 99 } 100 101 template <typename Stream> 102 void Unserialize(Stream& s) 103 { 104 const uint8_t prefix{ser_readdata8(s)}; 105 if (prefix != DB_BLOCK_HEIGHT) { 106 throw std::ios_base::failure("Invalid format for coinstatsindex DB height key"); 107 } 108 height = ser_readdata32be(s); 109 } 110 }; 111 112 struct DBHashKey { 113 uint256 block_hash; 114 115 explicit DBHashKey(const uint256& hash_in) : block_hash(hash_in) {} 116 117 SERIALIZE_METHODS(DBHashKey, obj) 118 { 119 uint8_t prefix{DB_BLOCK_HASH}; 120 READWRITE(prefix); 121 if (prefix != DB_BLOCK_HASH) { 122 throw std::ios_base::failure("Invalid format for coinstatsindex DB hash key"); 123 } 124 125 READWRITE(obj.block_hash); 126 } 127 }; 128 129 }; // namespace 130 131 std::unique_ptr<CoinStatsIndex> g_coin_stats_index; 132 133 CoinStatsIndex::CoinStatsIndex(std::unique_ptr<interfaces::Chain> chain, size_t n_cache_size, bool f_memory, bool f_wipe) 134 : BaseIndex(std::move(chain), "coinstatsindex") 135 { 136 // An earlier version of the index used "indexes/coinstats" but it contained 137 // a bug and is superseded by a fixed version at "indexes/coinstatsindex". 138 // The original index is kept around until the next release in case users 139 // decide to downgrade their node. 140 auto old_path = gArgs.GetDataDirNet() / "indexes" / "coinstats"; 141 if (fs::exists(old_path)) { 142 // TODO: Change this to deleting the old index with v31. 143 LogWarning("Old version of coinstatsindex found at %s. This folder can be safely deleted unless you " \ 144 "plan to downgrade your node to version 29 or lower.", fs::PathToString(old_path)); 145 } 146 fs::path path{gArgs.GetDataDirNet() / "indexes" / "coinstatsindex"}; 147 fs::create_directories(path); 148 149 m_db = std::make_unique<CoinStatsIndex::DB>(path / "db", n_cache_size, f_memory, f_wipe); 150 } 151 152 bool CoinStatsIndex::CustomAppend(const interfaces::BlockInfo& block) 153 { 154 const CAmount block_subsidy{GetBlockSubsidy(block.height, Params().GetConsensus())}; 155 m_total_subsidy += block_subsidy; 156 157 // Ignore genesis block 158 if (block.height > 0) { 159 uint256 expected_block_hash{*Assert(block.prev_hash)}; 160 if (m_current_block_hash != expected_block_hash) { 161 LogError("previous block header belongs to unexpected block %s; expected %s", 162 m_current_block_hash.ToString(), expected_block_hash.ToString()); 163 return false; 164 } 165 166 // Add the new utxos created from the block 167 assert(block.data); 168 for (size_t i = 0; i < block.data->vtx.size(); ++i) { 169 const auto& tx{block.data->vtx.at(i)}; 170 const bool is_coinbase{tx->IsCoinBase()}; 171 172 // Skip duplicate txid coinbase transactions (BIP30). 173 if (is_coinbase && IsBIP30Unspendable(block.hash, block.height)) { 174 m_total_unspendables_bip30 += block_subsidy; 175 continue; 176 } 177 178 for (uint32_t j = 0; j < tx->vout.size(); ++j) { 179 const CTxOut& out{tx->vout[j]}; 180 const Coin coin{out, block.height, is_coinbase}; 181 const COutPoint outpoint{tx->GetHash(), j}; 182 183 // Skip unspendable coins 184 if (coin.out.scriptPubKey.IsUnspendable()) { 185 m_total_unspendables_scripts += coin.out.nValue; 186 continue; 187 } 188 189 ApplyCoinHash(m_muhash, outpoint, coin); 190 191 if (is_coinbase) { 192 m_total_coinbase_amount += coin.out.nValue; 193 } else { 194 m_total_new_outputs_ex_coinbase_amount += coin.out.nValue; 195 } 196 197 ++m_transaction_output_count; 198 m_total_amount += coin.out.nValue; 199 m_bogo_size += GetBogoSize(coin.out.scriptPubKey); 200 } 201 202 // The coinbase tx has no undo data since no former output is spent 203 if (!is_coinbase) { 204 const auto& tx_undo{Assert(block.undo_data)->vtxundo.at(i - 1)}; 205 206 for (size_t j = 0; j < tx_undo.vprevout.size(); ++j) { 207 const Coin& coin{tx_undo.vprevout[j]}; 208 const COutPoint outpoint{tx->vin[j].prevout.hash, tx->vin[j].prevout.n}; 209 210 RemoveCoinHash(m_muhash, outpoint, coin); 211 212 m_total_prevout_spent_amount += coin.out.nValue; 213 214 --m_transaction_output_count; 215 m_total_amount -= coin.out.nValue; 216 m_bogo_size -= GetBogoSize(coin.out.scriptPubKey); 217 } 218 } 219 } 220 } else { 221 // genesis block 222 m_total_unspendables_genesis_block += block_subsidy; 223 } 224 225 // If spent prevouts + block subsidy are still a higher amount than 226 // new outputs + coinbase + current unspendable amount this means 227 // the miner did not claim the full block reward. Unclaimed block 228 // rewards are also unspendable. 229 const CAmount temp_total_unspendable_amount{m_total_unspendables_genesis_block + m_total_unspendables_bip30 + m_total_unspendables_scripts + m_total_unspendables_unclaimed_rewards}; 230 const arith_uint256 unclaimed_rewards{(m_total_prevout_spent_amount + m_total_subsidy) - (m_total_new_outputs_ex_coinbase_amount + m_total_coinbase_amount + temp_total_unspendable_amount)}; 231 assert(unclaimed_rewards <= arith_uint256(std::numeric_limits<CAmount>::max())); 232 m_total_unspendables_unclaimed_rewards += static_cast<CAmount>(unclaimed_rewards.GetLow64()); 233 234 std::pair<uint256, DBVal> value; 235 value.first = block.hash; 236 value.second.transaction_output_count = m_transaction_output_count; 237 value.second.bogo_size = m_bogo_size; 238 value.second.total_amount = m_total_amount; 239 value.second.total_subsidy = m_total_subsidy; 240 value.second.total_prevout_spent_amount = m_total_prevout_spent_amount; 241 value.second.total_new_outputs_ex_coinbase_amount = m_total_new_outputs_ex_coinbase_amount; 242 value.second.total_coinbase_amount = m_total_coinbase_amount; 243 value.second.total_unspendables_genesis_block = m_total_unspendables_genesis_block; 244 value.second.total_unspendables_bip30 = m_total_unspendables_bip30; 245 value.second.total_unspendables_scripts = m_total_unspendables_scripts; 246 value.second.total_unspendables_unclaimed_rewards = m_total_unspendables_unclaimed_rewards; 247 248 uint256 out; 249 m_muhash.Finalize(out); 250 value.second.muhash = out; 251 252 m_current_block_hash = block.hash; 253 254 // Intentionally do not update DB_MUHASH here so it stays in sync with 255 // DB_BEST_BLOCK, and the index is not corrupted if there is an unclean shutdown. 256 m_db->Write(DBHeightKey(block.height), value); 257 return true; 258 } 259 260 [[nodiscard]] static bool CopyHeightIndexToHashIndex(CDBIterator& db_it, CDBBatch& batch, 261 const std::string& index_name, int height) 262 { 263 DBHeightKey key{height}; 264 db_it.Seek(key); 265 266 if (!db_it.GetKey(key) || key.height != height) { 267 LogError("unexpected key in %s: expected (%c, %d)", 268 index_name, DB_BLOCK_HEIGHT, height); 269 return false; 270 } 271 272 std::pair<uint256, DBVal> value; 273 if (!db_it.GetValue(value)) { 274 LogError("unable to read value in %s at key (%c, %d)", 275 index_name, DB_BLOCK_HEIGHT, height); 276 return false; 277 } 278 279 batch.Write(DBHashKey(value.first), value.second); 280 return true; 281 } 282 283 bool CoinStatsIndex::CustomRemove(const interfaces::BlockInfo& block) 284 { 285 CDBBatch batch(*m_db); 286 std::unique_ptr<CDBIterator> db_it(m_db->NewIterator()); 287 288 // During a reorg, copy the block's hash digest from the height index to the hash index, 289 // ensuring it's still accessible after the height index entry is overwritten. 290 if (!CopyHeightIndexToHashIndex(*db_it, batch, m_name, block.height)) { 291 return false; 292 } 293 294 m_db->WriteBatch(batch); 295 296 if (!RevertBlock(block)) { 297 return false; // failure cause logged internally 298 } 299 300 return true; 301 } 302 303 static bool LookUpOne(const CDBWrapper& db, const interfaces::BlockRef& block, DBVal& result) 304 { 305 // First check if the result is stored under the height index and the value 306 // there matches the block hash. This should be the case if the block is on 307 // the active chain. 308 std::pair<uint256, DBVal> read_out; 309 if (!db.Read(DBHeightKey(block.height), read_out)) { 310 return false; 311 } 312 if (read_out.first == block.hash) { 313 result = std::move(read_out.second); 314 return true; 315 } 316 317 // If value at the height index corresponds to an different block, the 318 // result will be stored in the hash index. 319 return db.Read(DBHashKey(block.hash), result); 320 } 321 322 std::optional<CCoinsStats> CoinStatsIndex::LookUpStats(const CBlockIndex& block_index) const 323 { 324 CCoinsStats stats{block_index.nHeight, block_index.GetBlockHash()}; 325 stats.index_used = true; 326 327 DBVal entry; 328 if (!LookUpOne(*m_db, {block_index.GetBlockHash(), block_index.nHeight}, entry)) { 329 return std::nullopt; 330 } 331 332 stats.hashSerialized = entry.muhash; 333 stats.nTransactionOutputs = entry.transaction_output_count; 334 stats.nBogoSize = entry.bogo_size; 335 stats.total_amount = entry.total_amount; 336 stats.total_subsidy = entry.total_subsidy; 337 stats.total_prevout_spent_amount = entry.total_prevout_spent_amount; 338 stats.total_new_outputs_ex_coinbase_amount = entry.total_new_outputs_ex_coinbase_amount; 339 stats.total_coinbase_amount = entry.total_coinbase_amount; 340 stats.total_unspendables_genesis_block = entry.total_unspendables_genesis_block; 341 stats.total_unspendables_bip30 = entry.total_unspendables_bip30; 342 stats.total_unspendables_scripts = entry.total_unspendables_scripts; 343 stats.total_unspendables_unclaimed_rewards = entry.total_unspendables_unclaimed_rewards; 344 345 return stats; 346 } 347 348 bool CoinStatsIndex::CustomInit(const std::optional<interfaces::BlockRef>& block) 349 { 350 if (!m_db->Read(DB_MUHASH, m_muhash)) { 351 // Check that the cause of the read failure is that the key does not 352 // exist. Any other errors indicate database corruption or a disk 353 // failure, and starting the index would cause further corruption. 354 if (m_db->Exists(DB_MUHASH)) { 355 LogError("Cannot read current %s state; index may be corrupted", 356 GetName()); 357 return false; 358 } 359 } 360 361 if (block) { 362 DBVal entry; 363 if (!LookUpOne(*m_db, *block, entry)) { 364 LogError("Cannot read current %s state; index may be corrupted", 365 GetName()); 366 return false; 367 } 368 369 uint256 out; 370 m_muhash.Finalize(out); 371 if (entry.muhash != out) { 372 LogError("Cannot read current %s state; index may be corrupted", 373 GetName()); 374 return false; 375 } 376 377 m_transaction_output_count = entry.transaction_output_count; 378 m_bogo_size = entry.bogo_size; 379 m_total_amount = entry.total_amount; 380 m_total_subsidy = entry.total_subsidy; 381 m_total_prevout_spent_amount = entry.total_prevout_spent_amount; 382 m_total_new_outputs_ex_coinbase_amount = entry.total_new_outputs_ex_coinbase_amount; 383 m_total_coinbase_amount = entry.total_coinbase_amount; 384 m_total_unspendables_genesis_block = entry.total_unspendables_genesis_block; 385 m_total_unspendables_bip30 = entry.total_unspendables_bip30; 386 m_total_unspendables_scripts = entry.total_unspendables_scripts; 387 m_total_unspendables_unclaimed_rewards = entry.total_unspendables_unclaimed_rewards; 388 m_current_block_hash = block->hash; 389 } 390 391 return true; 392 } 393 394 bool CoinStatsIndex::CustomCommit(CDBBatch& batch) 395 { 396 // DB_MUHASH should always be committed in a batch together with DB_BEST_BLOCK 397 // to prevent an inconsistent state of the DB. 398 batch.Write(DB_MUHASH, m_muhash); 399 return true; 400 } 401 402 interfaces::Chain::NotifyOptions CoinStatsIndex::CustomOptions() 403 { 404 interfaces::Chain::NotifyOptions options; 405 options.connect_undo_data = true; 406 options.disconnect_data = true; 407 options.disconnect_undo_data = true; 408 return options; 409 } 410 411 // Revert a single block as part of a reorg 412 bool CoinStatsIndex::RevertBlock(const interfaces::BlockInfo& block) 413 { 414 std::pair<uint256, DBVal> read_out; 415 416 // Ignore genesis block 417 if (block.height > 0) { 418 if (!m_db->Read(DBHeightKey(block.height - 1), read_out)) { 419 return false; 420 } 421 422 uint256 expected_block_hash{*block.prev_hash}; 423 if (read_out.first != expected_block_hash) { 424 LogWarning("previous block header belongs to unexpected block %s; expected %s", 425 read_out.first.ToString(), expected_block_hash.ToString()); 426 427 if (!m_db->Read(DBHashKey(expected_block_hash), read_out)) { 428 LogError("previous block header not found; expected %s", 429 expected_block_hash.ToString()); 430 return false; 431 } 432 } 433 } 434 435 // Roll back muhash by removing the new UTXOs that were created by the 436 // block and reapplying the old UTXOs that were spent by the block 437 assert(block.data); 438 assert(block.undo_data); 439 for (size_t i = 0; i < block.data->vtx.size(); ++i) { 440 const auto& tx{block.data->vtx.at(i)}; 441 const bool is_coinbase{tx->IsCoinBase()}; 442 443 if (is_coinbase && IsBIP30Unspendable(block.hash, block.height)) { 444 continue; 445 } 446 447 for (uint32_t j = 0; j < tx->vout.size(); ++j) { 448 const CTxOut& out{tx->vout[j]}; 449 const COutPoint outpoint{tx->GetHash(), j}; 450 const Coin coin{out, block.height, is_coinbase}; 451 452 if (!coin.out.scriptPubKey.IsUnspendable()) { 453 RemoveCoinHash(m_muhash, outpoint, coin); 454 } 455 } 456 457 // The coinbase tx has no undo data since no former output is spent 458 if (!is_coinbase) { 459 const auto& tx_undo{block.undo_data->vtxundo.at(i - 1)}; 460 461 for (size_t j = 0; j < tx_undo.vprevout.size(); ++j) { 462 const Coin& coin{tx_undo.vprevout[j]}; 463 const COutPoint outpoint{tx->vin[j].prevout.hash, tx->vin[j].prevout.n}; 464 ApplyCoinHash(m_muhash, outpoint, coin); 465 } 466 } 467 } 468 469 // Check that the rolled back muhash is consistent with the DB read out 470 uint256 out; 471 m_muhash.Finalize(out); 472 Assert(read_out.second.muhash == out); 473 474 // Apply the other values from the DB to the member variables 475 m_transaction_output_count = read_out.second.transaction_output_count; 476 m_total_amount = read_out.second.total_amount; 477 m_bogo_size = read_out.second.bogo_size; 478 m_total_subsidy = read_out.second.total_subsidy; 479 m_total_prevout_spent_amount = read_out.second.total_prevout_spent_amount; 480 m_total_new_outputs_ex_coinbase_amount = read_out.second.total_new_outputs_ex_coinbase_amount; 481 m_total_coinbase_amount = read_out.second.total_coinbase_amount; 482 m_total_unspendables_genesis_block = read_out.second.total_unspendables_genesis_block; 483 m_total_unspendables_bip30 = read_out.second.total_unspendables_bip30; 484 m_total_unspendables_scripts = read_out.second.total_unspendables_scripts; 485 m_total_unspendables_unclaimed_rewards = read_out.second.total_unspendables_unclaimed_rewards; 486 m_current_block_hash = *block.prev_hash; 487 488 return true; 489 }