1  // Copyright (c) 2020-2022 The Bitcoin Core developers
  2  // Distributed under the MIT software license, see the accompanying
  3  // file COPYING or http://www.opensource.org/licenses/mit-license.php.
  4  
  5  #include <chainparams.h>
  6  #include <coins.h>
  7  #include <common/args.h>
  8  #include <crypto/muhash.h>
  9  #include <index/coinstatsindex.h>
 10  #include <kernel/coinstats.h>
 11  #include <logging.h>
 12  #include <node/blockstorage.h>
 13  #include <serialize.h>
 14  #include <txdb.h>
 15  #include <undo.h>
 16  #include <validation.h>
 17  
 18  using kernel::ApplyCoinHash;
 19  using kernel::CCoinsStats;
 20  using kernel::GetBogoSize;
 21  using kernel::RemoveCoinHash;
 22  
 23  static constexpr uint8_t DB_BLOCK_HASH{'s'};
 24  static constexpr uint8_t DB_BLOCK_HEIGHT{'t'};
 25  static constexpr uint8_t DB_MUHASH{'M'};
 26  
 27  namespace {
 28  
 29  struct DBVal {
 30      uint256 muhash;
 31      uint64_t transaction_output_count;
 32      uint64_t bogo_size;
 33      CAmount total_amount;
 34      CAmount total_subsidy;
 35      CAmount total_unspendable_amount;
 36      CAmount total_prevout_spent_amount;
 37      CAmount total_new_outputs_ex_coinbase_amount;
 38      CAmount total_coinbase_amount;
 39      CAmount total_unspendables_genesis_block;
 40      CAmount total_unspendables_bip30;
 41      CAmount total_unspendables_scripts;
 42      CAmount total_unspendables_unclaimed_rewards;
 43  
 44      SERIALIZE_METHODS(DBVal, obj)
 45      {
 46          READWRITE(obj.muhash);
 47          READWRITE(obj.transaction_output_count);
 48          READWRITE(obj.bogo_size);
 49          READWRITE(obj.total_amount);
 50          READWRITE(obj.total_subsidy);
 51          READWRITE(obj.total_unspendable_amount);
 52          READWRITE(obj.total_prevout_spent_amount);
 53          READWRITE(obj.total_new_outputs_ex_coinbase_amount);
 54          READWRITE(obj.total_coinbase_amount);
 55          READWRITE(obj.total_unspendables_genesis_block);
 56          READWRITE(obj.total_unspendables_bip30);
 57          READWRITE(obj.total_unspendables_scripts);
 58          READWRITE(obj.total_unspendables_unclaimed_rewards);
 59      }
 60  };
 61  
 62  struct DBHeightKey {
 63      int height;
 64  
 65      explicit DBHeightKey(int height_in) : height(height_in) {}
 66  
 67      template <typename Stream>
 68      void Serialize(Stream& s) const
 69      {
 70          ser_writedata8(s, DB_BLOCK_HEIGHT);
 71          ser_writedata32be(s, height);
 72      }
 73  
 74      template <typename Stream>
 75      void Unserialize(Stream& s)
 76      {
 77          const uint8_t prefix{ser_readdata8(s)};
 78          if (prefix != DB_BLOCK_HEIGHT) {
 79              throw std::ios_base::failure("Invalid format for coinstatsindex DB height key");
 80          }
 81          height = ser_readdata32be(s);
 82      }
 83  };
 84  
 85  struct DBHashKey {
 86      uint256 block_hash;
 87  
 88      explicit DBHashKey(const uint256& hash_in) : block_hash(hash_in) {}
 89  
 90      SERIALIZE_METHODS(DBHashKey, obj)
 91      {
 92          uint8_t prefix{DB_BLOCK_HASH};
 93          READWRITE(prefix);
 94          if (prefix != DB_BLOCK_HASH) {
 95              throw std::ios_base::failure("Invalid format for coinstatsindex DB hash key");
 96          }
 97  
 98          READWRITE(obj.block_hash);
 99      }
100  };
101  
102  }; // namespace
103  
104  std::unique_ptr<CoinStatsIndex> g_coin_stats_index;
105  
106  CoinStatsIndex::CoinStatsIndex(std::unique_ptr<interfaces::Chain> chain, size_t n_cache_size, bool f_memory, bool f_wipe)
107      : BaseIndex(std::move(chain), "coinstatsindex")
108  {
109      fs::path path{gArgs.GetDataDirNet() / "indexes" / "coinstats"};
110      fs::create_directories(path);
111  
112      m_db = std::make_unique<CoinStatsIndex::DB>(path / "db", n_cache_size, f_memory, f_wipe);
113  }
114  
115  bool CoinStatsIndex::CustomAppend(const interfaces::BlockInfo& block)
116  {
117      CBlockUndo block_undo;
118      const CAmount block_subsidy{GetBlockSubsidy(block.height, Params().GetConsensus())};
119      m_total_subsidy += block_subsidy;
120  
121      // Ignore genesis block
122      if (block.height > 0) {
123          // pindex variable gives indexing code access to node internals. It
124          // will be removed in upcoming commit
125          const CBlockIndex* pindex = WITH_LOCK(cs_main, return m_chainstate->m_blockman.LookupBlockIndex(block.hash));
126          if (!m_chainstate->m_blockman.UndoReadFromDisk(block_undo, *pindex)) {
127              return false;
128          }
129  
130          std::pair<uint256, DBVal> read_out;
131          if (!m_db->Read(DBHeightKey(block.height - 1), read_out)) {
132              return false;
133          }
134  
135          uint256 expected_block_hash{*Assert(block.prev_hash)};
136          if (read_out.first != expected_block_hash) {
137              LogPrintf("WARNING: previous block header belongs to unexpected block %s; expected %s\n",
138                        read_out.first.ToString(), expected_block_hash.ToString());
139  
140              if (!m_db->Read(DBHashKey(expected_block_hash), read_out)) {
141                  LogError("%s: previous block header not found; expected %s\n",
142                               __func__, expected_block_hash.ToString());
143                  return false;
144              }
145          }
146  
147          // Add the new utxos created from the block
148          assert(block.data);
149          for (size_t i = 0; i < block.data->vtx.size(); ++i) {
150              const auto& tx{block.data->vtx.at(i)};
151  
152              // Skip duplicate txid coinbase transactions (BIP30).
153              if (IsBIP30Unspendable(*pindex) && tx->IsCoinBase()) {
154                  m_total_unspendable_amount += block_subsidy;
155                  m_total_unspendables_bip30 += block_subsidy;
156                  continue;
157              }
158  
159              for (uint32_t j = 0; j < tx->vout.size(); ++j) {
160                  const CTxOut& out{tx->vout[j]};
161                  Coin coin{out, block.height, tx->IsCoinBase()};
162                  COutPoint outpoint{tx->GetHash(), j};
163  
164                  // Skip unspendable coins
165                  if (coin.out.scriptPubKey.IsUnspendable()) {
166                      m_total_unspendable_amount += coin.out.nValue;
167                      m_total_unspendables_scripts += coin.out.nValue;
168                      continue;
169                  }
170  
171                  ApplyCoinHash(m_muhash, outpoint, coin);
172  
173                  if (tx->IsCoinBase()) {
174                      m_total_coinbase_amount += coin.out.nValue;
175                  } else {
176                      m_total_new_outputs_ex_coinbase_amount += coin.out.nValue;
177                  }
178  
179                  ++m_transaction_output_count;
180                  m_total_amount += coin.out.nValue;
181                  m_bogo_size += GetBogoSize(coin.out.scriptPubKey);
182              }
183  
184              // The coinbase tx has no undo data since no former output is spent
185              if (!tx->IsCoinBase()) {
186                  const auto& tx_undo{block_undo.vtxundo.at(i - 1)};
187  
188                  for (size_t j = 0; j < tx_undo.vprevout.size(); ++j) {
189                      Coin coin{tx_undo.vprevout[j]};
190                      COutPoint outpoint{tx->vin[j].prevout.hash, tx->vin[j].prevout.n};
191  
192                      RemoveCoinHash(m_muhash, outpoint, coin);
193  
194                      m_total_prevout_spent_amount += coin.out.nValue;
195  
196                      --m_transaction_output_count;
197                      m_total_amount -= coin.out.nValue;
198                      m_bogo_size -= GetBogoSize(coin.out.scriptPubKey);
199                  }
200              }
201          }
202      } else {
203          // genesis block
204          m_total_unspendable_amount += block_subsidy;
205          m_total_unspendables_genesis_block += block_subsidy;
206      }
207  
208      // If spent prevouts + block subsidy are still a higher amount than
209      // new outputs + coinbase + current unspendable amount this means
210      // the miner did not claim the full block reward. Unclaimed block
211      // rewards are also unspendable.
212      const CAmount unclaimed_rewards{(m_total_prevout_spent_amount + m_total_subsidy) - (m_total_new_outputs_ex_coinbase_amount + m_total_coinbase_amount + m_total_unspendable_amount)};
213      m_total_unspendable_amount += unclaimed_rewards;
214      m_total_unspendables_unclaimed_rewards += unclaimed_rewards;
215  
216      std::pair<uint256, DBVal> value;
217      value.first = block.hash;
218      value.second.transaction_output_count = m_transaction_output_count;
219      value.second.bogo_size = m_bogo_size;
220      value.second.total_amount = m_total_amount;
221      value.second.total_subsidy = m_total_subsidy;
222      value.second.total_unspendable_amount = m_total_unspendable_amount;
223      value.second.total_prevout_spent_amount = m_total_prevout_spent_amount;
224      value.second.total_new_outputs_ex_coinbase_amount = m_total_new_outputs_ex_coinbase_amount;
225      value.second.total_coinbase_amount = m_total_coinbase_amount;
226      value.second.total_unspendables_genesis_block = m_total_unspendables_genesis_block;
227      value.second.total_unspendables_bip30 = m_total_unspendables_bip30;
228      value.second.total_unspendables_scripts = m_total_unspendables_scripts;
229      value.second.total_unspendables_unclaimed_rewards = m_total_unspendables_unclaimed_rewards;
230  
231      uint256 out;
232      m_muhash.Finalize(out);
233      value.second.muhash = out;
234  
235      // Intentionally do not update DB_MUHASH here so it stays in sync with
236      // DB_BEST_BLOCK, and the index is not corrupted if there is an unclean shutdown.
237      return m_db->Write(DBHeightKey(block.height), value);
238  }
239  
240  [[nodiscard]] static bool CopyHeightIndexToHashIndex(CDBIterator& db_it, CDBBatch& batch,
241                                         const std::string& index_name,
242                                         int start_height, int stop_height)
243  {
244      DBHeightKey key{start_height};
245      db_it.Seek(key);
246  
247      for (int height = start_height; height <= stop_height; ++height) {
248          if (!db_it.GetKey(key) || key.height != height) {
249              LogError("%s: unexpected key in %s: expected (%c, %d)\n",
250                           __func__, index_name, DB_BLOCK_HEIGHT, height);
251              return false;
252          }
253  
254          std::pair<uint256, DBVal> value;
255          if (!db_it.GetValue(value)) {
256              LogError("%s: unable to read value in %s at key (%c, %d)\n",
257                           __func__, index_name, DB_BLOCK_HEIGHT, height);
258              return false;
259          }
260  
261          batch.Write(DBHashKey(value.first), std::move(value.second));
262  
263          db_it.Next();
264      }
265      return true;
266  }
267  
268  bool CoinStatsIndex::CustomRewind(const interfaces::BlockKey& current_tip, const interfaces::BlockKey& new_tip)
269  {
270      CDBBatch batch(*m_db);
271      std::unique_ptr<CDBIterator> db_it(m_db->NewIterator());
272  
273      // During a reorg, we need to copy all hash digests for blocks that are
274      // getting disconnected from the height index to the hash index so we can
275      // still find them when the height index entries are overwritten.
276      if (!CopyHeightIndexToHashIndex(*db_it, batch, m_name, new_tip.height, current_tip.height)) {
277          return false;
278      }
279  
280      if (!m_db->WriteBatch(batch)) return false;
281  
282      {
283          LOCK(cs_main);
284          const CBlockIndex* iter_tip{m_chainstate->m_blockman.LookupBlockIndex(current_tip.hash)};
285          const CBlockIndex* new_tip_index{m_chainstate->m_blockman.LookupBlockIndex(new_tip.hash)};
286  
287          do {
288              CBlock block;
289  
290              if (!m_chainstate->m_blockman.ReadBlockFromDisk(block, *iter_tip)) {
291                  LogError("%s: Failed to read block %s from disk\n",
292                               __func__, iter_tip->GetBlockHash().ToString());
293                  return false;
294              }
295  
296              if (!ReverseBlock(block, iter_tip)) {
297                  return false; // failure cause logged internally
298              }
299  
300              iter_tip = iter_tip->GetAncestor(iter_tip->nHeight - 1);
301          } while (new_tip_index != iter_tip);
302      }
303  
304      return true;
305  }
306  
307  static bool LookUpOne(const CDBWrapper& db, const interfaces::BlockKey& block, DBVal& result)
308  {
309      // First check if the result is stored under the height index and the value
310      // there matches the block hash. This should be the case if the block is on
311      // the active chain.
312      std::pair<uint256, DBVal> read_out;
313      if (!db.Read(DBHeightKey(block.height), read_out)) {
314          return false;
315      }
316      if (read_out.first == block.hash) {
317          result = std::move(read_out.second);
318          return true;
319      }
320  
321      // If value at the height index corresponds to an different block, the
322      // result will be stored in the hash index.
323      return db.Read(DBHashKey(block.hash), result);
324  }
325  
326  std::optional<CCoinsStats> CoinStatsIndex::LookUpStats(const CBlockIndex& block_index) const
327  {
328      CCoinsStats stats{block_index.nHeight, block_index.GetBlockHash()};
329      stats.index_used = true;
330  
331      DBVal entry;
332      if (!LookUpOne(*m_db, {block_index.GetBlockHash(), block_index.nHeight}, entry)) {
333          return std::nullopt;
334      }
335  
336      stats.hashSerialized = entry.muhash;
337      stats.nTransactionOutputs = entry.transaction_output_count;
338      stats.nBogoSize = entry.bogo_size;
339      stats.total_amount = entry.total_amount;
340      stats.total_subsidy = entry.total_subsidy;
341      stats.total_unspendable_amount = entry.total_unspendable_amount;
342      stats.total_prevout_spent_amount = entry.total_prevout_spent_amount;
343      stats.total_new_outputs_ex_coinbase_amount = entry.total_new_outputs_ex_coinbase_amount;
344      stats.total_coinbase_amount = entry.total_coinbase_amount;
345      stats.total_unspendables_genesis_block = entry.total_unspendables_genesis_block;
346      stats.total_unspendables_bip30 = entry.total_unspendables_bip30;
347      stats.total_unspendables_scripts = entry.total_unspendables_scripts;
348      stats.total_unspendables_unclaimed_rewards = entry.total_unspendables_unclaimed_rewards;
349  
350      return stats;
351  }
352  
353  bool CoinStatsIndex::CustomInit(const std::optional<interfaces::BlockKey>& block)
354  {
355      if (!m_db->Read(DB_MUHASH, m_muhash)) {
356          // Check that the cause of the read failure is that the key does not
357          // exist. Any other errors indicate database corruption or a disk
358          // failure, and starting the index would cause further corruption.
359          if (m_db->Exists(DB_MUHASH)) {
360              LogError("%s: Cannot read current %s state; index may be corrupted\n",
361                           __func__, GetName());
362              return false;
363          }
364      }
365  
366      if (block) {
367          DBVal entry;
368          if (!LookUpOne(*m_db, *block, entry)) {
369              LogError("%s: Cannot read current %s state; index may be corrupted\n",
370                           __func__, GetName());
371              return false;
372          }
373  
374          uint256 out;
375          m_muhash.Finalize(out);
376          if (entry.muhash != out) {
377              LogError("%s: Cannot read current %s state; index may be corrupted\n",
378                           __func__, GetName());
379              return false;
380          }
381  
382          m_transaction_output_count = entry.transaction_output_count;
383          m_bogo_size = entry.bogo_size;
384          m_total_amount = entry.total_amount;
385          m_total_subsidy = entry.total_subsidy;
386          m_total_unspendable_amount = entry.total_unspendable_amount;
387          m_total_prevout_spent_amount = entry.total_prevout_spent_amount;
388          m_total_new_outputs_ex_coinbase_amount = entry.total_new_outputs_ex_coinbase_amount;
389          m_total_coinbase_amount = entry.total_coinbase_amount;
390          m_total_unspendables_genesis_block = entry.total_unspendables_genesis_block;
391          m_total_unspendables_bip30 = entry.total_unspendables_bip30;
392          m_total_unspendables_scripts = entry.total_unspendables_scripts;
393          m_total_unspendables_unclaimed_rewards = entry.total_unspendables_unclaimed_rewards;
394      }
395  
396      return true;
397  }
398  
399  bool CoinStatsIndex::CustomCommit(CDBBatch& batch)
400  {
401      // DB_MUHASH should always be committed in a batch together with DB_BEST_BLOCK
402      // to prevent an inconsistent state of the DB.
403      batch.Write(DB_MUHASH, m_muhash);
404      return true;
405  }
406  
407  // Reverse a single block as part of a reorg
408  bool CoinStatsIndex::ReverseBlock(const CBlock& block, const CBlockIndex* pindex)
409  {
410      CBlockUndo block_undo;
411      std::pair<uint256, DBVal> read_out;
412  
413      const CAmount block_subsidy{GetBlockSubsidy(pindex->nHeight, Params().GetConsensus())};
414      m_total_subsidy -= block_subsidy;
415  
416      // Ignore genesis block
417      if (pindex->nHeight > 0) {
418          if (!m_chainstate->m_blockman.UndoReadFromDisk(block_undo, *pindex)) {
419              return false;
420          }
421  
422          if (!m_db->Read(DBHeightKey(pindex->nHeight - 1), read_out)) {
423              return false;
424          }
425  
426          uint256 expected_block_hash{pindex->pprev->GetBlockHash()};
427          if (read_out.first != expected_block_hash) {
428              LogPrintf("WARNING: previous block header belongs to unexpected block %s; expected %s\n",
429                        read_out.first.ToString(), expected_block_hash.ToString());
430  
431              if (!m_db->Read(DBHashKey(expected_block_hash), read_out)) {
432                  LogError("%s: previous block header not found; expected %s\n",
433                               __func__, expected_block_hash.ToString());
434                  return false;
435              }
436          }
437      }
438  
439      // Remove the new UTXOs that were created from the block
440      for (size_t i = 0; i < block.vtx.size(); ++i) {
441          const auto& tx{block.vtx.at(i)};
442  
443          for (uint32_t j = 0; j < tx->vout.size(); ++j) {
444              const CTxOut& out{tx->vout[j]};
445              COutPoint outpoint{tx->GetHash(), j};
446              Coin coin{out, pindex->nHeight, tx->IsCoinBase()};
447  
448              // Skip unspendable coins
449              if (coin.out.scriptPubKey.IsUnspendable()) {
450                  m_total_unspendable_amount -= coin.out.nValue;
451                  m_total_unspendables_scripts -= coin.out.nValue;
452                  continue;
453              }
454  
455              RemoveCoinHash(m_muhash, outpoint, coin);
456  
457              if (tx->IsCoinBase()) {
458                  m_total_coinbase_amount -= coin.out.nValue;
459              } else {
460                  m_total_new_outputs_ex_coinbase_amount -= coin.out.nValue;
461              }
462  
463              --m_transaction_output_count;
464              m_total_amount -= coin.out.nValue;
465              m_bogo_size -= GetBogoSize(coin.out.scriptPubKey);
466          }
467  
468          // The coinbase tx has no undo data since no former output is spent
469          if (!tx->IsCoinBase()) {
470              const auto& tx_undo{block_undo.vtxundo.at(i - 1)};
471  
472              for (size_t j = 0; j < tx_undo.vprevout.size(); ++j) {
473                  Coin coin{tx_undo.vprevout[j]};
474                  COutPoint outpoint{tx->vin[j].prevout.hash, tx->vin[j].prevout.n};
475  
476                  ApplyCoinHash(m_muhash, outpoint, coin);
477  
478                  m_total_prevout_spent_amount -= coin.out.nValue;
479  
480                  m_transaction_output_count++;
481                  m_total_amount += coin.out.nValue;
482                  m_bogo_size += GetBogoSize(coin.out.scriptPubKey);
483              }
484          }
485      }
486  
487      const CAmount unclaimed_rewards{(m_total_new_outputs_ex_coinbase_amount + m_total_coinbase_amount + m_total_unspendable_amount) - (m_total_prevout_spent_amount + m_total_subsidy)};
488      m_total_unspendable_amount -= unclaimed_rewards;
489      m_total_unspendables_unclaimed_rewards -= unclaimed_rewards;
490  
491      // Check that the rolled back internal values are consistent with the DB read out
492      uint256 out;
493      m_muhash.Finalize(out);
494      Assert(read_out.second.muhash == out);
495  
496      Assert(m_transaction_output_count == read_out.second.transaction_output_count);
497      Assert(m_total_amount == read_out.second.total_amount);
498      Assert(m_bogo_size == read_out.second.bogo_size);
499      Assert(m_total_subsidy == read_out.second.total_subsidy);
500      Assert(m_total_unspendable_amount == read_out.second.total_unspendable_amount);
501      Assert(m_total_prevout_spent_amount == read_out.second.total_prevout_spent_amount);
502      Assert(m_total_new_outputs_ex_coinbase_amount == read_out.second.total_new_outputs_ex_coinbase_amount);
503      Assert(m_total_coinbase_amount == read_out.second.total_coinbase_amount);
504      Assert(m_total_unspendables_genesis_block == read_out.second.total_unspendables_genesis_block);
505      Assert(m_total_unspendables_bip30 == read_out.second.total_unspendables_bip30);
506      Assert(m_total_unspendables_scripts == read_out.second.total_unspendables_scripts);
507      Assert(m_total_unspendables_unclaimed_rewards == read_out.second.total_unspendables_unclaimed_rewards);
508  
509      return true;
510  }