1  // Copyright (c) 2009-2010 Satoshi Nakamoto
  2  // Copyright (c) 2009-2022 The Bitcoin Core developers
  3  // Distributed under the MIT software license, see the accompanying
  4  // file COPYING or http://www.opensource.org/licenses/mit-license.php.
  5  
  6  #include <compat/compat.h>
  7  #include <logging.h>
  8  #include <util/fs.h>
  9  #include <util/time.h>
 10  #include <wallet/bdb.h>
 11  #include <wallet/db.h>
 12  
 13  #include <sync.h>
 14  #include <util/check.h>
 15  #include <util/fs_helpers.h>
 16  #include <util/strencodings.h>
 17  #include <util/translation.h>
 18  
 19  #include <stdint.h>
 20  
 21  #include <db_cxx.h>
 22  #include <sys/stat.h>
 23  
 24  // Windows may not define S_IRUSR or S_IWUSR. We define both
 25  // here, with the same values as glibc (see stat.h).
 26  #ifdef WIN32
 27  #ifndef S_IRUSR
 28  #define S_IRUSR             0400
 29  #define S_IWUSR             0200
 30  #endif
 31  #endif
 32  
 33  static_assert(BDB_DB_FILE_ID_LEN == DB_FILE_ID_LEN, "DB_FILE_ID_LEN should be 20.");
 34  
 35  namespace wallet {
 36  namespace {
 37  
 38  //! Make sure database has a unique fileid within the environment. If it
 39  //! doesn't, throw an error. BDB caches do not work properly when more than one
 40  //! open database has the same fileid (values written to one database may show
 41  //! up in reads to other databases).
 42  //!
 43  //! BerkeleyDB generates unique fileids by default
 44  //! (https://docs.oracle.com/cd/E17275_01/html/programmer_reference/program_copy.html),
 45  //! so bitcoin should never create different databases with the same fileid, but
 46  //! this error can be triggered if users manually copy database files.
 47  void CheckUniqueFileid(const BerkeleyEnvironment& env, const std::string& filename, Db& db, WalletDatabaseFileId& fileid)
 48  {
 49      if (env.IsMock()) return;
 50  
 51      int ret = db.get_mpf()->get_fileid(fileid.value);
 52      if (ret != 0) {
 53          throw std::runtime_error(strprintf("BerkeleyDatabase: Can't open database %s (get_fileid failed with %d)", filename, ret));
 54      }
 55  
 56      for (const auto& item : env.m_fileids) {
 57          if (fileid == item.second && &fileid != &item.second) {
 58              throw std::runtime_error(strprintf("BerkeleyDatabase: Can't open database %s (duplicates fileid %s from %s)", filename,
 59                  HexStr(item.second.value), item.first));
 60          }
 61      }
 62  }
 63  
 64  RecursiveMutex cs_db;
 65  std::map<std::string, std::weak_ptr<BerkeleyEnvironment>> g_dbenvs GUARDED_BY(cs_db); //!< Map from directory name to db environment.
 66  } // namespace
 67  
 68  bool WalletDatabaseFileId::operator==(const WalletDatabaseFileId& rhs) const
 69  {
 70      return memcmp(value, &rhs.value, sizeof(value)) == 0;
 71  }
 72  
 73  /**
 74   * @param[in] env_directory Path to environment directory
 75   * @return A shared pointer to the BerkeleyEnvironment object for the wallet directory, never empty because ~BerkeleyEnvironment
 76   * erases the weak pointer from the g_dbenvs map.
 77   * @post A new BerkeleyEnvironment weak pointer is inserted into g_dbenvs if the directory path key was not already in the map.
 78   */
 79  std::shared_ptr<BerkeleyEnvironment> GetBerkeleyEnv(const fs::path& env_directory, bool use_shared_memory)
 80  {
 81      LOCK(cs_db);
 82      auto inserted = g_dbenvs.emplace(fs::PathToString(env_directory), std::weak_ptr<BerkeleyEnvironment>());
 83      if (inserted.second) {
 84          auto env = std::make_shared<BerkeleyEnvironment>(env_directory, use_shared_memory);
 85          inserted.first->second = env;
 86          return env;
 87      }
 88      return inserted.first->second.lock();
 89  }
 90  
 91  //
 92  // BerkeleyBatch
 93  //
 94  
 95  void BerkeleyEnvironment::Close()
 96  {
 97      if (!fDbEnvInit)
 98          return;
 99  
100      fDbEnvInit = false;
101  
102      for (auto& db : m_databases) {
103          BerkeleyDatabase& database = db.second.get();
104          assert(database.m_refcount <= 0);
105          if (database.m_db) {
106              database.m_db->close(0);
107              database.m_db.reset();
108          }
109      }
110  
111      FILE* error_file = nullptr;
112      dbenv->get_errfile(&error_file);
113  
114      int ret = dbenv->close(0);
115      if (ret != 0)
116          LogPrintf("BerkeleyEnvironment::Close: Error %d closing database environment: %s\n", ret, DbEnv::strerror(ret));
117      if (!fMockDb)
118          DbEnv(uint32_t{0}).remove(strPath.c_str(), 0);
119  
120      if (error_file) fclose(error_file);
121  
122      UnlockDirectory(fs::PathFromString(strPath), ".walletlock");
123  }
124  
125  void BerkeleyEnvironment::Reset()
126  {
127      dbenv.reset(new DbEnv(DB_CXX_NO_EXCEPTIONS));
128      fDbEnvInit = false;
129      fMockDb = false;
130  }
131  
132  BerkeleyEnvironment::BerkeleyEnvironment(const fs::path& dir_path, bool use_shared_memory) : strPath(fs::PathToString(dir_path)), m_use_shared_memory(use_shared_memory)
133  {
134      Reset();
135  }
136  
137  BerkeleyEnvironment::~BerkeleyEnvironment()
138  {
139      LOCK(cs_db);
140      g_dbenvs.erase(strPath);
141      Close();
142  }
143  
144  bool BerkeleyEnvironment::Open(bilingual_str& err)
145  {
146      if (fDbEnvInit) {
147          return true;
148      }
149  
150      fs::path pathIn = fs::PathFromString(strPath);
151      TryCreateDirectories(pathIn);
152      if (util::LockDirectory(pathIn, ".walletlock") != util::LockResult::Success) {
153          LogPrintf("Cannot obtain a lock on wallet directory %s. Another instance may be using it.\n", strPath);
154          err = strprintf(_("Error initializing wallet database environment %s!"), fs::quoted(fs::PathToString(Directory())));
155          return false;
156      }
157  
158      fs::path pathLogDir = pathIn / "database";
159      TryCreateDirectories(pathLogDir);
160      fs::path pathErrorFile = pathIn / "db.log";
161      LogPrintf("BerkeleyEnvironment::Open: LogDir=%s ErrorFile=%s\n", fs::PathToString(pathLogDir), fs::PathToString(pathErrorFile));
162  
163      unsigned int nEnvFlags = 0;
164      if (!m_use_shared_memory) {
165          nEnvFlags |= DB_PRIVATE;
166      }
167  
168      dbenv->set_lg_dir(fs::PathToString(pathLogDir).c_str());
169      dbenv->set_cachesize(0, 0x100000, 1); // 1 MiB should be enough for just the wallet
170      dbenv->set_lg_bsize(0x10000);
171      dbenv->set_lg_max(1048576);
172      dbenv->set_lk_max_locks(40000);
173      dbenv->set_lk_max_objects(40000);
174      dbenv->set_errfile(fsbridge::fopen(pathErrorFile, "a")); /// debug
175      dbenv->set_flags(DB_AUTO_COMMIT, 1);
176      dbenv->set_flags(DB_TXN_WRITE_NOSYNC, 1);
177      dbenv->log_set_config(DB_LOG_AUTO_REMOVE, 1);
178      int ret = dbenv->open(strPath.c_str(),
179                           DB_CREATE |
180                               DB_INIT_LOCK |
181                               DB_INIT_LOG |
182                               DB_INIT_MPOOL |
183                               DB_INIT_TXN |
184                               DB_THREAD |
185                               DB_RECOVER |
186                               nEnvFlags,
187                           S_IRUSR | S_IWUSR);
188      if (ret != 0) {
189          LogPrintf("BerkeleyEnvironment::Open: Error %d opening database environment: %s\n", ret, DbEnv::strerror(ret));
190          int ret2 = dbenv->close(0);
191          if (ret2 != 0) {
192              LogPrintf("BerkeleyEnvironment::Open: Error %d closing failed database environment: %s\n", ret2, DbEnv::strerror(ret2));
193          }
194          Reset();
195          err = strprintf(_("Error initializing wallet database environment %s!"), fs::quoted(fs::PathToString(Directory())));
196          if (ret == DB_RUNRECOVERY) {
197              err += Untranslated(" ") + _("This error could occur if this wallet was not shutdown cleanly and was last loaded using a build with a newer version of Berkeley DB. If so, please use the software that last loaded this wallet");
198          }
199          return false;
200      }
201  
202      fDbEnvInit = true;
203      fMockDb = false;
204      return true;
205  }
206  
207  //! Construct an in-memory mock Berkeley environment for testing
208  BerkeleyEnvironment::BerkeleyEnvironment() : m_use_shared_memory(false)
209  {
210      Reset();
211  
212      LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::MakeMock\n");
213  
214      dbenv->set_cachesize(1, 0, 1);
215      dbenv->set_lg_bsize(10485760 * 4);
216      dbenv->set_lg_max(10485760);
217      dbenv->set_lk_max_locks(10000);
218      dbenv->set_lk_max_objects(10000);
219      dbenv->set_flags(DB_AUTO_COMMIT, 1);
220      dbenv->log_set_config(DB_LOG_IN_MEMORY, 1);
221      int ret = dbenv->open(nullptr,
222                           DB_CREATE |
223                               DB_INIT_LOCK |
224                               DB_INIT_LOG |
225                               DB_INIT_MPOOL |
226                               DB_INIT_TXN |
227                               DB_THREAD |
228                               DB_PRIVATE,
229                           S_IRUSR | S_IWUSR);
230      if (ret > 0) {
231          throw std::runtime_error(strprintf("BerkeleyEnvironment::MakeMock: Error %d opening database environment.", ret));
232      }
233  
234      fDbEnvInit = true;
235      fMockDb = true;
236  }
237  
238  /** RAII class that automatically cleanses its data on destruction */
239  class SafeDbt final
240  {
241      Dbt m_dbt;
242  
243  public:
244      // construct Dbt with internally-managed data
245      SafeDbt();
246      // construct Dbt with provided data
247      SafeDbt(void* data, size_t size);
248      ~SafeDbt();
249  
250      // delegate to Dbt
251      const void* get_data() const;
252      uint32_t get_size() const;
253  
254      // conversion operator to access the underlying Dbt
255      operator Dbt*();
256  };
257  
258  SafeDbt::SafeDbt()
259  {
260      m_dbt.set_flags(DB_DBT_MALLOC);
261  }
262  
263  SafeDbt::SafeDbt(void* data, size_t size)
264      : m_dbt(data, size)
265  {
266  }
267  
268  SafeDbt::~SafeDbt()
269  {
270      if (m_dbt.get_data() != nullptr) {
271          // Clear memory, e.g. in case it was a private key
272          memory_cleanse(m_dbt.get_data(), m_dbt.get_size());
273          // under DB_DBT_MALLOC, data is malloced by the Dbt, but must be
274          // freed by the caller.
275          // https://docs.oracle.com/cd/E17275_01/html/api_reference/C/dbt.html
276          if (m_dbt.get_flags() & DB_DBT_MALLOC) {
277              free(m_dbt.get_data());
278          }
279      }
280  }
281  
282  const void* SafeDbt::get_data() const
283  {
284      return m_dbt.get_data();
285  }
286  
287  uint32_t SafeDbt::get_size() const
288  {
289      return m_dbt.get_size();
290  }
291  
292  SafeDbt::operator Dbt*()
293  {
294      return &m_dbt;
295  }
296  
297  static Span<const std::byte> SpanFromDbt(const SafeDbt& dbt)
298  {
299      return {reinterpret_cast<const std::byte*>(dbt.get_data()), dbt.get_size()};
300  }
301  
302  BerkeleyDatabase::BerkeleyDatabase(std::shared_ptr<BerkeleyEnvironment> env, fs::path filename, const DatabaseOptions& options) :
303      WalletDatabase(), env(std::move(env)), m_filename(std::move(filename)), m_max_log_mb(options.max_log_mb)
304  {
305      auto inserted = this->env->m_databases.emplace(m_filename, std::ref(*this));
306      assert(inserted.second);
307  }
308  
309  bool BerkeleyDatabase::Verify(bilingual_str& errorStr)
310  {
311      fs::path walletDir = env->Directory();
312      fs::path file_path = walletDir / m_filename;
313  
314      LogPrintf("Using BerkeleyDB version %s\n", BerkeleyDatabaseVersion());
315      LogPrintf("Using wallet %s\n", fs::PathToString(file_path));
316  
317      if (!env->Open(errorStr)) {
318          return false;
319      }
320  
321      if (fs::exists(file_path))
322      {
323          assert(m_refcount == 0);
324  
325          Db db(env->dbenv.get(), 0);
326          const std::string strFile = fs::PathToString(m_filename);
327          int result = db.verify(strFile.c_str(), nullptr, nullptr, 0);
328          if (result != 0) {
329              errorStr = strprintf(_("%s corrupt. Try using the wallet tool bitcoin-wallet to salvage or restoring a backup."), fs::quoted(fs::PathToString(file_path)));
330              return false;
331          }
332      }
333      // also return true if files does not exists
334      return true;
335  }
336  
337  void BerkeleyEnvironment::CheckpointLSN(const std::string& strFile)
338  {
339      dbenv->txn_checkpoint(0, 0, 0);
340      if (fMockDb)
341          return;
342      dbenv->lsn_reset(strFile.c_str(), 0);
343  }
344  
345  BerkeleyDatabase::~BerkeleyDatabase()
346  {
347      if (env) {
348          LOCK(cs_db);
349          env->CloseDb(m_filename);
350          assert(!m_db);
351          size_t erased = env->m_databases.erase(m_filename);
352          assert(erased == 1);
353          env->m_fileids.erase(fs::PathToString(m_filename));
354      }
355  }
356  
357  BerkeleyBatch::BerkeleyBatch(BerkeleyDatabase& database, const bool read_only, bool fFlushOnCloseIn) : m_database(database)
358  {
359      database.AddRef();
360      database.Open();
361      fReadOnly = read_only;
362      fFlushOnClose = fFlushOnCloseIn;
363      env = database.env.get();
364      pdb = database.m_db.get();
365      strFile = fs::PathToString(database.m_filename);
366  }
367  
368  void BerkeleyDatabase::Open()
369  {
370      unsigned int nFlags = DB_THREAD | DB_CREATE;
371  
372      {
373          LOCK(cs_db);
374          bilingual_str open_err;
375          if (!env->Open(open_err))
376              throw std::runtime_error("BerkeleyDatabase: Failed to open database environment.");
377  
378          if (m_db == nullptr) {
379              int ret;
380              std::unique_ptr<Db> pdb_temp = std::make_unique<Db>(env->dbenv.get(), 0);
381              const std::string strFile = fs::PathToString(m_filename);
382  
383              bool fMockDb = env->IsMock();
384              if (fMockDb) {
385                  DbMpoolFile* mpf = pdb_temp->get_mpf();
386                  ret = mpf->set_flags(DB_MPOOL_NOFILE, 1);
387                  if (ret != 0) {
388                      throw std::runtime_error(strprintf("BerkeleyDatabase: Failed to configure for no temp file backing for database %s", strFile));
389                  }
390              }
391  
392              ret = pdb_temp->open(nullptr,                             // Txn pointer
393                              fMockDb ? nullptr : strFile.c_str(),      // Filename
394                              fMockDb ? strFile.c_str() : "main",       // Logical db name
395                              DB_BTREE,                                 // Database type
396                              nFlags,                                   // Flags
397                              0);
398  
399              if (ret != 0) {
400                  throw std::runtime_error(strprintf("BerkeleyDatabase: Error %d, can't open database %s", ret, strFile));
401              }
402  
403              // Call CheckUniqueFileid on the containing BDB environment to
404              // avoid BDB data consistency bugs that happen when different data
405              // files in the same environment have the same fileid.
406              CheckUniqueFileid(*env, strFile, *pdb_temp, this->env->m_fileids[strFile]);
407  
408              m_db.reset(pdb_temp.release());
409  
410          }
411      }
412  }
413  
414  void BerkeleyBatch::Flush()
415  {
416      if (activeTxn)
417          return;
418  
419      // Flush database activity from memory pool to disk log
420      unsigned int nMinutes = 0;
421      if (fReadOnly)
422          nMinutes = 1;
423  
424      if (env) { // env is nullptr for dummy databases (i.e. in tests). Don't actually flush if env is nullptr so we don't segfault
425          env->dbenv->txn_checkpoint(nMinutes ? m_database.m_max_log_mb * 1024 : 0, nMinutes, 0);
426      }
427  }
428  
429  void BerkeleyDatabase::IncrementUpdateCounter()
430  {
431      ++nUpdateCounter;
432  }
433  
434  BerkeleyBatch::~BerkeleyBatch()
435  {
436      Close();
437      m_database.RemoveRef();
438  }
439  
440  void BerkeleyBatch::Close()
441  {
442      if (!pdb)
443          return;
444      if (activeTxn)
445          activeTxn->abort();
446      activeTxn = nullptr;
447      pdb = nullptr;
448  
449      if (fFlushOnClose)
450          Flush();
451  }
452  
453  void BerkeleyEnvironment::CloseDb(const fs::path& filename)
454  {
455      {
456          LOCK(cs_db);
457          auto it = m_databases.find(filename);
458          assert(it != m_databases.end());
459          BerkeleyDatabase& database = it->second.get();
460          if (database.m_db) {
461              // Close the database handle
462              database.m_db->close(0);
463              database.m_db.reset();
464          }
465      }
466  }
467  
468  void BerkeleyEnvironment::ReloadDbEnv()
469  {
470      // Make sure that no Db's are in use
471      AssertLockNotHeld(cs_db);
472      std::unique_lock<RecursiveMutex> lock(cs_db);
473      m_db_in_use.wait(lock, [this](){
474          for (auto& db : m_databases) {
475              if (db.second.get().m_refcount > 0) return false;
476          }
477          return true;
478      });
479  
480      std::vector<fs::path> filenames;
481      filenames.reserve(m_databases.size());
482      for (const auto& it : m_databases) {
483          filenames.push_back(it.first);
484      }
485      // Close the individual Db's
486      for (const fs::path& filename : filenames) {
487          CloseDb(filename);
488      }
489      // Reset the environment
490      Flush(true); // This will flush and close the environment
491      Reset();
492      bilingual_str open_err;
493      Open(open_err);
494  }
495  
496  DbTxn* BerkeleyEnvironment::TxnBegin(int flags)
497  {
498      DbTxn* ptxn = nullptr;
499      int ret = dbenv->txn_begin(nullptr, &ptxn, flags);
500      if (!ptxn || ret != 0)
501          return nullptr;
502      return ptxn;
503  }
504  
505  bool BerkeleyDatabase::Rewrite(const char* pszSkip)
506  {
507      while (true) {
508          {
509              LOCK(cs_db);
510              const std::string strFile = fs::PathToString(m_filename);
511              if (m_refcount <= 0) {
512                  // Flush log data to the dat file
513                  env->CloseDb(m_filename);
514                  env->CheckpointLSN(strFile);
515                  m_refcount = -1;
516  
517                  bool fSuccess = true;
518                  LogPrintf("BerkeleyBatch::Rewrite: Rewriting %s...\n", strFile);
519                  std::string strFileRes = strFile + ".rewrite";
520                  { // surround usage of db with extra {}
521                      BerkeleyBatch db(*this, true);
522                      std::unique_ptr<Db> pdbCopy = std::make_unique<Db>(env->dbenv.get(), 0);
523  
524                      int ret = pdbCopy->open(nullptr,               // Txn pointer
525                                              strFileRes.c_str(), // Filename
526                                              "main",             // Logical db name
527                                              DB_BTREE,           // Database type
528                                              DB_CREATE,          // Flags
529                                              0);
530                      if (ret > 0) {
531                          LogPrintf("BerkeleyBatch::Rewrite: Can't create database file %s\n", strFileRes);
532                          fSuccess = false;
533                      }
534  
535                      std::unique_ptr<DatabaseCursor> cursor = db.GetNewCursor();
536                      if (cursor) {
537                          while (fSuccess) {
538                              DataStream ssKey{};
539                              DataStream ssValue{};
540                              DatabaseCursor::Status ret1 = cursor->Next(ssKey, ssValue);
541                              if (ret1 == DatabaseCursor::Status::DONE) {
542                                  break;
543                              } else if (ret1 == DatabaseCursor::Status::FAIL) {
544                                  fSuccess = false;
545                                  break;
546                              }
547                              if (pszSkip &&
548                                  strncmp((const char*)ssKey.data(), pszSkip, std::min(ssKey.size(), strlen(pszSkip))) == 0)
549                                  continue;
550                              if (strncmp((const char*)ssKey.data(), "\x07version", 8) == 0) {
551                                  // Update version:
552                                  ssValue.clear();
553                                  ssValue << CLIENT_VERSION;
554                              }
555                              Dbt datKey(ssKey.data(), ssKey.size());
556                              Dbt datValue(ssValue.data(), ssValue.size());
557                              int ret2 = pdbCopy->put(nullptr, &datKey, &datValue, DB_NOOVERWRITE);
558                              if (ret2 > 0)
559                                  fSuccess = false;
560                          }
561                          cursor.reset();
562                      }
563                      if (fSuccess) {
564                          db.Close();
565                          env->CloseDb(m_filename);
566                          if (pdbCopy->close(0))
567                              fSuccess = false;
568                      } else {
569                          pdbCopy->close(0);
570                      }
571                  }
572                  if (fSuccess) {
573                      Db dbA(env->dbenv.get(), 0);
574                      if (dbA.remove(strFile.c_str(), nullptr, 0))
575                          fSuccess = false;
576                      Db dbB(env->dbenv.get(), 0);
577                      if (dbB.rename(strFileRes.c_str(), nullptr, strFile.c_str(), 0))
578                          fSuccess = false;
579                  }
580                  if (!fSuccess)
581                      LogPrintf("BerkeleyBatch::Rewrite: Failed to rewrite database file %s\n", strFileRes);
582                  return fSuccess;
583              }
584          }
585          UninterruptibleSleep(std::chrono::milliseconds{100});
586      }
587  }
588  
589  
590  void BerkeleyEnvironment::Flush(bool fShutdown)
591  {
592      const auto start{SteadyClock::now()};
593      // Flush log data to the actual data file on all files that are not in use
594      LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: [%s] Flush(%s)%s\n", strPath, fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started");
595      if (!fDbEnvInit)
596          return;
597      {
598          LOCK(cs_db);
599          bool no_dbs_accessed = true;
600          for (auto& db_it : m_databases) {
601              const fs::path& filename = db_it.first;
602              int nRefCount = db_it.second.get().m_refcount;
603              if (nRefCount < 0) continue;
604              const std::string strFile = fs::PathToString(filename);
605              LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: Flushing %s (refcount = %d)...\n", strFile, nRefCount);
606              if (nRefCount == 0) {
607                  // Move log data to the dat file
608                  CloseDb(filename);
609                  LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: %s checkpoint\n", strFile);
610                  dbenv->txn_checkpoint(0, 0, 0);
611                  LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: %s detach\n", strFile);
612                  if (!fMockDb)
613                      dbenv->lsn_reset(strFile.c_str(), 0);
614                  LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: %s closed\n", strFile);
615                  nRefCount = -1;
616              } else {
617                  no_dbs_accessed = false;
618              }
619          }
620          LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: Flush(%s)%s took %15dms\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started", Ticks<std::chrono::milliseconds>(SteadyClock::now() - start));
621          if (fShutdown) {
622              char** listp;
623              if (no_dbs_accessed) {
624                  dbenv->log_archive(&listp, DB_ARCH_REMOVE);
625                  Close();
626                  if (!fMockDb) {
627                      fs::remove_all(fs::PathFromString(strPath) / "database");
628                  }
629              }
630          }
631      }
632  }
633  
634  bool BerkeleyDatabase::PeriodicFlush()
635  {
636      // Don't flush if we can't acquire the lock.
637      TRY_LOCK(cs_db, lockDb);
638      if (!lockDb) return false;
639  
640      // Don't flush if any databases are in use
641      for (auto& it : env->m_databases) {
642          if (it.second.get().m_refcount > 0) return false;
643      }
644  
645      // Don't flush if there haven't been any batch writes for this database.
646      if (m_refcount < 0) return false;
647  
648      const std::string strFile = fs::PathToString(m_filename);
649      LogPrint(BCLog::WALLETDB, "Flushing %s\n", strFile);
650      const auto start{SteadyClock::now()};
651  
652      // Flush wallet file so it's self contained
653      env->CloseDb(m_filename);
654      env->CheckpointLSN(strFile);
655      m_refcount = -1;
656  
657      LogPrint(BCLog::WALLETDB, "Flushed %s %dms\n", strFile, Ticks<std::chrono::milliseconds>(SteadyClock::now() - start));
658  
659      return true;
660  }
661  
662  bool BerkeleyDatabase::Backup(const std::string& strDest) const
663  {
664      const std::string strFile = fs::PathToString(m_filename);
665      while (true)
666      {
667          {
668              LOCK(cs_db);
669              if (m_refcount <= 0)
670              {
671                  // Flush log data to the dat file
672                  env->CloseDb(m_filename);
673                  env->CheckpointLSN(strFile);
674  
675                  // Copy wallet file
676                  fs::path pathSrc = env->Directory() / m_filename;
677                  fs::path pathDest(fs::PathFromString(strDest));
678                  if (fs::is_directory(pathDest))
679                      pathDest /= m_filename;
680  
681                  try {
682                      if (fs::exists(pathDest) && fs::equivalent(pathSrc, pathDest)) {
683                          LogPrintf("cannot backup to wallet source file %s\n", fs::PathToString(pathDest));
684                          return false;
685                      }
686  
687                      fs::copy_file(pathSrc, pathDest, fs::copy_options::overwrite_existing);
688                      LogPrintf("copied %s to %s\n", strFile, fs::PathToString(pathDest));
689                      return true;
690                  } catch (const fs::filesystem_error& e) {
691                      LogPrintf("error copying %s to %s - %s\n", strFile, fs::PathToString(pathDest), fsbridge::get_filesystem_error_message(e));
692                      return false;
693                  }
694              }
695          }
696          UninterruptibleSleep(std::chrono::milliseconds{100});
697      }
698  }
699  
700  void BerkeleyDatabase::Flush()
701  {
702      env->Flush(false);
703  }
704  
705  void BerkeleyDatabase::Close()
706  {
707      env->Flush(true);
708  }
709  
710  void BerkeleyDatabase::ReloadDbEnv()
711  {
712      env->ReloadDbEnv();
713  }
714  
715  BerkeleyCursor::BerkeleyCursor(BerkeleyDatabase& database, const BerkeleyBatch& batch, Span<const std::byte> prefix)
716      : m_key_prefix(prefix.begin(), prefix.end())
717  {
718      if (!database.m_db.get()) {
719          throw std::runtime_error(STR_INTERNAL_BUG("BerkeleyDatabase does not exist"));
720      }
721      // Transaction argument to cursor is only needed when using the cursor to
722      // write to the database. Read-only cursors do not need a txn pointer.
723      int ret = database.m_db->cursor(batch.txn(), &m_cursor, 0);
724      if (ret != 0) {
725          throw std::runtime_error(STR_INTERNAL_BUG(strprintf("BDB Cursor could not be created. Returned %d", ret)));
726      }
727  }
728  
729  DatabaseCursor::Status BerkeleyCursor::Next(DataStream& ssKey, DataStream& ssValue)
730  {
731      if (m_cursor == nullptr) return Status::FAIL;
732      // Read at cursor
733      SafeDbt datKey(m_key_prefix.data(), m_key_prefix.size());
734      SafeDbt datValue;
735      int ret = -1;
736      if (m_first && !m_key_prefix.empty()) {
737          ret = m_cursor->get(datKey, datValue, DB_SET_RANGE);
738      } else {
739          ret = m_cursor->get(datKey, datValue, DB_NEXT);
740      }
741      m_first = false;
742      if (ret == DB_NOTFOUND) {
743          return Status::DONE;
744      }
745      if (ret != 0) {
746          return Status::FAIL;
747      }
748  
749      Span<const std::byte> raw_key = SpanFromDbt(datKey);
750      if (!m_key_prefix.empty() && std::mismatch(raw_key.begin(), raw_key.end(), m_key_prefix.begin(), m_key_prefix.end()).second != m_key_prefix.end()) {
751          return Status::DONE;
752      }
753  
754      // Convert to streams
755      ssKey.clear();
756      ssKey.write(raw_key);
757      ssValue.clear();
758      ssValue.write(SpanFromDbt(datValue));
759      return Status::MORE;
760  }
761  
762  BerkeleyCursor::~BerkeleyCursor()
763  {
764      if (!m_cursor) return;
765      m_cursor->close();
766      m_cursor = nullptr;
767  }
768  
769  std::unique_ptr<DatabaseCursor> BerkeleyBatch::GetNewCursor()
770  {
771      if (!pdb) return nullptr;
772      return std::make_unique<BerkeleyCursor>(m_database, *this);
773  }
774  
775  std::unique_ptr<DatabaseCursor> BerkeleyBatch::GetNewPrefixCursor(Span<const std::byte> prefix)
776  {
777      if (!pdb) return nullptr;
778      return std::make_unique<BerkeleyCursor>(m_database, *this, prefix);
779  }
780  
781  bool BerkeleyBatch::TxnBegin()
782  {
783      if (!pdb || activeTxn)
784          return false;
785      DbTxn* ptxn = env->TxnBegin(DB_TXN_WRITE_NOSYNC);
786      if (!ptxn)
787          return false;
788      activeTxn = ptxn;
789      return true;
790  }
791  
792  bool BerkeleyBatch::TxnCommit()
793  {
794      if (!pdb || !activeTxn)
795          return false;
796      int ret = activeTxn->commit(0);
797      activeTxn = nullptr;
798      return (ret == 0);
799  }
800  
801  bool BerkeleyBatch::TxnAbort()
802  {
803      if (!pdb || !activeTxn)
804          return false;
805      int ret = activeTxn->abort();
806      activeTxn = nullptr;
807      return (ret == 0);
808  }
809  
810  bool BerkeleyDatabaseSanityCheck()
811  {
812      int major, minor;
813      DbEnv::version(&major, &minor, nullptr);
814  
815      /* If the major version differs, or the minor version of library is *older*
816       * than the header that was compiled against, flag an error.
817       */
818      if (major != DB_VERSION_MAJOR || minor < DB_VERSION_MINOR) {
819          LogPrintf("BerkeleyDB database version conflict: header version is %d.%d, library version is %d.%d\n",
820              DB_VERSION_MAJOR, DB_VERSION_MINOR, major, minor);
821          return false;
822      }
823  
824      return true;
825  }
826  
827  std::string BerkeleyDatabaseVersion()
828  {
829      return DbEnv::version(nullptr, nullptr, nullptr);
830  }
831  
832  bool BerkeleyBatch::ReadKey(DataStream&& key, DataStream& value)
833  {
834      if (!pdb)
835          return false;
836  
837      SafeDbt datKey(key.data(), key.size());
838  
839      SafeDbt datValue;
840      int ret = pdb->get(activeTxn, datKey, datValue, 0);
841      if (ret == 0 && datValue.get_data() != nullptr) {
842          value.clear();
843          value.write(SpanFromDbt(datValue));
844          return true;
845      }
846      return false;
847  }
848  
849  bool BerkeleyBatch::WriteKey(DataStream&& key, DataStream&& value, bool overwrite)
850  {
851      if (!pdb)
852          return false;
853      if (fReadOnly)
854          assert(!"Write called on database in read-only mode");
855  
856      SafeDbt datKey(key.data(), key.size());
857  
858      SafeDbt datValue(value.data(), value.size());
859  
860      int ret = pdb->put(activeTxn, datKey, datValue, (overwrite ? 0 : DB_NOOVERWRITE));
861      return (ret == 0);
862  }
863  
864  bool BerkeleyBatch::EraseKey(DataStream&& key)
865  {
866      if (!pdb)
867          return false;
868      if (fReadOnly)
869          assert(!"Erase called on database in read-only mode");
870  
871      SafeDbt datKey(key.data(), key.size());
872  
873      int ret = pdb->del(activeTxn, datKey, 0);
874      return (ret == 0 || ret == DB_NOTFOUND);
875  }
876  
877  bool BerkeleyBatch::HasKey(DataStream&& key)
878  {
879      if (!pdb)
880          return false;
881  
882      SafeDbt datKey(key.data(), key.size());
883  
884      int ret = pdb->exists(activeTxn, datKey, 0);
885      return ret == 0;
886  }
887  
888  bool BerkeleyBatch::ErasePrefix(Span<const std::byte> prefix)
889  {
890      // Because this function erases records one by one, ensure that it is executed within a txn context.
891      // Otherwise, consistency is at risk; it's possible that certain records are removed while others
892      // remain due to an internal failure during the procedure.
893      // Additionally, the Dbc::del() cursor delete call below would fail without an active transaction.
894      if (!Assume(activeTxn)) return false;
895  
896      auto cursor{std::make_unique<BerkeleyCursor>(m_database, *this)};
897      // const_cast is safe below even though prefix_key is an in/out parameter,
898      // because we are not using the DB_DBT_USERMEM flag, so BDB will allocate
899      // and return a different output data pointer
900      Dbt prefix_key{const_cast<std::byte*>(prefix.data()), static_cast<uint32_t>(prefix.size())}, prefix_value{};
901      int ret{cursor->dbc()->get(&prefix_key, &prefix_value, DB_SET_RANGE)};
902      for (int flag{DB_CURRENT}; ret == 0; flag = DB_NEXT) {
903          SafeDbt key, value;
904          ret = cursor->dbc()->get(key, value, flag);
905          if (ret != 0 || key.get_size() < prefix.size() || memcmp(key.get_data(), prefix.data(), prefix.size()) != 0) break;
906          ret = cursor->dbc()->del(0);
907      }
908      cursor.reset();
909      return ret == 0 || ret == DB_NOTFOUND;
910  }
911  
912  void BerkeleyDatabase::AddRef()
913  {
914      LOCK(cs_db);
915      if (m_refcount < 0) {
916          m_refcount = 1;
917      } else {
918          m_refcount++;
919      }
920  }
921  
922  void BerkeleyDatabase::RemoveRef()
923  {
924      LOCK(cs_db);
925      m_refcount--;
926      if (env) env->m_db_in_use.notify_all();
927  }
928  
929  std::unique_ptr<DatabaseBatch> BerkeleyDatabase::MakeBatch(bool flush_on_close)
930  {
931      return std::make_unique<BerkeleyBatch>(*this, false, flush_on_close);
932  }
933  
934  std::unique_ptr<BerkeleyDatabase> MakeBerkeleyDatabase(const fs::path& path, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error)
935  {
936      fs::path data_file = BDBDataFile(path);
937      std::unique_ptr<BerkeleyDatabase> db;
938      {
939          LOCK(cs_db); // Lock env.m_databases until insert in BerkeleyDatabase constructor
940          fs::path data_filename = data_file.filename();
941          std::shared_ptr<BerkeleyEnvironment> env = GetBerkeleyEnv(data_file.parent_path(), options.use_shared_memory);
942          if (env->m_databases.count(data_filename)) {
943              error = Untranslated(strprintf("Refusing to load database. Data file '%s' is already loaded.", fs::PathToString(env->Directory() / data_filename)));
944              status = DatabaseStatus::FAILED_ALREADY_LOADED;
945              return nullptr;
946          }
947          db = std::make_unique<BerkeleyDatabase>(std::move(env), std::move(data_filename), options);
948      }
949  
950      if (options.verify && !db->Verify(error)) {
951          status = DatabaseStatus::FAILED_VERIFY;
952          return nullptr;
953      }
954  
955      status = DatabaseStatus::SUCCESS;
956      return db;
957  }
958  } // namespace wallet