1  // Copyright (c) 2012-present The Bitcoin Core developers
  2  // Distributed under the MIT software license, see the accompanying
  3  // file COPYING or http://www.opensource.org/licenses/mit-license.php.
  4  
  5  #include <dbwrapper.h>
  6  
  7  #include <leveldb/cache.h>
  8  #include <leveldb/db.h>
  9  #include <leveldb/env.h>
 10  #include <leveldb/filter_policy.h>
 11  #include <leveldb/helpers/memenv/memenv.h>
 12  #include <leveldb/iterator.h>
 13  #include <leveldb/options.h>
 14  #include <leveldb/slice.h>
 15  #include <leveldb/status.h>
 16  #include <leveldb/write_batch.h>
 17  #include <logging.h>
 18  #include <random.h>
 19  #include <serialize.h>
 20  #include <span.h>
 21  #include <streams.h>
 22  #include <util/byte_units.h>
 23  #include <util/fs.h>
 24  #include <util/fs_helpers.h>
 25  #include <util/log.h>
 26  #include <util/obfuscation.h>
 27  #include <util/strencodings.h>
 28  
 29  #include <algorithm>
 30  #include <cassert>
 31  #include <cstdarg>
 32  #include <cstdint>
 33  #include <cstdio>
 34  #include <memory>
 35  #include <optional>
 36  #include <utility>
 37  
 38  static auto CharCast(const std::byte* data) { return reinterpret_cast<const char*>(data); }
 39  
 40  bool DestroyDB(const std::string& path_str)
 41  {
 42      return leveldb::DestroyDB(path_str, {}).ok();
 43  }
 44  
 45  /** Handle database error by throwing dbwrapper_error exception.
 46   */
 47  static void HandleError(const leveldb::Status& status)
 48  {
 49      if (status.ok())
 50          return;
 51      const std::string errmsg = "Fatal LevelDB error: " + status.ToString();
 52      LogError("%s", errmsg);
 53      LogInfo("You can use -debug=leveldb to get more complete diagnostic messages");
 54      throw dbwrapper_error(errmsg);
 55  }
 56  
 57  class CBitcoinLevelDBLogger : public leveldb::Logger {
 58  public:
 59      // This code is adapted from posix_logger.h, which is why it is using vsprintf.
 60      // Please do not do this in normal code
 61      void Logv(const char * format, va_list ap) override {
 62              if (!LogAcceptCategory(BCLog::LEVELDB, util::log::Level::Debug)) {
 63                  return;
 64              }
 65              char buffer[500];
 66              for (int iter = 0; iter < 2; iter++) {
 67                  char* base;
 68                  int bufsize;
 69                  if (iter == 0) {
 70                      bufsize = sizeof(buffer);
 71                      base = buffer;
 72                  }
 73                  else {
 74                      bufsize = 30000;
 75                      base = new char[bufsize];
 76                  }
 77                  char* p = base;
 78                  char* limit = base + bufsize;
 79  
 80                  // Print the message
 81                  if (p < limit) {
 82                      va_list backup_ap;
 83                      va_copy(backup_ap, ap);
 84                      // Do not use vsnprintf elsewhere in bitcoin source code, see above.
 85                      p += vsnprintf(p, limit - p, format, backup_ap);
 86                      va_end(backup_ap);
 87                  }
 88  
 89                  // Truncate to available space if necessary
 90                  if (p >= limit) {
 91                      if (iter == 0) {
 92                          continue;       // Try again with larger buffer
 93                      }
 94                      else {
 95                          p = limit - 1;
 96                      }
 97                  }
 98  
 99                  // Add newline if necessary
100                  if (p == base || p[-1] != '\n') {
101                      *p++ = '\n';
102                  }
103  
104                  assert(p <= limit);
105                  base[std::min(bufsize - 1, (int)(p - base))] = '\0';
106                  LogDebug(BCLog::LEVELDB, "%s\n", util::RemoveSuffixView(base, "\n"));
107                  if (base != buffer) {
108                      delete[] base;
109                  }
110                  break;
111              }
112      }
113  };
114  
115  static void SetMaxOpenFiles(leveldb::Options *options) {
116      // On most platforms the default setting of max_open_files (which is 1000)
117      // is optimal. On Windows using a large file count is OK because the handles
118      // do not interfere with select() loops. On 64-bit Unix hosts this value is
119      // also OK, because up to that amount LevelDB will use an mmap
120      // implementation that does not use extra file descriptors (the fds are
121      // closed after being mmap'ed).
122      //
123      // Increasing the value beyond the default is dangerous because LevelDB will
124      // fall back to a non-mmap implementation when the file count is too large.
125      // On 32-bit Unix host we should decrease the value because the handles use
126      // up real fds, and we want to avoid fd exhaustion issues.
127      //
128      // See PR #12495 for further discussion.
129  
130      int default_open_files = options->max_open_files;
131  #ifndef WIN32
132      if (sizeof(void*) < 8) {
133          options->max_open_files = 64;
134      }
135  #endif
136      LogDebug(BCLog::LEVELDB, "LevelDB using max_open_files=%d (default=%d)\n",
137               options->max_open_files, default_open_files);
138  }
139  
140  static leveldb::Options GetOptions(size_t nCacheSize)
141  {
142      leveldb::Options options;
143      options.block_cache = leveldb::NewLRUCache(nCacheSize / 2);
144      options.write_buffer_size = nCacheSize / 4; // up to two write buffers may be held in memory simultaneously
145      options.filter_policy = leveldb::NewBloomFilterPolicy(10);
146      options.compression = leveldb::kNoCompression;
147      options.info_log = new CBitcoinLevelDBLogger();
148      if (leveldb::kMajorVersion > 1 || (leveldb::kMajorVersion == 1 && leveldb::kMinorVersion >= 16)) {
149          // LevelDB versions before 1.16 consider short writes to be corruption. Only trigger error
150          // on corruption in later versions.
151          options.paranoid_checks = true;
152      }
153      options.max_file_size = std::max(options.max_file_size, DBWRAPPER_MAX_FILE_SIZE);
154      SetMaxOpenFiles(&options);
155      return options;
156  }
157  
158  struct CDBBatch::WriteBatchImpl {
159      leveldb::WriteBatch batch;
160  };
161  
162  CDBBatch::CDBBatch(const CDBWrapper& _parent)
163      : parent{_parent},
164        m_impl_batch{std::make_unique<CDBBatch::WriteBatchImpl>()}
165  {
166      Clear();
167  };
168  
169  CDBBatch::~CDBBatch() = default;
170  
171  void CDBBatch::Clear()
172  {
173      m_impl_batch->batch.Clear();
174  }
175  
176  void CDBBatch::WriteImpl(std::span<const std::byte> key, DataStream& ssValue)
177  {
178      leveldb::Slice slKey(CharCast(key.data()), key.size());
179      dbwrapper_private::GetObfuscation(parent)(ssValue);
180      leveldb::Slice slValue(CharCast(ssValue.data()), ssValue.size());
181      m_impl_batch->batch.Put(slKey, slValue);
182  }
183  
184  void CDBBatch::EraseImpl(std::span<const std::byte> key)
185  {
186      leveldb::Slice slKey(CharCast(key.data()), key.size());
187      m_impl_batch->batch.Delete(slKey);
188  }
189  
190  size_t CDBBatch::ApproximateSize() const
191  {
192      return m_impl_batch->batch.ApproximateSize();
193  }
194  
195  struct LevelDBContext {
196      //! custom environment this database is using (may be nullptr in case of default environment)
197      leveldb::Env* penv;
198  
199      //! database options used
200      leveldb::Options options;
201  
202      //! options used when reading from the database
203      leveldb::ReadOptions readoptions;
204  
205      //! options used when iterating over values of the database
206      leveldb::ReadOptions iteroptions;
207  
208      //! options used when writing to the database
209      leveldb::WriteOptions writeoptions;
210  
211      //! options used when sync writing to the database
212      leveldb::WriteOptions syncoptions;
213  
214      //! the database itself
215      leveldb::DB* pdb;
216  };
217  
218  CDBWrapper::CDBWrapper(const DBParams& params)
219      : m_db_context{std::make_unique<LevelDBContext>()}, m_name{fs::PathToString(params.path.stem())}
220  {
221      DBContext().penv = nullptr;
222      DBContext().readoptions.verify_checksums = true;
223      DBContext().iteroptions.verify_checksums = true;
224      DBContext().iteroptions.fill_cache = false;
225      DBContext().syncoptions.sync = true;
226      DBContext().options = GetOptions(params.cache_bytes);
227      DBContext().options.create_if_missing = true;
228      if (params.memory_only) {
229          DBContext().penv = leveldb::NewMemEnv(leveldb::Env::Default());
230          DBContext().options.env = DBContext().penv;
231      } else {
232          if (params.wipe_data) {
233              LogInfo("Wiping LevelDB in %s", fs::PathToString(params.path));
234              leveldb::Status result = leveldb::DestroyDB(fs::PathToString(params.path), DBContext().options);
235              HandleError(result);
236          }
237          TryCreateDirectories(params.path);
238          LogInfo("Opening LevelDB in %s", fs::PathToString(params.path));
239      }
240      // PathToString() return value is safe to pass to leveldb open function,
241      // because on POSIX leveldb passes the byte string directly to ::open(), and
242      // on Windows it converts from UTF-8 to UTF-16 before calling ::CreateFileW
243      // (see env_posix.cc and env_windows.cc).
244      leveldb::Status status = leveldb::DB::Open(DBContext().options, fs::PathToString(params.path), &DBContext().pdb);
245      HandleError(status);
246      LogInfo("Opened LevelDB successfully");
247  
248      if (params.options.force_compact) {
249          LogInfo("Starting database compaction of %s", fs::PathToString(params.path));
250          DBContext().pdb->CompactRange(nullptr, nullptr);
251          LogInfo("Finished database compaction of %s", fs::PathToString(params.path));
252      }
253  
254      if (!Read(OBFUSCATION_KEY, m_obfuscation) && params.obfuscate && IsEmpty()) {
255          // Generate and write the new obfuscation key.
256          const Obfuscation obfuscation{FastRandomContext{}.randbytes<Obfuscation::KEY_SIZE>()};
257          assert(!m_obfuscation); // Make sure the key is written without obfuscation.
258          Write(OBFUSCATION_KEY, obfuscation);
259          m_obfuscation = obfuscation;
260          LogInfo("Wrote new obfuscation key for %s: %s", fs::PathToString(params.path), m_obfuscation.HexKey());
261      }
262      LogInfo("Using obfuscation key for %s: %s", fs::PathToString(params.path), m_obfuscation.HexKey());
263  }
264  
265  CDBWrapper::~CDBWrapper()
266  {
267      delete DBContext().pdb;
268      DBContext().pdb = nullptr;
269      delete DBContext().options.filter_policy;
270      DBContext().options.filter_policy = nullptr;
271      delete DBContext().options.info_log;
272      DBContext().options.info_log = nullptr;
273      delete DBContext().options.block_cache;
274      DBContext().options.block_cache = nullptr;
275      delete DBContext().penv;
276      DBContext().options.env = nullptr;
277  }
278  
279  void CDBWrapper::WriteBatch(CDBBatch& batch, bool fSync)
280  {
281      const bool log_memory = LogAcceptCategory(BCLog::LEVELDB, util::log::Level::Debug);
282      double mem_before = 0;
283      if (log_memory) {
284          mem_before = DynamicMemoryUsage() / double(1_MiB);
285      }
286      leveldb::Status status = DBContext().pdb->Write(fSync ? DBContext().syncoptions : DBContext().writeoptions, &batch.m_impl_batch->batch);
287      HandleError(status);
288      if (log_memory) {
289          double mem_after{DynamicMemoryUsage() / double(1_MiB)};
290          LogDebug(BCLog::LEVELDB, "WriteBatch memory usage: db=%s, before=%.1fMiB, after=%.1fMiB\n",
291                   m_name, mem_before, mem_after);
292      }
293  }
294  
295  size_t CDBWrapper::DynamicMemoryUsage() const
296  {
297      std::string memory;
298      std::optional<size_t> parsed;
299      if (!DBContext().pdb->GetProperty("leveldb.approximate-memory-usage", &memory) || !(parsed = ToIntegral<size_t>(memory))) {
300          LogDebug(BCLog::LEVELDB, "Failed to get approximate-memory-usage property\n");
301          return 0;
302      }
303      return parsed.value();
304  }
305  
306  std::optional<std::string> CDBWrapper::ReadImpl(std::span<const std::byte> key) const
307  {
308      leveldb::Slice slKey(CharCast(key.data()), key.size());
309      std::string strValue;
310      leveldb::Status status = DBContext().pdb->Get(DBContext().readoptions, slKey, &strValue);
311      if (!status.ok()) {
312          if (status.IsNotFound())
313              return std::nullopt;
314          LogError("LevelDB read failure: %s", status.ToString());
315          HandleError(status);
316      }
317      return strValue;
318  }
319  
320  bool CDBWrapper::ExistsImpl(std::span<const std::byte> key) const
321  {
322      leveldb::Slice slKey(CharCast(key.data()), key.size());
323  
324      std::string strValue;
325      leveldb::Status status = DBContext().pdb->Get(DBContext().readoptions, slKey, &strValue);
326      if (!status.ok()) {
327          if (status.IsNotFound())
328              return false;
329          LogError("LevelDB read failure: %s", status.ToString());
330          HandleError(status);
331      }
332      return true;
333  }
334  
335  size_t CDBWrapper::EstimateSizeImpl(std::span<const std::byte> key1, std::span<const std::byte> key2) const
336  {
337      leveldb::Slice slKey1(CharCast(key1.data()), key1.size());
338      leveldb::Slice slKey2(CharCast(key2.data()), key2.size());
339      uint64_t size = 0;
340      leveldb::Range range(slKey1, slKey2);
341      DBContext().pdb->GetApproximateSizes(&range, 1, &size);
342      return size;
343  }
344  
345  bool CDBWrapper::IsEmpty()
346  {
347      std::unique_ptr<CDBIterator> it(NewIterator());
348      it->SeekToFirst();
349      return !(it->Valid());
350  }
351  
352  struct CDBIterator::IteratorImpl {
353      const std::unique_ptr<leveldb::Iterator> iter;
354  
355      explicit IteratorImpl(leveldb::Iterator* _iter) : iter{_iter} {}
356  };
357  
358  CDBIterator::CDBIterator(const CDBWrapper& _parent, std::unique_ptr<IteratorImpl> _piter) : parent(_parent),
359                                                                                              m_impl_iter(std::move(_piter)) {}
360  
361  CDBIterator* CDBWrapper::NewIterator()
362  {
363      return new CDBIterator{*this, std::make_unique<CDBIterator::IteratorImpl>(DBContext().pdb->NewIterator(DBContext().iteroptions))};
364  }
365  
366  void CDBIterator::SeekImpl(std::span<const std::byte> key)
367  {
368      leveldb::Slice slKey(CharCast(key.data()), key.size());
369      m_impl_iter->iter->Seek(slKey);
370  }
371  
372  std::span<const std::byte> CDBIterator::GetKeyImpl() const
373  {
374      // The returned span borrows from the current iterator entry and is only
375      // valid until the iterator is advanced.
376      return MakeByteSpan(m_impl_iter->iter->key());
377  }
378  
379  std::span<const std::byte> CDBIterator::GetValueImpl() const
380  {
381      return MakeByteSpan(m_impl_iter->iter->value());
382  }
383  
384  CDBIterator::~CDBIterator() = default;
385  bool CDBIterator::Valid() const { return m_impl_iter->iter->Valid(); }
386  void CDBIterator::SeekToFirst() { m_impl_iter->iter->SeekToFirst(); }
387  void CDBIterator::Next() { m_impl_iter->iter->Next(); }
388  
389  namespace dbwrapper_private {
390  
391  const Obfuscation& GetObfuscation(const CDBWrapper& w)
392  {
393      return w.m_obfuscation;
394  }
395  
396  } // namespace dbwrapper_private