1  // Copyright (c) 2012-present The Bitcoin Core developers
  2  // Distributed under the MIT software license, see the accompanying
  3  // file COPYING or http://www.opensource.org/licenses/mit-license.php.
  4  
  5  #include <dbwrapper.h>
  6  
  7  #include <logging.h>
  8  #include <random.h>
  9  #include <serialize.h>
 10  #include <span.h>
 11  #include <streams.h>
 12  #include <util/fs.h>
 13  #include <util/fs_helpers.h>
 14  #include <util/obfuscation.h>
 15  #include <util/strencodings.h>
 16  
 17  #include <algorithm>
 18  #include <cassert>
 19  #include <cstdarg>
 20  #include <cstdint>
 21  #include <cstdio>
 22  #include <leveldb/cache.h>
 23  #include <leveldb/db.h>
 24  #include <leveldb/env.h>
 25  #include <leveldb/filter_policy.h>
 26  #include <leveldb/helpers/memenv/memenv.h>
 27  #include <leveldb/iterator.h>
 28  #include <leveldb/options.h>
 29  #include <leveldb/slice.h>
 30  #include <leveldb/status.h>
 31  #include <leveldb/write_batch.h>
 32  #include <memory>
 33  #include <optional>
 34  #include <utility>
 35  
 36  static auto CharCast(const std::byte* data) { return reinterpret_cast<const char*>(data); }
 37  
 38  bool DestroyDB(const std::string& path_str)
 39  {
 40      return leveldb::DestroyDB(path_str, {}).ok();
 41  }
 42  
 43  /** Handle database error by throwing dbwrapper_error exception.
 44   */
 45  static void HandleError(const leveldb::Status& status)
 46  {
 47      if (status.ok())
 48          return;
 49      const std::string errmsg = "Fatal LevelDB error: " + status.ToString();
 50      LogError("%s", errmsg);
 51      LogInfo("You can use -debug=leveldb to get more complete diagnostic messages");
 52      throw dbwrapper_error(errmsg);
 53  }
 54  
 55  class CBitcoinLevelDBLogger : public leveldb::Logger {
 56  public:
 57      // This code is adapted from posix_logger.h, which is why it is using vsprintf.
 58      // Please do not do this in normal code
 59      void Logv(const char * format, va_list ap) override {
 60              if (!LogAcceptCategory(BCLog::LEVELDB, BCLog::Level::Debug)) {
 61                  return;
 62              }
 63              char buffer[500];
 64              for (int iter = 0; iter < 2; iter++) {
 65                  char* base;
 66                  int bufsize;
 67                  if (iter == 0) {
 68                      bufsize = sizeof(buffer);
 69                      base = buffer;
 70                  }
 71                  else {
 72                      bufsize = 30000;
 73                      base = new char[bufsize];
 74                  }
 75                  char* p = base;
 76                  char* limit = base + bufsize;
 77  
 78                  // Print the message
 79                  if (p < limit) {
 80                      va_list backup_ap;
 81                      va_copy(backup_ap, ap);
 82                      // Do not use vsnprintf elsewhere in bitcoin source code, see above.
 83                      p += vsnprintf(p, limit - p, format, backup_ap);
 84                      va_end(backup_ap);
 85                  }
 86  
 87                  // Truncate to available space if necessary
 88                  if (p >= limit) {
 89                      if (iter == 0) {
 90                          continue;       // Try again with larger buffer
 91                      }
 92                      else {
 93                          p = limit - 1;
 94                      }
 95                  }
 96  
 97                  // Add newline if necessary
 98                  if (p == base || p[-1] != '\n') {
 99                      *p++ = '\n';
100                  }
101  
102                  assert(p <= limit);
103                  base[std::min(bufsize - 1, (int)(p - base))] = '\0';
104                  LogDebug(BCLog::LEVELDB, "%s\n", util::RemoveSuffixView(base, "\n"));
105                  if (base != buffer) {
106                      delete[] base;
107                  }
108                  break;
109              }
110      }
111  };
112  
113  static void SetMaxOpenFiles(leveldb::Options *options) {
114      // On most platforms the default setting of max_open_files (which is 1000)
115      // is optimal. On Windows using a large file count is OK because the handles
116      // do not interfere with select() loops. On 64-bit Unix hosts this value is
117      // also OK, because up to that amount LevelDB will use an mmap
118      // implementation that does not use extra file descriptors (the fds are
119      // closed after being mmap'ed).
120      //
121      // Increasing the value beyond the default is dangerous because LevelDB will
122      // fall back to a non-mmap implementation when the file count is too large.
123      // On 32-bit Unix host we should decrease the value because the handles use
124      // up real fds, and we want to avoid fd exhaustion issues.
125      //
126      // See PR #12495 for further discussion.
127  
128      int default_open_files = options->max_open_files;
129  #ifndef WIN32
130      if (sizeof(void*) < 8) {
131          options->max_open_files = 64;
132      }
133  #endif
134      LogDebug(BCLog::LEVELDB, "LevelDB using max_open_files=%d (default=%d)\n",
135               options->max_open_files, default_open_files);
136  }
137  
138  static leveldb::Options GetOptions(size_t nCacheSize)
139  {
140      leveldb::Options options;
141      options.block_cache = leveldb::NewLRUCache(nCacheSize / 2);
142      options.write_buffer_size = nCacheSize / 4; // up to two write buffers may be held in memory simultaneously
143      options.filter_policy = leveldb::NewBloomFilterPolicy(10);
144      options.compression = leveldb::kNoCompression;
145      options.info_log = new CBitcoinLevelDBLogger();
146      if (leveldb::kMajorVersion > 1 || (leveldb::kMajorVersion == 1 && leveldb::kMinorVersion >= 16)) {
147          // LevelDB versions before 1.16 consider short writes to be corruption. Only trigger error
148          // on corruption in later versions.
149          options.paranoid_checks = true;
150      }
151      options.max_file_size = std::max(options.max_file_size, DBWRAPPER_MAX_FILE_SIZE);
152      SetMaxOpenFiles(&options);
153      return options;
154  }
155  
156  struct CDBBatch::WriteBatchImpl {
157      leveldb::WriteBatch batch;
158  };
159  
160  CDBBatch::CDBBatch(const CDBWrapper& _parent)
161      : parent{_parent},
162        m_impl_batch{std::make_unique<CDBBatch::WriteBatchImpl>()}
163  {
164      Clear();
165  };
166  
167  CDBBatch::~CDBBatch() = default;
168  
169  void CDBBatch::Clear()
170  {
171      m_impl_batch->batch.Clear();
172  }
173  
174  void CDBBatch::WriteImpl(std::span<const std::byte> key, DataStream& ssValue)
175  {
176      leveldb::Slice slKey(CharCast(key.data()), key.size());
177      dbwrapper_private::GetObfuscation(parent)(ssValue);
178      leveldb::Slice slValue(CharCast(ssValue.data()), ssValue.size());
179      m_impl_batch->batch.Put(slKey, slValue);
180  }
181  
182  void CDBBatch::EraseImpl(std::span<const std::byte> key)
183  {
184      leveldb::Slice slKey(CharCast(key.data()), key.size());
185      m_impl_batch->batch.Delete(slKey);
186  }
187  
188  size_t CDBBatch::ApproximateSize() const
189  {
190      return m_impl_batch->batch.ApproximateSize();
191  }
192  
193  struct LevelDBContext {
194      //! custom environment this database is using (may be nullptr in case of default environment)
195      leveldb::Env* penv;
196  
197      //! database options used
198      leveldb::Options options;
199  
200      //! options used when reading from the database
201      leveldb::ReadOptions readoptions;
202  
203      //! options used when iterating over values of the database
204      leveldb::ReadOptions iteroptions;
205  
206      //! options used when writing to the database
207      leveldb::WriteOptions writeoptions;
208  
209      //! options used when sync writing to the database
210      leveldb::WriteOptions syncoptions;
211  
212      //! the database itself
213      leveldb::DB* pdb;
214  };
215  
216  CDBWrapper::CDBWrapper(const DBParams& params)
217      : m_db_context{std::make_unique<LevelDBContext>()}, m_name{fs::PathToString(params.path.stem())}, m_path{params.path}, m_is_memory{params.memory_only}
218  {
219      DBContext().penv = nullptr;
220      DBContext().readoptions.verify_checksums = true;
221      DBContext().iteroptions.verify_checksums = true;
222      DBContext().iteroptions.fill_cache = false;
223      DBContext().syncoptions.sync = true;
224      DBContext().options = GetOptions(params.cache_bytes);
225      DBContext().options.create_if_missing = true;
226      if (params.memory_only) {
227          DBContext().penv = leveldb::NewMemEnv(leveldb::Env::Default());
228          DBContext().options.env = DBContext().penv;
229      } else {
230          if (params.wipe_data) {
231              LogInfo("Wiping LevelDB in %s", fs::PathToString(params.path));
232              leveldb::Status result = leveldb::DestroyDB(fs::PathToString(params.path), DBContext().options);
233              HandleError(result);
234          }
235          TryCreateDirectories(params.path);
236          LogInfo("Opening LevelDB in %s", fs::PathToString(params.path));
237      }
238      // PathToString() return value is safe to pass to leveldb open function,
239      // because on POSIX leveldb passes the byte string directly to ::open(), and
240      // on Windows it converts from UTF-8 to UTF-16 before calling ::CreateFileW
241      // (see env_posix.cc and env_windows.cc).
242      leveldb::Status status = leveldb::DB::Open(DBContext().options, fs::PathToString(params.path), &DBContext().pdb);
243      HandleError(status);
244      LogInfo("Opened LevelDB successfully");
245  
246      if (params.options.force_compact) {
247          LogInfo("Starting database compaction of %s", fs::PathToString(params.path));
248          DBContext().pdb->CompactRange(nullptr, nullptr);
249          LogInfo("Finished database compaction of %s", fs::PathToString(params.path));
250      }
251  
252      if (!Read(OBFUSCATION_KEY, m_obfuscation) && params.obfuscate && IsEmpty()) {
253          // Generate and write the new obfuscation key.
254          const Obfuscation obfuscation{FastRandomContext{}.randbytes<Obfuscation::KEY_SIZE>()};
255          assert(!m_obfuscation); // Make sure the key is written without obfuscation.
256          Write(OBFUSCATION_KEY, obfuscation);
257          m_obfuscation = obfuscation;
258          LogInfo("Wrote new obfuscation key for %s: %s", fs::PathToString(params.path), m_obfuscation.HexKey());
259      }
260      LogInfo("Using obfuscation key for %s: %s", fs::PathToString(params.path), m_obfuscation.HexKey());
261  }
262  
263  CDBWrapper::~CDBWrapper()
264  {
265      delete DBContext().pdb;
266      DBContext().pdb = nullptr;
267      delete DBContext().options.filter_policy;
268      DBContext().options.filter_policy = nullptr;
269      delete DBContext().options.info_log;
270      DBContext().options.info_log = nullptr;
271      delete DBContext().options.block_cache;
272      DBContext().options.block_cache = nullptr;
273      delete DBContext().penv;
274      DBContext().options.env = nullptr;
275  }
276  
277  void CDBWrapper::WriteBatch(CDBBatch& batch, bool fSync)
278  {
279      const bool log_memory = LogAcceptCategory(BCLog::LEVELDB, BCLog::Level::Debug);
280      double mem_before = 0;
281      if (log_memory) {
282          mem_before = DynamicMemoryUsage() / 1024.0 / 1024;
283      }
284      leveldb::Status status = DBContext().pdb->Write(fSync ? DBContext().syncoptions : DBContext().writeoptions, &batch.m_impl_batch->batch);
285      HandleError(status);
286      if (log_memory) {
287          double mem_after = DynamicMemoryUsage() / 1024.0 / 1024;
288          LogDebug(BCLog::LEVELDB, "WriteBatch memory usage: db=%s, before=%.1fMiB, after=%.1fMiB\n",
289                   m_name, mem_before, mem_after);
290      }
291  }
292  
293  size_t CDBWrapper::DynamicMemoryUsage() const
294  {
295      std::string memory;
296      std::optional<size_t> parsed;
297      if (!DBContext().pdb->GetProperty("leveldb.approximate-memory-usage", &memory) || !(parsed = ToIntegral<size_t>(memory))) {
298          LogDebug(BCLog::LEVELDB, "Failed to get approximate-memory-usage property\n");
299          return 0;
300      }
301      return parsed.value();
302  }
303  
304  std::optional<std::string> CDBWrapper::ReadImpl(std::span<const std::byte> key) const
305  {
306      leveldb::Slice slKey(CharCast(key.data()), key.size());
307      std::string strValue;
308      leveldb::Status status = DBContext().pdb->Get(DBContext().readoptions, slKey, &strValue);
309      if (!status.ok()) {
310          if (status.IsNotFound())
311              return std::nullopt;
312          LogError("LevelDB read failure: %s", status.ToString());
313          HandleError(status);
314      }
315      return strValue;
316  }
317  
318  bool CDBWrapper::ExistsImpl(std::span<const std::byte> key) const
319  {
320      leveldb::Slice slKey(CharCast(key.data()), key.size());
321  
322      std::string strValue;
323      leveldb::Status status = DBContext().pdb->Get(DBContext().readoptions, slKey, &strValue);
324      if (!status.ok()) {
325          if (status.IsNotFound())
326              return false;
327          LogError("LevelDB read failure: %s", status.ToString());
328          HandleError(status);
329      }
330      return true;
331  }
332  
333  size_t CDBWrapper::EstimateSizeImpl(std::span<const std::byte> key1, std::span<const std::byte> key2) const
334  {
335      leveldb::Slice slKey1(CharCast(key1.data()), key1.size());
336      leveldb::Slice slKey2(CharCast(key2.data()), key2.size());
337      uint64_t size = 0;
338      leveldb::Range range(slKey1, slKey2);
339      DBContext().pdb->GetApproximateSizes(&range, 1, &size);
340      return size;
341  }
342  
343  bool CDBWrapper::IsEmpty()
344  {
345      std::unique_ptr<CDBIterator> it(NewIterator());
346      it->SeekToFirst();
347      return !(it->Valid());
348  }
349  
350  struct CDBIterator::IteratorImpl {
351      const std::unique_ptr<leveldb::Iterator> iter;
352  
353      explicit IteratorImpl(leveldb::Iterator* _iter) : iter{_iter} {}
354  };
355  
356  CDBIterator::CDBIterator(const CDBWrapper& _parent, std::unique_ptr<IteratorImpl> _piter) : parent(_parent),
357                                                                                              m_impl_iter(std::move(_piter)) {}
358  
359  CDBIterator* CDBWrapper::NewIterator()
360  {
361      return new CDBIterator{*this, std::make_unique<CDBIterator::IteratorImpl>(DBContext().pdb->NewIterator(DBContext().iteroptions))};
362  }
363  
364  void CDBIterator::SeekImpl(std::span<const std::byte> key)
365  {
366      leveldb::Slice slKey(CharCast(key.data()), key.size());
367      m_impl_iter->iter->Seek(slKey);
368  }
369  
370  std::span<const std::byte> CDBIterator::GetKeyImpl() const
371  {
372      return MakeByteSpan(m_impl_iter->iter->key());
373  }
374  
375  std::span<const std::byte> CDBIterator::GetValueImpl() const
376  {
377      return MakeByteSpan(m_impl_iter->iter->value());
378  }
379  
380  CDBIterator::~CDBIterator() = default;
381  bool CDBIterator::Valid() const { return m_impl_iter->iter->Valid(); }
382  void CDBIterator::SeekToFirst() { m_impl_iter->iter->SeekToFirst(); }
383  void CDBIterator::Next() { m_impl_iter->iter->Next(); }
384  
385  namespace dbwrapper_private {
386  
387  const Obfuscation& GetObfuscation(const CDBWrapper& w)
388  {
389      return w.m_obfuscation;
390  }
391  
392  } // namespace dbwrapper_private