1  // Copyright (c) 2012-present The Bitcoin Core developers
  2  // Distributed under the MIT software license, see the accompanying
  3  // file COPYING or http://www.opensource.org/licenses/mit-license.php.
  4  
  5  #include <dbwrapper.h>
  6  
  7  #include <leveldb/cache.h>
  8  #include <leveldb/db.h>
  9  #include <leveldb/env.h>
 10  #include <leveldb/filter_policy.h>
 11  #include <leveldb/helpers/memenv/memenv.h>
 12  #include <leveldb/iterator.h>
 13  #include <leveldb/options.h>
 14  #include <leveldb/slice.h>
 15  #include <leveldb/status.h>
 16  #include <leveldb/write_batch.h>
 17  #include <logging.h>
 18  #include <random.h>
 19  #include <serialize.h>
 20  #include <span.h>
 21  #include <streams.h>
 22  #include <util/fs.h>
 23  #include <util/fs_helpers.h>
 24  #include <util/log.h>
 25  #include <util/obfuscation.h>
 26  #include <util/strencodings.h>
 27  
 28  #include <algorithm>
 29  #include <cassert>
 30  #include <cstdarg>
 31  #include <cstdint>
 32  #include <cstdio>
 33  #include <memory>
 34  #include <optional>
 35  #include <utility>
 36  
 37  static auto CharCast(const std::byte* data) { return reinterpret_cast<const char*>(data); }
 38  
 39  bool DestroyDB(const std::string& path_str)
 40  {
 41      return leveldb::DestroyDB(path_str, {}).ok();
 42  }
 43  
 44  /** Handle database error by throwing dbwrapper_error exception.
 45   */
 46  static void HandleError(const leveldb::Status& status)
 47  {
 48      if (status.ok())
 49          return;
 50      const std::string errmsg = "Fatal LevelDB error: " + status.ToString();
 51      LogError("%s", errmsg);
 52      LogInfo("You can use -debug=leveldb to get more complete diagnostic messages");
 53      throw dbwrapper_error(errmsg);
 54  }
 55  
 56  class CBitcoinLevelDBLogger : public leveldb::Logger {
 57  public:
 58      // This code is adapted from posix_logger.h, which is why it is using vsprintf.
 59      // Please do not do this in normal code
 60      void Logv(const char * format, va_list ap) override {
 61              if (!LogAcceptCategory(BCLog::LEVELDB, util::log::Level::Debug)) {
 62                  return;
 63              }
 64              char buffer[500];
 65              for (int iter = 0; iter < 2; iter++) {
 66                  char* base;
 67                  int bufsize;
 68                  if (iter == 0) {
 69                      bufsize = sizeof(buffer);
 70                      base = buffer;
 71                  }
 72                  else {
 73                      bufsize = 30000;
 74                      base = new char[bufsize];
 75                  }
 76                  char* p = base;
 77                  char* limit = base + bufsize;
 78  
 79                  // Print the message
 80                  if (p < limit) {
 81                      va_list backup_ap;
 82                      va_copy(backup_ap, ap);
 83                      // Do not use vsnprintf elsewhere in bitcoin source code, see above.
 84                      p += vsnprintf(p, limit - p, format, backup_ap);
 85                      va_end(backup_ap);
 86                  }
 87  
 88                  // Truncate to available space if necessary
 89                  if (p >= limit) {
 90                      if (iter == 0) {
 91                          continue;       // Try again with larger buffer
 92                      }
 93                      else {
 94                          p = limit - 1;
 95                      }
 96                  }
 97  
 98                  // Add newline if necessary
 99                  if (p == base || p[-1] != '\n') {
100                      *p++ = '\n';
101                  }
102  
103                  assert(p <= limit);
104                  base[std::min(bufsize - 1, (int)(p - base))] = '\0';
105                  LogDebug(BCLog::LEVELDB, "%s\n", util::RemoveSuffixView(base, "\n"));
106                  if (base != buffer) {
107                      delete[] base;
108                  }
109                  break;
110              }
111      }
112  };
113  
114  static void SetMaxOpenFiles(leveldb::Options *options) {
115      // On most platforms the default setting of max_open_files (which is 1000)
116      // is optimal. On Windows using a large file count is OK because the handles
117      // do not interfere with select() loops. On 64-bit Unix hosts this value is
118      // also OK, because up to that amount LevelDB will use an mmap
119      // implementation that does not use extra file descriptors (the fds are
120      // closed after being mmap'ed).
121      //
122      // Increasing the value beyond the default is dangerous because LevelDB will
123      // fall back to a non-mmap implementation when the file count is too large.
124      // On 32-bit Unix host we should decrease the value because the handles use
125      // up real fds, and we want to avoid fd exhaustion issues.
126      //
127      // See PR #12495 for further discussion.
128  
129      int default_open_files = options->max_open_files;
130  #ifndef WIN32
131      if (sizeof(void*) < 8) {
132          options->max_open_files = 64;
133      }
134  #endif
135      LogDebug(BCLog::LEVELDB, "LevelDB using max_open_files=%d (default=%d)\n",
136               options->max_open_files, default_open_files);
137  }
138  
139  static leveldb::Options GetOptions(size_t nCacheSize)
140  {
141      leveldb::Options options;
142      options.block_cache = leveldb::NewLRUCache(nCacheSize / 2);
143      options.write_buffer_size = nCacheSize / 4; // up to two write buffers may be held in memory simultaneously
144      options.filter_policy = leveldb::NewBloomFilterPolicy(10);
145      options.compression = leveldb::kNoCompression;
146      options.info_log = new CBitcoinLevelDBLogger();
147      if (leveldb::kMajorVersion > 1 || (leveldb::kMajorVersion == 1 && leveldb::kMinorVersion >= 16)) {
148          // LevelDB versions before 1.16 consider short writes to be corruption. Only trigger error
149          // on corruption in later versions.
150          options.paranoid_checks = true;
151      }
152      options.max_file_size = std::max(options.max_file_size, DBWRAPPER_MAX_FILE_SIZE);
153      SetMaxOpenFiles(&options);
154      return options;
155  }
156  
157  struct CDBBatch::WriteBatchImpl {
158      leveldb::WriteBatch batch;
159  };
160  
161  CDBBatch::CDBBatch(const CDBWrapper& _parent)
162      : parent{_parent},
163        m_impl_batch{std::make_unique<CDBBatch::WriteBatchImpl>()}
164  {
165      Clear();
166  };
167  
168  CDBBatch::~CDBBatch() = default;
169  
170  void CDBBatch::Clear()
171  {
172      m_impl_batch->batch.Clear();
173  }
174  
175  void CDBBatch::WriteImpl(std::span<const std::byte> key, DataStream& ssValue)
176  {
177      leveldb::Slice slKey(CharCast(key.data()), key.size());
178      dbwrapper_private::GetObfuscation(parent)(ssValue);
179      leveldb::Slice slValue(CharCast(ssValue.data()), ssValue.size());
180      m_impl_batch->batch.Put(slKey, slValue);
181  }
182  
183  void CDBBatch::EraseImpl(std::span<const std::byte> key)
184  {
185      leveldb::Slice slKey(CharCast(key.data()), key.size());
186      m_impl_batch->batch.Delete(slKey);
187  }
188  
189  size_t CDBBatch::ApproximateSize() const
190  {
191      return m_impl_batch->batch.ApproximateSize();
192  }
193  
194  struct LevelDBContext {
195      //! custom environment this database is using (may be nullptr in case of default environment)
196      leveldb::Env* penv;
197  
198      //! database options used
199      leveldb::Options options;
200  
201      //! options used when reading from the database
202      leveldb::ReadOptions readoptions;
203  
204      //! options used when iterating over values of the database
205      leveldb::ReadOptions iteroptions;
206  
207      //! options used when writing to the database
208      leveldb::WriteOptions writeoptions;
209  
210      //! options used when sync writing to the database
211      leveldb::WriteOptions syncoptions;
212  
213      //! the database itself
214      leveldb::DB* pdb;
215  };
216  
217  CDBWrapper::CDBWrapper(const DBParams& params)
218      : m_db_context{std::make_unique<LevelDBContext>()}, m_name{fs::PathToString(params.path.stem())}
219  {
220      DBContext().penv = nullptr;
221      DBContext().readoptions.verify_checksums = true;
222      DBContext().iteroptions.verify_checksums = true;
223      DBContext().iteroptions.fill_cache = false;
224      DBContext().syncoptions.sync = true;
225      DBContext().options = GetOptions(params.cache_bytes);
226      DBContext().options.create_if_missing = true;
227      if (params.memory_only) {
228          DBContext().penv = leveldb::NewMemEnv(leveldb::Env::Default());
229          DBContext().options.env = DBContext().penv;
230      } else {
231          if (params.wipe_data) {
232              LogInfo("Wiping LevelDB in %s", fs::PathToString(params.path));
233              leveldb::Status result = leveldb::DestroyDB(fs::PathToString(params.path), DBContext().options);
234              HandleError(result);
235          }
236          TryCreateDirectories(params.path);
237          LogInfo("Opening LevelDB in %s", fs::PathToString(params.path));
238      }
239      // PathToString() return value is safe to pass to leveldb open function,
240      // because on POSIX leveldb passes the byte string directly to ::open(), and
241      // on Windows it converts from UTF-8 to UTF-16 before calling ::CreateFileW
242      // (see env_posix.cc and env_windows.cc).
243      leveldb::Status status = leveldb::DB::Open(DBContext().options, fs::PathToString(params.path), &DBContext().pdb);
244      HandleError(status);
245      LogInfo("Opened LevelDB successfully");
246  
247      if (params.options.force_compact) {
248          LogInfo("Starting database compaction of %s", fs::PathToString(params.path));
249          DBContext().pdb->CompactRange(nullptr, nullptr);
250          LogInfo("Finished database compaction of %s", fs::PathToString(params.path));
251      }
252  
253      if (!Read(OBFUSCATION_KEY, m_obfuscation) && params.obfuscate && IsEmpty()) {
254          // Generate and write the new obfuscation key.
255          const Obfuscation obfuscation{FastRandomContext{}.randbytes<Obfuscation::KEY_SIZE>()};
256          assert(!m_obfuscation); // Make sure the key is written without obfuscation.
257          Write(OBFUSCATION_KEY, obfuscation);
258          m_obfuscation = obfuscation;
259          LogInfo("Wrote new obfuscation key for %s: %s", fs::PathToString(params.path), m_obfuscation.HexKey());
260      }
261      LogInfo("Using obfuscation key for %s: %s", fs::PathToString(params.path), m_obfuscation.HexKey());
262  }
263  
264  CDBWrapper::~CDBWrapper()
265  {
266      delete DBContext().pdb;
267      DBContext().pdb = nullptr;
268      delete DBContext().options.filter_policy;
269      DBContext().options.filter_policy = nullptr;
270      delete DBContext().options.info_log;
271      DBContext().options.info_log = nullptr;
272      delete DBContext().options.block_cache;
273      DBContext().options.block_cache = nullptr;
274      delete DBContext().penv;
275      DBContext().options.env = nullptr;
276  }
277  
278  void CDBWrapper::WriteBatch(CDBBatch& batch, bool fSync)
279  {
280      const bool log_memory = LogAcceptCategory(BCLog::LEVELDB, util::log::Level::Debug);
281      double mem_before = 0;
282      if (log_memory) {
283          mem_before = DynamicMemoryUsage() / 1024.0 / 1024;
284      }
285      leveldb::Status status = DBContext().pdb->Write(fSync ? DBContext().syncoptions : DBContext().writeoptions, &batch.m_impl_batch->batch);
286      HandleError(status);
287      if (log_memory) {
288          double mem_after = DynamicMemoryUsage() / 1024.0 / 1024;
289          LogDebug(BCLog::LEVELDB, "WriteBatch memory usage: db=%s, before=%.1fMiB, after=%.1fMiB\n",
290                   m_name, mem_before, mem_after);
291      }
292  }
293  
294  size_t CDBWrapper::DynamicMemoryUsage() const
295  {
296      std::string memory;
297      std::optional<size_t> parsed;
298      if (!DBContext().pdb->GetProperty("leveldb.approximate-memory-usage", &memory) || !(parsed = ToIntegral<size_t>(memory))) {
299          LogDebug(BCLog::LEVELDB, "Failed to get approximate-memory-usage property\n");
300          return 0;
301      }
302      return parsed.value();
303  }
304  
305  std::optional<std::string> CDBWrapper::ReadImpl(std::span<const std::byte> key) const
306  {
307      leveldb::Slice slKey(CharCast(key.data()), key.size());
308      std::string strValue;
309      leveldb::Status status = DBContext().pdb->Get(DBContext().readoptions, slKey, &strValue);
310      if (!status.ok()) {
311          if (status.IsNotFound())
312              return std::nullopt;
313          LogError("LevelDB read failure: %s", status.ToString());
314          HandleError(status);
315      }
316      return strValue;
317  }
318  
319  bool CDBWrapper::ExistsImpl(std::span<const std::byte> key) const
320  {
321      leveldb::Slice slKey(CharCast(key.data()), key.size());
322  
323      std::string strValue;
324      leveldb::Status status = DBContext().pdb->Get(DBContext().readoptions, slKey, &strValue);
325      if (!status.ok()) {
326          if (status.IsNotFound())
327              return false;
328          LogError("LevelDB read failure: %s", status.ToString());
329          HandleError(status);
330      }
331      return true;
332  }
333  
334  size_t CDBWrapper::EstimateSizeImpl(std::span<const std::byte> key1, std::span<const std::byte> key2) const
335  {
336      leveldb::Slice slKey1(CharCast(key1.data()), key1.size());
337      leveldb::Slice slKey2(CharCast(key2.data()), key2.size());
338      uint64_t size = 0;
339      leveldb::Range range(slKey1, slKey2);
340      DBContext().pdb->GetApproximateSizes(&range, 1, &size);
341      return size;
342  }
343  
344  bool CDBWrapper::IsEmpty()
345  {
346      std::unique_ptr<CDBIterator> it(NewIterator());
347      it->SeekToFirst();
348      return !(it->Valid());
349  }
350  
351  struct CDBIterator::IteratorImpl {
352      const std::unique_ptr<leveldb::Iterator> iter;
353  
354      explicit IteratorImpl(leveldb::Iterator* _iter) : iter{_iter} {}
355  };
356  
357  CDBIterator::CDBIterator(const CDBWrapper& _parent, std::unique_ptr<IteratorImpl> _piter) : parent(_parent),
358                                                                                              m_impl_iter(std::move(_piter)) {}
359  
360  CDBIterator* CDBWrapper::NewIterator()
361  {
362      return new CDBIterator{*this, std::make_unique<CDBIterator::IteratorImpl>(DBContext().pdb->NewIterator(DBContext().iteroptions))};
363  }
364  
365  void CDBIterator::SeekImpl(std::span<const std::byte> key)
366  {
367      leveldb::Slice slKey(CharCast(key.data()), key.size());
368      m_impl_iter->iter->Seek(slKey);
369  }
370  
371  std::span<const std::byte> CDBIterator::GetKeyImpl() const
372  {
373      return MakeByteSpan(m_impl_iter->iter->key());
374  }
375  
376  std::span<const std::byte> CDBIterator::GetValueImpl() const
377  {
378      return MakeByteSpan(m_impl_iter->iter->value());
379  }
380  
381  CDBIterator::~CDBIterator() = default;
382  bool CDBIterator::Valid() const { return m_impl_iter->iter->Valid(); }
383  void CDBIterator::SeekToFirst() { m_impl_iter->iter->SeekToFirst(); }
384  void CDBIterator::Next() { m_impl_iter->iter->Next(); }
385  
386  namespace dbwrapper_private {
387  
388  const Obfuscation& GetObfuscation(const CDBWrapper& w)
389  {
390      return w.m_obfuscation;
391  }
392  
393  } // namespace dbwrapper_private