1  // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
  2  // Use of this source code is governed by a BSD-style license that can be
  3  // found in the LICENSE file. See the AUTHORS file for names of contributors.
  4  
  5  #include <sys/types.h>
  6  
  7  #include "db/db_impl.h"
  8  #include "db/filename.h"
  9  #include "db/log_format.h"
 10  #include "db/version_set.h"
 11  #include "leveldb/cache.h"
 12  #include "leveldb/db.h"
 13  #include "leveldb/table.h"
 14  #include "leveldb/write_batch.h"
 15  #include "util/logging.h"
 16  #include "util/testharness.h"
 17  #include "util/testutil.h"
 18  
 19  namespace leveldb {
 20  
 21  static const int kValueSize = 1000;
 22  
 23  class CorruptionTest {
 24   public:
 25    CorruptionTest()
 26        : db_(nullptr),
 27          dbname_("/memenv/corruption_test"),
 28          tiny_cache_(NewLRUCache(100)) {
 29      options_.env = &env_;
 30      options_.block_cache = tiny_cache_;
 31      DestroyDB(dbname_, options_);
 32  
 33      options_.create_if_missing = true;
 34      Reopen();
 35      options_.create_if_missing = false;
 36    }
 37  
 38    ~CorruptionTest() {
 39      delete db_;
 40      delete tiny_cache_;
 41    }
 42  
 43    Status TryReopen() {
 44      delete db_;
 45      db_ = nullptr;
 46      return DB::Open(options_, dbname_, &db_);
 47    }
 48  
 49    void Reopen() { ASSERT_OK(TryReopen()); }
 50  
 51    void RepairDB() {
 52      delete db_;
 53      db_ = nullptr;
 54      ASSERT_OK(::leveldb::RepairDB(dbname_, options_));
 55    }
 56  
 57    void Build(int n) {
 58      std::string key_space, value_space;
 59      WriteBatch batch;
 60      for (int i = 0; i < n; i++) {
 61        // if ((i % 100) == 0) fprintf(stderr, "@ %d of %d\n", i, n);
 62        Slice key = Key(i, &key_space);
 63        batch.Clear();
 64        batch.Put(key, Value(i, &value_space));
 65        WriteOptions options;
 66        // Corrupt() doesn't work without this sync on windows; stat reports 0 for
 67        // the file size.
 68        if (i == n - 1) {
 69          options.sync = true;
 70        }
 71        ASSERT_OK(db_->Write(options, &batch));
 72      }
 73    }
 74  
 75    void Check(int min_expected, int max_expected) {
 76      int next_expected = 0;
 77      int missed = 0;
 78      int bad_keys = 0;
 79      int bad_values = 0;
 80      int correct = 0;
 81      std::string value_space;
 82      Iterator* iter = db_->NewIterator(ReadOptions());
 83      for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
 84        uint64_t key;
 85        Slice in(iter->key());
 86        if (in == "" || in == "~") {
 87          // Ignore boundary keys.
 88          continue;
 89        }
 90        if (!ConsumeDecimalNumber(&in, &key) || !in.empty() ||
 91            key < next_expected) {
 92          bad_keys++;
 93          continue;
 94        }
 95        missed += (key - next_expected);
 96        next_expected = key + 1;
 97        if (iter->value() != Value(key, &value_space)) {
 98          bad_values++;
 99        } else {
100          correct++;
101        }
102      }
103      delete iter;
104  
105      fprintf(stderr,
106              "expected=%d..%d; got=%d; bad_keys=%d; bad_values=%d; missed=%d\n",
107              min_expected, max_expected, correct, bad_keys, bad_values, missed);
108      ASSERT_LE(min_expected, correct);
109      ASSERT_GE(max_expected, correct);
110    }
111  
112    void Corrupt(FileType filetype, int offset, int bytes_to_corrupt) {
113      // Pick file to corrupt
114      std::vector<std::string> filenames;
115      ASSERT_OK(env_.target()->GetChildren(dbname_, &filenames));
116      uint64_t number;
117      FileType type;
118      std::string fname;
119      int picked_number = -1;
120      for (size_t i = 0; i < filenames.size(); i++) {
121        if (ParseFileName(filenames[i], &number, &type) && type == filetype &&
122            int(number) > picked_number) {  // Pick latest file
123          fname = dbname_ + "/" + filenames[i];
124          picked_number = number;
125        }
126      }
127      ASSERT_TRUE(!fname.empty()) << filetype;
128  
129      uint64_t file_size;
130      ASSERT_OK(env_.target()->GetFileSize(fname, &file_size));
131  
132      if (offset < 0) {
133        // Relative to end of file; make it absolute
134        if (-offset > file_size) {
135          offset = 0;
136        } else {
137          offset = file_size + offset;
138        }
139      }
140      if (offset > file_size) {
141        offset = file_size;
142      }
143      if (offset + bytes_to_corrupt > file_size) {
144        bytes_to_corrupt = file_size - offset;
145      }
146  
147      // Do it
148      std::string contents;
149      Status s = ReadFileToString(env_.target(), fname, &contents);
150      ASSERT_TRUE(s.ok()) << s.ToString();
151      for (int i = 0; i < bytes_to_corrupt; i++) {
152        contents[i + offset] ^= 0x80;
153      }
154      s = WriteStringToFile(env_.target(), contents, fname);
155      ASSERT_TRUE(s.ok()) << s.ToString();
156    }
157  
158    int Property(const std::string& name) {
159      std::string property;
160      int result;
161      if (db_->GetProperty(name, &property) &&
162          sscanf(property.c_str(), "%d", &result) == 1) {
163        return result;
164      } else {
165        return -1;
166      }
167    }
168  
169    // Return the ith key
170    Slice Key(int i, std::string* storage) {
171      char buf[100];
172      snprintf(buf, sizeof(buf), "%016d", i);
173      storage->assign(buf, strlen(buf));
174      return Slice(*storage);
175    }
176  
177    // Return the value to associate with the specified key
178    Slice Value(int k, std::string* storage) {
179      Random r(k);
180      return test::RandomString(&r, kValueSize, storage);
181    }
182  
183    test::ErrorEnv env_;
184    Options options_;
185    DB* db_;
186  
187   private:
188    std::string dbname_;
189    Cache* tiny_cache_;
190  };
191  
192  TEST(CorruptionTest, Recovery) {
193    Build(100);
194    Check(100, 100);
195    Corrupt(kLogFile, 19, 1);  // WriteBatch tag for first record
196    Corrupt(kLogFile, log::kBlockSize + 1000, 1);  // Somewhere in second block
197    Reopen();
198  
199    // The 64 records in the first two log blocks are completely lost.
200    Check(36, 36);
201  }
202  
203  TEST(CorruptionTest, RecoverWriteError) {
204    env_.writable_file_error_ = true;
205    Status s = TryReopen();
206    ASSERT_TRUE(!s.ok());
207  }
208  
209  TEST(CorruptionTest, NewFileErrorDuringWrite) {
210    // Do enough writing to force minor compaction
211    env_.writable_file_error_ = true;
212    const int num = 3 + (Options().write_buffer_size / kValueSize);
213    std::string value_storage;
214    Status s;
215    for (int i = 0; s.ok() && i < num; i++) {
216      WriteBatch batch;
217      batch.Put("a", Value(100, &value_storage));
218      s = db_->Write(WriteOptions(), &batch);
219    }
220    ASSERT_TRUE(!s.ok());
221    ASSERT_GE(env_.num_writable_file_errors_, 1);
222    env_.writable_file_error_ = false;
223    Reopen();
224  }
225  
226  TEST(CorruptionTest, TableFile) {
227    Build(100);
228    DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
229    dbi->TEST_CompactMemTable();
230    dbi->TEST_CompactRange(0, nullptr, nullptr);
231    dbi->TEST_CompactRange(1, nullptr, nullptr);
232  
233    Corrupt(kTableFile, 100, 1);
234    Check(90, 99);
235  }
236  
237  TEST(CorruptionTest, TableFileRepair) {
238    options_.block_size = 2 * kValueSize;  // Limit scope of corruption
239    options_.paranoid_checks = true;
240    Reopen();
241    Build(100);
242    DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
243    dbi->TEST_CompactMemTable();
244    dbi->TEST_CompactRange(0, nullptr, nullptr);
245    dbi->TEST_CompactRange(1, nullptr, nullptr);
246  
247    Corrupt(kTableFile, 100, 1);
248    RepairDB();
249    Reopen();
250    Check(95, 99);
251  }
252  
253  TEST(CorruptionTest, TableFileIndexData) {
254    Build(10000);  // Enough to build multiple Tables
255    DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
256    dbi->TEST_CompactMemTable();
257  
258    Corrupt(kTableFile, -2000, 500);
259    Reopen();
260    Check(5000, 9999);
261  }
262  
263  TEST(CorruptionTest, MissingDescriptor) {
264    Build(1000);
265    RepairDB();
266    Reopen();
267    Check(1000, 1000);
268  }
269  
270  TEST(CorruptionTest, SequenceNumberRecovery) {
271    ASSERT_OK(db_->Put(WriteOptions(), "foo", "v1"));
272    ASSERT_OK(db_->Put(WriteOptions(), "foo", "v2"));
273    ASSERT_OK(db_->Put(WriteOptions(), "foo", "v3"));
274    ASSERT_OK(db_->Put(WriteOptions(), "foo", "v4"));
275    ASSERT_OK(db_->Put(WriteOptions(), "foo", "v5"));
276    RepairDB();
277    Reopen();
278    std::string v;
279    ASSERT_OK(db_->Get(ReadOptions(), "foo", &v));
280    ASSERT_EQ("v5", v);
281    // Write something.  If sequence number was not recovered properly,
282    // it will be hidden by an earlier write.
283    ASSERT_OK(db_->Put(WriteOptions(), "foo", "v6"));
284    ASSERT_OK(db_->Get(ReadOptions(), "foo", &v));
285    ASSERT_EQ("v6", v);
286    Reopen();
287    ASSERT_OK(db_->Get(ReadOptions(), "foo", &v));
288    ASSERT_EQ("v6", v);
289  }
290  
291  TEST(CorruptionTest, CorruptedDescriptor) {
292    ASSERT_OK(db_->Put(WriteOptions(), "foo", "hello"));
293    DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
294    dbi->TEST_CompactMemTable();
295    dbi->TEST_CompactRange(0, nullptr, nullptr);
296  
297    Corrupt(kDescriptorFile, 0, 1000);
298    Status s = TryReopen();
299    ASSERT_TRUE(!s.ok());
300  
301    RepairDB();
302    Reopen();
303    std::string v;
304    ASSERT_OK(db_->Get(ReadOptions(), "foo", &v));
305    ASSERT_EQ("hello", v);
306  }
307  
308  TEST(CorruptionTest, CompactionInputError) {
309    Build(10);
310    DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
311    dbi->TEST_CompactMemTable();
312    const int last = config::kMaxMemCompactLevel;
313    ASSERT_EQ(1, Property("leveldb.num-files-at-level" + NumberToString(last)));
314  
315    Corrupt(kTableFile, 100, 1);
316    Check(5, 9);
317  
318    // Force compactions by writing lots of values
319    Build(10000);
320    Check(10000, 10000);
321  }
322  
323  TEST(CorruptionTest, CompactionInputErrorParanoid) {
324    options_.paranoid_checks = true;
325    options_.write_buffer_size = 512 << 10;
326    Reopen();
327    DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
328  
329    // Make multiple inputs so we need to compact.
330    for (int i = 0; i < 2; i++) {
331      Build(10);
332      dbi->TEST_CompactMemTable();
333      Corrupt(kTableFile, 100, 1);
334      env_.SleepForMicroseconds(100000);
335    }
336    dbi->CompactRange(nullptr, nullptr);
337  
338    // Write must fail because of corrupted table
339    std::string tmp1, tmp2;
340    Status s = db_->Put(WriteOptions(), Key(5, &tmp1), Value(5, &tmp2));
341    ASSERT_TRUE(!s.ok()) << "write did not fail in corrupted paranoid db";
342  }
343  
344  TEST(CorruptionTest, UnrelatedKeys) {
345    Build(10);
346    DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
347    dbi->TEST_CompactMemTable();
348    Corrupt(kTableFile, 100, 1);
349  
350    std::string tmp1, tmp2;
351    ASSERT_OK(db_->Put(WriteOptions(), Key(1000, &tmp1), Value(1000, &tmp2)));
352    std::string v;
353    ASSERT_OK(db_->Get(ReadOptions(), Key(1000, &tmp1), &v));
354    ASSERT_EQ(Value(1000, &tmp2).ToString(), v);
355    dbi->TEST_CompactMemTable();
356    ASSERT_OK(db_->Get(ReadOptions(), Key(1000, &tmp1), &v));
357    ASSERT_EQ(Value(1000, &tmp2).ToString(), v);
358  }
359  
360  }  // namespace leveldb
361  
362  int main(int argc, char** argv) { return leveldb::test::RunAllTests(); }