streams_tests.cpp
1 // Copyright (c) 2012-present The Bitcoin Core developers 2 // Distributed under the MIT software license, see the accompanying 3 // file COPYING or http://www.opensource.org/licenses/mit-license.php. 4 5 #include <flatfile.h> 6 #include <node/blockstorage.h> 7 #include <streams.h> 8 #include <test/util/common.h> 9 #include <test/util/random.h> 10 #include <test/util/setup_common.h> 11 #include <util/fs.h> 12 #include <util/obfuscation.h> 13 #include <util/strencodings.h> 14 15 #include <boost/test/unit_test.hpp> 16 17 using namespace std::string_literals; 18 using namespace util::hex_literals; 19 20 BOOST_FIXTURE_TEST_SUITE(streams_tests, BasicTestingSetup) 21 22 // Check optimized obfuscation with random offsets and sizes to ensure proper 23 // handling of key wrapping. Also verify it roundtrips. 24 BOOST_AUTO_TEST_CASE(xor_random_chunks) 25 { 26 auto apply_random_xor_chunks{[&](std::span<std::byte> target, const Obfuscation& obfuscation) { 27 for (size_t offset{0}; offset < target.size();) { 28 const size_t chunk_size{1 + m_rng.randrange(target.size() - offset)}; 29 obfuscation(target.subspan(offset, chunk_size), offset); 30 offset += chunk_size; 31 } 32 }}; 33 34 for (size_t test{0}; test < 100; ++test) { 35 const size_t write_size{1 + m_rng.randrange(100U)}; 36 const std::vector original{m_rng.randbytes<std::byte>(write_size)}; 37 std::vector roundtrip{original}; 38 39 const auto key_bytes{m_rng.randbool() ? m_rng.randbytes<Obfuscation::KEY_SIZE>() : std::array<std::byte, Obfuscation::KEY_SIZE>{}}; 40 const Obfuscation obfuscation{key_bytes}; 41 apply_random_xor_chunks(roundtrip, obfuscation); 42 BOOST_CHECK_EQUAL(roundtrip.size(), original.size()); 43 for (size_t i{0}; i < original.size(); ++i) { 44 BOOST_CHECK_EQUAL(roundtrip[i], original[i] ^ key_bytes[i % Obfuscation::KEY_SIZE]); 45 } 46 47 apply_random_xor_chunks(roundtrip, obfuscation); 48 BOOST_CHECK_EQUAL_COLLECTIONS(roundtrip.begin(), roundtrip.end(), original.begin(), original.end()); 49 } 50 } 51 52 BOOST_AUTO_TEST_CASE(obfuscation_hexkey) 53 { 54 const auto key_bytes{m_rng.randbytes<Obfuscation::KEY_SIZE>()}; 55 56 const Obfuscation obfuscation{key_bytes}; 57 BOOST_CHECK_EQUAL(obfuscation.HexKey(), HexStr(key_bytes)); 58 } 59 60 BOOST_AUTO_TEST_CASE(obfuscation_serialize) 61 { 62 Obfuscation obfuscation{}; 63 BOOST_CHECK(!obfuscation); 64 65 // Test loading a key. 66 std::vector key_in{m_rng.randbytes<std::byte>(Obfuscation::KEY_SIZE)}; 67 DataStream ds_in; 68 ds_in << key_in; 69 BOOST_CHECK_EQUAL(ds_in.size(), 1 + Obfuscation::KEY_SIZE); // serialized as a vector 70 ds_in >> obfuscation; 71 72 // Test saving the key. 73 std::vector<std::byte> key_out; 74 DataStream ds_out; 75 ds_out << obfuscation; 76 ds_out >> key_out; 77 78 // Make sure saved key is the same. 79 BOOST_CHECK_EQUAL_COLLECTIONS(key_in.begin(), key_in.end(), key_out.begin(), key_out.end()); 80 } 81 82 BOOST_AUTO_TEST_CASE(obfuscation_empty) 83 { 84 const Obfuscation null_obf{}; 85 BOOST_CHECK(!null_obf); 86 87 const Obfuscation non_null_obf{"ff00ff00ff00ff00"_hex}; 88 BOOST_CHECK(non_null_obf); 89 } 90 91 BOOST_AUTO_TEST_CASE(xor_file) 92 { 93 fs::path xor_path{m_args.GetDataDirBase() / "test_xor.bin"}; 94 auto raw_file{[&](const auto& mode) { return fsbridge::fopen(xor_path, mode); }}; 95 const std::vector<uint8_t> test1{1, 2, 3}; 96 const std::vector<uint8_t> test2{4, 5}; 97 const Obfuscation obfuscation{"ff00ff00ff00ff00"_hex}; 98 99 { 100 // Check errors for missing file 101 AutoFile xor_file{raw_file("rb"), obfuscation}; 102 BOOST_CHECK_EXCEPTION(xor_file << std::byte{}, std::ios_base::failure, HasReason{"AutoFile::write: file handle is nullptr"}); 103 BOOST_CHECK_EXCEPTION(xor_file >> std::byte{}, std::ios_base::failure, HasReason{"AutoFile::read: file handle is nullptr"}); 104 BOOST_CHECK_EXCEPTION(xor_file.ignore(1), std::ios_base::failure, HasReason{"AutoFile::ignore: file handle is nullptr"}); 105 BOOST_CHECK_EXCEPTION(xor_file.size(), std::ios_base::failure, HasReason{"AutoFile::size: file handle is nullptr"}); 106 } 107 { 108 #ifdef __MINGW64__ 109 // Temporary workaround for https://github.com/bitcoin/bitcoin/issues/30210 110 const char* mode = "wb"; 111 #else 112 const char* mode = "wbx"; 113 #endif 114 AutoFile xor_file{raw_file(mode), obfuscation}; 115 xor_file << test1 << test2; 116 BOOST_CHECK_EQUAL(xor_file.size(), 7); 117 BOOST_REQUIRE_EQUAL(xor_file.fclose(), 0); 118 } 119 { 120 // Read raw from disk 121 AutoFile non_xor_file{raw_file("rb")}; 122 std::vector<std::byte> raw(7); 123 non_xor_file >> std::span{raw}; 124 BOOST_CHECK_EQUAL(HexStr(raw), "fc01fd03fd04fa"); 125 // Check that no padding exists 126 BOOST_CHECK_EXCEPTION(non_xor_file.ignore(1), std::ios_base::failure, HasReason{"AutoFile::ignore: end of file"}); 127 BOOST_CHECK_EQUAL(non_xor_file.size(), 7); 128 } 129 { 130 AutoFile xor_file{raw_file("rb"), obfuscation}; 131 std::vector<std::byte> read1, read2; 132 xor_file >> read1 >> read2; 133 BOOST_CHECK_EQUAL(HexStr(read1), HexStr(test1)); 134 BOOST_CHECK_EQUAL(HexStr(read2), HexStr(test2)); 135 // Check that eof was reached 136 BOOST_CHECK_EXCEPTION(xor_file >> std::byte{}, std::ios_base::failure, HasReason{"AutoFile::read: end of file"}); 137 BOOST_CHECK_EQUAL(xor_file.size(), 7); 138 } 139 { 140 AutoFile xor_file{raw_file("rb"), obfuscation}; 141 std::vector<std::byte> read2; 142 // Check that ignore works 143 xor_file.ignore(4); 144 xor_file >> read2; 145 BOOST_CHECK_EQUAL(HexStr(read2), HexStr(test2)); 146 // Check that ignore and read fail now 147 BOOST_CHECK_EXCEPTION(xor_file.ignore(1), std::ios_base::failure, HasReason{"AutoFile::ignore: end of file"}); 148 BOOST_CHECK_EXCEPTION(xor_file >> std::byte{}, std::ios_base::failure, HasReason{"AutoFile::read: end of file"}); 149 BOOST_CHECK_EQUAL(xor_file.size(), 7); 150 } 151 } 152 153 BOOST_AUTO_TEST_CASE(streams_vector_writer) 154 { 155 unsigned char a(1); 156 unsigned char b(2); 157 unsigned char bytes[] = {3, 4, 5, 6}; 158 std::vector<unsigned char> vch; 159 160 // Each test runs twice. Serializing a second time at the same starting 161 // point should yield the same results, even if the first test grew the 162 // vector. 163 164 VectorWriter{vch, 0, a, b}; 165 BOOST_CHECK((vch == std::vector<unsigned char>{{1, 2}})); 166 VectorWriter{vch, 0, a, b}; 167 BOOST_CHECK((vch == std::vector<unsigned char>{{1, 2}})); 168 vch.clear(); 169 170 VectorWriter{vch, 2, a, b}; 171 BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 1, 2}})); 172 VectorWriter{vch, 2, a, b}; 173 BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 1, 2}})); 174 vch.clear(); 175 176 vch.resize(5, 0); 177 VectorWriter{vch, 2, a, b}; 178 BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 1, 2, 0}})); 179 VectorWriter{vch, 2, a, b}; 180 BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 1, 2, 0}})); 181 vch.clear(); 182 183 vch.resize(4, 0); 184 VectorWriter{vch, 3, a, b}; 185 BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 0, 1, 2}})); 186 VectorWriter{vch, 3, a, b}; 187 BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 0, 1, 2}})); 188 vch.clear(); 189 190 vch.resize(4, 0); 191 VectorWriter{vch, 4, a, b}; 192 BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 0, 0, 1, 2}})); 193 VectorWriter{vch, 4, a, b}; 194 BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 0, 0, 1, 2}})); 195 vch.clear(); 196 197 VectorWriter{vch, 0, bytes}; 198 BOOST_CHECK((vch == std::vector<unsigned char>{{3, 4, 5, 6}})); 199 VectorWriter{vch, 0, bytes}; 200 BOOST_CHECK((vch == std::vector<unsigned char>{{3, 4, 5, 6}})); 201 vch.clear(); 202 203 vch.resize(4, 8); 204 VectorWriter{vch, 2, a, bytes, b}; 205 BOOST_CHECK((vch == std::vector<unsigned char>{{8, 8, 1, 3, 4, 5, 6, 2}})); 206 VectorWriter{vch, 2, a, bytes, b}; 207 BOOST_CHECK((vch == std::vector<unsigned char>{{8, 8, 1, 3, 4, 5, 6, 2}})); 208 vch.clear(); 209 } 210 211 BOOST_AUTO_TEST_CASE(streams_span_writer) 212 { 213 unsigned char a(1); 214 unsigned char b(2); 215 unsigned char bytes[] = {3, 4, 5, 6}; 216 std::array<std::byte, 8> arr{}; 217 218 // Test operator<< 219 SpanWriter writer{arr}; 220 writer << a << b; 221 BOOST_CHECK_EQUAL(HexStr(arr), "0102000000000000"); 222 223 // Use variadic constructor and write to subspan. 224 SpanWriter{std::span{arr}.subspan(2), a, bytes, b}; 225 BOOST_CHECK_EQUAL(HexStr(arr), "0102010304050602"); 226 227 // Writing past the end throws 228 std::array<std::byte, 1> small{}; 229 BOOST_CHECK_THROW(SpanWriter(std::span{small}, a, b), std::ios_base::failure); 230 BOOST_CHECK_THROW(SpanWriter(std::span{small}) << a << b, std::ios_base::failure); 231 } 232 233 BOOST_AUTO_TEST_CASE(streams_vector_reader) 234 { 235 std::vector<unsigned char> vch = {1, 255, 3, 4, 5, 6}; 236 237 SpanReader reader{vch}; 238 BOOST_CHECK_EQUAL(reader.size(), 6U); 239 BOOST_CHECK(!reader.empty()); 240 241 // Read a single byte as an unsigned char. 242 unsigned char a; 243 reader >> a; 244 BOOST_CHECK_EQUAL(a, 1); 245 BOOST_CHECK_EQUAL(reader.size(), 5U); 246 BOOST_CHECK(!reader.empty()); 247 248 // Read a single byte as a int8_t. 249 int8_t b; 250 reader >> b; 251 BOOST_CHECK_EQUAL(b, -1); 252 BOOST_CHECK_EQUAL(reader.size(), 4U); 253 BOOST_CHECK(!reader.empty()); 254 255 // Read a 4 bytes as an unsigned int. 256 unsigned int c; 257 reader >> c; 258 BOOST_CHECK_EQUAL(c, 100992003U); // 3,4,5,6 in little-endian base-256 259 BOOST_CHECK_EQUAL(reader.size(), 0U); 260 BOOST_CHECK(reader.empty()); 261 262 // Reading after end of byte vector throws an error. 263 signed int d; 264 BOOST_CHECK_THROW(reader >> d, std::ios_base::failure); 265 266 // Read a 4 bytes as a signed int from the beginning of the buffer. 267 SpanReader new_reader{vch}; 268 new_reader >> d; 269 BOOST_CHECK_EQUAL(d, 67370753); // 1,255,3,4 in little-endian base-256 270 BOOST_CHECK_EQUAL(new_reader.size(), 2U); 271 BOOST_CHECK(!new_reader.empty()); 272 273 // Reading after end of byte vector throws an error even if the reader is 274 // not totally empty. 275 BOOST_CHECK_THROW(new_reader >> d, std::ios_base::failure); 276 } 277 278 BOOST_AUTO_TEST_CASE(streams_vector_reader_rvalue) 279 { 280 std::vector<uint8_t> data{0x82, 0xa7, 0x31}; 281 SpanReader reader{data}; 282 uint32_t varint = 0; 283 // Deserialize into r-value 284 reader >> VARINT(varint); 285 BOOST_CHECK_EQUAL(varint, 54321U); 286 BOOST_CHECK(reader.empty()); 287 } 288 289 BOOST_AUTO_TEST_CASE(bitstream_reader_writer) 290 { 291 DataStream data{}; 292 293 BitStreamWriter bit_writer{data}; 294 bit_writer.Write(0, 1); 295 bit_writer.Write(2, 2); 296 bit_writer.Write(6, 3); 297 bit_writer.Write(11, 4); 298 bit_writer.Write(1, 5); 299 bit_writer.Write(32, 6); 300 bit_writer.Write(7, 7); 301 bit_writer.Write(30497, 16); 302 bit_writer.Flush(); 303 304 DataStream data_copy{data}; 305 uint32_t serialized_int1; 306 data >> serialized_int1; 307 BOOST_CHECK_EQUAL(serialized_int1, uint32_t{0x7700C35A}); // NOTE: Serialized as LE 308 uint16_t serialized_int2; 309 data >> serialized_int2; 310 BOOST_CHECK_EQUAL(serialized_int2, uint16_t{0x1072}); // NOTE: Serialized as LE 311 312 BitStreamReader bit_reader{data_copy}; 313 BOOST_CHECK_EQUAL(bit_reader.Read(1), 0U); 314 BOOST_CHECK_EQUAL(bit_reader.Read(2), 2U); 315 BOOST_CHECK_EQUAL(bit_reader.Read(3), 6U); 316 BOOST_CHECK_EQUAL(bit_reader.Read(4), 11U); 317 BOOST_CHECK_EQUAL(bit_reader.Read(5), 1U); 318 BOOST_CHECK_EQUAL(bit_reader.Read(6), 32U); 319 BOOST_CHECK_EQUAL(bit_reader.Read(7), 7U); 320 BOOST_CHECK_EQUAL(bit_reader.Read(16), 30497U); 321 BOOST_CHECK_THROW(bit_reader.Read(8), std::ios_base::failure); 322 } 323 324 BOOST_AUTO_TEST_CASE(streams_serializedata_xor) 325 { 326 // Degenerate case 327 { 328 DataStream ds{}; 329 Obfuscation{}(ds); 330 BOOST_CHECK_EQUAL(""s, ds.str()); 331 } 332 333 { 334 const Obfuscation obfuscation{"ffffffffffffffff"_hex}; 335 336 DataStream ds{"0ff0"_hex}; 337 obfuscation(ds); 338 BOOST_CHECK_EQUAL("\xf0\x0f"s, ds.str()); 339 } 340 341 { 342 const Obfuscation obfuscation{"ff0fff0fff0fff0f"_hex}; 343 344 DataStream ds{"f00f"_hex}; 345 obfuscation(ds); 346 BOOST_CHECK_EQUAL("\x0f\x00"s, ds.str()); 347 } 348 } 349 350 BOOST_AUTO_TEST_CASE(streams_buffered_file) 351 { 352 fs::path streams_test_filename = m_args.GetDataDirBase() / "streams_test_tmp"; 353 AutoFile file{fsbridge::fopen(streams_test_filename, "w+b")}; 354 355 // The value at each offset is the offset. 356 for (uint8_t j = 0; j < 40; ++j) { 357 file << j; 358 } 359 file.seek(0, SEEK_SET); 360 361 // The buffer size (second arg) must be greater than the rewind 362 // amount (third arg). 363 try { 364 BufferedFile bfbad{file, 25, 25}; 365 BOOST_CHECK(false); 366 } catch (const std::exception& e) { 367 BOOST_CHECK(strstr(e.what(), 368 "Rewind limit must be less than buffer size") != nullptr); 369 } 370 371 // The buffer is 25 bytes, allow rewinding 10 bytes. 372 BufferedFile bf{file, 25, 10}; 373 BOOST_CHECK(!bf.eof()); 374 375 uint8_t i; 376 bf >> i; 377 BOOST_CHECK_EQUAL(i, 0); 378 bf >> i; 379 BOOST_CHECK_EQUAL(i, 1); 380 381 // After reading bytes 0 and 1, we're positioned at 2. 382 BOOST_CHECK_EQUAL(bf.GetPos(), 2U); 383 384 // Rewind to offset 0, ok (within the 10 byte window). 385 BOOST_CHECK(bf.SetPos(0)); 386 bf >> i; 387 BOOST_CHECK_EQUAL(i, 0); 388 389 // We can go forward to where we've been, but beyond may fail. 390 BOOST_CHECK(bf.SetPos(2)); 391 bf >> i; 392 BOOST_CHECK_EQUAL(i, 2); 393 394 // If you know the maximum number of bytes that should be 395 // read to deserialize the variable, you can limit the read 396 // extent. The current file offset is 3, so the following 397 // SetLimit() allows zero bytes to be read. 398 BOOST_CHECK(bf.SetLimit(3)); 399 try { 400 bf >> i; 401 BOOST_CHECK(false); 402 } catch (const std::exception& e) { 403 BOOST_CHECK(strstr(e.what(), 404 "Attempt to position past buffer limit") != nullptr); 405 } 406 // The default argument removes the limit completely. 407 BOOST_CHECK(bf.SetLimit()); 408 // The read position should still be at 3 (no change). 409 BOOST_CHECK_EQUAL(bf.GetPos(), 3U); 410 411 // Read from current offset, 3, forward until position 10. 412 for (uint8_t j = 3; j < 10; ++j) { 413 bf >> i; 414 BOOST_CHECK_EQUAL(i, j); 415 } 416 BOOST_CHECK_EQUAL(bf.GetPos(), 10U); 417 418 // We're guaranteed (just barely) to be able to rewind to zero. 419 BOOST_CHECK(bf.SetPos(0)); 420 BOOST_CHECK_EQUAL(bf.GetPos(), 0U); 421 bf >> i; 422 BOOST_CHECK_EQUAL(i, 0); 423 424 // We can set the position forward again up to the farthest 425 // into the stream we've been, but no farther. (Attempting 426 // to go farther may succeed, but it's not guaranteed.) 427 BOOST_CHECK(bf.SetPos(10)); 428 bf >> i; 429 BOOST_CHECK_EQUAL(i, 10); 430 BOOST_CHECK_EQUAL(bf.GetPos(), 11U); 431 432 // Now it's only guaranteed that we can rewind to offset 1 433 // (current read position, 11, minus rewind amount, 10). 434 BOOST_CHECK(bf.SetPos(1)); 435 BOOST_CHECK_EQUAL(bf.GetPos(), 1U); 436 bf >> i; 437 BOOST_CHECK_EQUAL(i, 1); 438 439 // We can stream into large variables, even larger than 440 // the buffer size. 441 BOOST_CHECK(bf.SetPos(11)); 442 { 443 uint8_t a[40 - 11]; 444 bf >> a; 445 for (uint8_t j = 0; j < sizeof(a); ++j) { 446 BOOST_CHECK_EQUAL(a[j], 11 + j); 447 } 448 } 449 BOOST_CHECK_EQUAL(bf.GetPos(), 40U); 450 451 // We've read the entire file, the next read should throw. 452 try { 453 bf >> i; 454 BOOST_CHECK(false); 455 } catch (const std::exception& e) { 456 BOOST_CHECK(strstr(e.what(), 457 "BufferedFile::Fill: end of file") != nullptr); 458 } 459 // Attempting to read beyond the end sets the EOF indicator. 460 BOOST_CHECK(bf.eof()); 461 462 // Still at offset 40, we can go back 10, to 30. 463 BOOST_CHECK_EQUAL(bf.GetPos(), 40U); 464 BOOST_CHECK(bf.SetPos(30)); 465 bf >> i; 466 BOOST_CHECK_EQUAL(i, 30); 467 BOOST_CHECK_EQUAL(bf.GetPos(), 31U); 468 469 // We're too far to rewind to position zero. 470 BOOST_CHECK(!bf.SetPos(0)); 471 // But we should now be positioned at least as far back as allowed 472 // by the rewind window (relative to our farthest read position, 40). 473 BOOST_CHECK(bf.GetPos() <= 30U); 474 475 BOOST_REQUIRE_EQUAL(file.fclose(), 0); 476 477 fs::remove(streams_test_filename); 478 } 479 480 BOOST_AUTO_TEST_CASE(streams_buffered_file_skip) 481 { 482 fs::path streams_test_filename = m_args.GetDataDirBase() / "streams_test_tmp"; 483 AutoFile file{fsbridge::fopen(streams_test_filename, "w+b")}; 484 // The value at each offset is the byte offset (e.g. byte 1 in the file has the value 0x01). 485 for (uint8_t j = 0; j < 40; ++j) { 486 file << j; 487 } 488 file.seek(0, SEEK_SET); 489 490 // The buffer is 25 bytes, allow rewinding 10 bytes. 491 BufferedFile bf{file, 25, 10}; 492 493 uint8_t i; 494 // This is like bf >> (7-byte-variable), in that it will cause data 495 // to be read from the file into memory, but it's not copied to us. 496 bf.SkipTo(7); 497 BOOST_CHECK_EQUAL(bf.GetPos(), 7U); 498 bf >> i; 499 BOOST_CHECK_EQUAL(i, 7); 500 501 // The bytes in the buffer up to offset 7 are valid and can be read. 502 BOOST_CHECK(bf.SetPos(0)); 503 bf >> i; 504 BOOST_CHECK_EQUAL(i, 0); 505 bf >> i; 506 BOOST_CHECK_EQUAL(i, 1); 507 508 bf.SkipTo(11); 509 bf >> i; 510 BOOST_CHECK_EQUAL(i, 11); 511 512 // SkipTo() honors the transfer limit; we can't position beyond the limit. 513 bf.SetLimit(13); 514 try { 515 bf.SkipTo(14); 516 BOOST_CHECK(false); 517 } catch (const std::exception& e) { 518 BOOST_CHECK(strstr(e.what(), "Attempt to position past buffer limit") != nullptr); 519 } 520 521 // We can position exactly to the transfer limit. 522 bf.SkipTo(13); 523 BOOST_CHECK_EQUAL(bf.GetPos(), 13U); 524 525 BOOST_REQUIRE_EQUAL(file.fclose(), 0); 526 fs::remove(streams_test_filename); 527 } 528 529 BOOST_AUTO_TEST_CASE(streams_buffered_file_rand) 530 { 531 // Make this test deterministic. 532 SeedRandomForTest(SeedRand::ZEROS); 533 534 fs::path streams_test_filename = m_args.GetDataDirBase() / "streams_test_tmp"; 535 for (int rep = 0; rep < 50; ++rep) { 536 AutoFile file{fsbridge::fopen(streams_test_filename, "w+b")}; 537 size_t fileSize = m_rng.randrange(256); 538 for (uint8_t i = 0; i < fileSize; ++i) { 539 file << i; 540 } 541 file.seek(0, SEEK_SET); 542 543 size_t bufSize = m_rng.randrange(300) + 1; 544 size_t rewindSize = m_rng.randrange(bufSize); 545 BufferedFile bf{file, bufSize, rewindSize}; 546 size_t currentPos = 0; 547 size_t maxPos = 0; 548 for (int step = 0; step < 100; ++step) { 549 if (currentPos >= fileSize) 550 break; 551 552 // We haven't read to the end of the file yet. 553 BOOST_CHECK(!bf.eof()); 554 BOOST_CHECK_EQUAL(bf.GetPos(), currentPos); 555 556 // Pretend the file consists of a series of objects of varying 557 // sizes; the boundaries of the objects can interact arbitrarily 558 // with the CBufferFile's internal buffer. These first three 559 // cases simulate objects of various sizes (1, 2, 5 bytes). 560 switch (m_rng.randrange(6)) { 561 case 0: { 562 uint8_t a[1]; 563 if (currentPos + 1 > fileSize) 564 continue; 565 bf.SetLimit(currentPos + 1); 566 bf >> a; 567 for (uint8_t i = 0; i < 1; ++i) { 568 BOOST_CHECK_EQUAL(a[i], currentPos); 569 currentPos++; 570 } 571 break; 572 } 573 case 1: { 574 uint8_t a[2]; 575 if (currentPos + 2 > fileSize) 576 continue; 577 bf.SetLimit(currentPos + 2); 578 bf >> a; 579 for (uint8_t i = 0; i < 2; ++i) { 580 BOOST_CHECK_EQUAL(a[i], currentPos); 581 currentPos++; 582 } 583 break; 584 } 585 case 2: { 586 uint8_t a[5]; 587 if (currentPos + 5 > fileSize) 588 continue; 589 bf.SetLimit(currentPos + 5); 590 bf >> a; 591 for (uint8_t i = 0; i < 5; ++i) { 592 BOOST_CHECK_EQUAL(a[i], currentPos); 593 currentPos++; 594 } 595 break; 596 } 597 case 3: { 598 // SkipTo is similar to the "read" cases above, except 599 // we don't receive the data. 600 size_t skip_length{static_cast<size_t>(m_rng.randrange(5))}; 601 if (currentPos + skip_length > fileSize) continue; 602 bf.SetLimit(currentPos + skip_length); 603 bf.SkipTo(currentPos + skip_length); 604 currentPos += skip_length; 605 break; 606 } 607 case 4: { 608 // Find a byte value (that is at or ahead of the current position). 609 size_t find = currentPos + m_rng.randrange(8); 610 if (find >= fileSize) 611 find = fileSize - 1; 612 bf.FindByte(std::byte(find)); 613 // The value at each offset is the offset. 614 BOOST_CHECK_EQUAL(bf.GetPos(), find); 615 currentPos = find; 616 617 bf.SetLimit(currentPos + 1); 618 uint8_t i; 619 bf >> i; 620 BOOST_CHECK_EQUAL(i, currentPos); 621 currentPos++; 622 break; 623 } 624 case 5: { 625 size_t requestPos = m_rng.randrange(maxPos + 4); 626 bool okay = bf.SetPos(requestPos); 627 // The new position may differ from the requested position 628 // because we may not be able to rewind beyond the rewind 629 // window, and we may not be able to move forward beyond the 630 // farthest position we've reached so far. 631 currentPos = bf.GetPos(); 632 BOOST_CHECK_EQUAL(okay, currentPos == requestPos); 633 // Check that we can position within the rewind window. 634 if (requestPos <= maxPos && 635 maxPos > rewindSize && 636 requestPos >= maxPos - rewindSize) { 637 // We requested a position within the rewind window. 638 BOOST_CHECK(okay); 639 } 640 break; 641 } 642 } 643 if (maxPos < currentPos) 644 maxPos = currentPos; 645 } 646 BOOST_REQUIRE_EQUAL(file.fclose(), 0); 647 } 648 fs::remove(streams_test_filename); 649 } 650 651 BOOST_AUTO_TEST_CASE(buffered_reader_matches_autofile_random_content) 652 { 653 const size_t file_size{1 + m_rng.randrange<size_t>(1 << 17)}; 654 const size_t buf_size{1 + m_rng.randrange(file_size)}; 655 const FlatFilePos pos{0, 0}; 656 657 const FlatFileSeq test_file{m_args.GetDataDirBase(), "buffered_file_test_random", node::BLOCKFILE_CHUNK_SIZE}; 658 const Obfuscation obfuscation{m_rng.randbytes<Obfuscation::KEY_SIZE>()}; 659 660 // Write out the file with random content 661 { 662 AutoFile f{test_file.Open(pos, /*read_only=*/false), obfuscation}; 663 f.write(m_rng.randbytes<std::byte>(file_size)); 664 BOOST_REQUIRE_EQUAL(f.fclose(), 0); 665 } 666 BOOST_CHECK_EQUAL(fs::file_size(test_file.FileName(pos)), file_size); 667 668 { 669 AutoFile direct_file{test_file.Open(pos, /*read_only=*/true), obfuscation}; 670 671 AutoFile buffered_file{test_file.Open(pos, /*read_only=*/true), obfuscation}; 672 BufferedReader buffered_reader{std::move(buffered_file), buf_size}; 673 674 for (size_t total_read{0}; total_read < file_size;) { 675 const size_t read{Assert(std::min(1 + m_rng.randrange(m_rng.randbool() ? buf_size : 2 * buf_size), file_size - total_read))}; 676 677 DataBuffer direct_file_buffer{read}; 678 direct_file.read(direct_file_buffer); 679 680 DataBuffer buffered_buffer{read}; 681 buffered_reader.read(buffered_buffer); 682 683 BOOST_CHECK_EQUAL_COLLECTIONS( 684 direct_file_buffer.begin(), direct_file_buffer.end(), 685 buffered_buffer.begin(), buffered_buffer.end() 686 ); 687 688 total_read += read; 689 } 690 691 { 692 DataBuffer excess_byte{1}; 693 BOOST_CHECK_EXCEPTION(direct_file.read(excess_byte), std::ios_base::failure, HasReason{"end of file"}); 694 } 695 696 { 697 DataBuffer excess_byte{1}; 698 BOOST_CHECK_EXCEPTION(buffered_reader.read(excess_byte), std::ios_base::failure, HasReason{"end of file"}); 699 } 700 } 701 702 fs::remove(test_file.FileName(pos)); 703 } 704 705 BOOST_AUTO_TEST_CASE(buffered_writer_matches_autofile_random_content) 706 { 707 const size_t file_size{1 + m_rng.randrange<size_t>(1 << 17)}; 708 const size_t buf_size{1 + m_rng.randrange(file_size)}; 709 const FlatFilePos pos{0, 0}; 710 711 const FlatFileSeq test_buffered{m_args.GetDataDirBase(), "buffered_write_test", node::BLOCKFILE_CHUNK_SIZE}; 712 const FlatFileSeq test_direct{m_args.GetDataDirBase(), "direct_write_test", node::BLOCKFILE_CHUNK_SIZE}; 713 const Obfuscation obfuscation{m_rng.randbytes<Obfuscation::KEY_SIZE>()}; 714 715 { 716 DataBuffer test_data{m_rng.randbytes<std::byte>(file_size)}; 717 718 AutoFile direct_file{test_direct.Open(pos, /*read_only=*/false), obfuscation}; 719 720 AutoFile buffered_file{test_buffered.Open(pos, /*read_only=*/false), obfuscation}; 721 { 722 BufferedWriter buffered{buffered_file, buf_size}; 723 724 for (size_t total_written{0}; total_written < file_size;) { 725 const size_t write_size{Assert(std::min(1 + m_rng.randrange(m_rng.randbool() ? buf_size : 2 * buf_size), file_size - total_written))}; 726 727 auto current_span = std::span{test_data}.subspan(total_written, write_size); 728 direct_file.write(current_span); 729 buffered.write(current_span); 730 731 total_written += write_size; 732 } 733 } 734 BOOST_REQUIRE_EQUAL(buffered_file.fclose(), 0); 735 BOOST_REQUIRE_EQUAL(direct_file.fclose(), 0); 736 } 737 738 // Compare the resulting files 739 DataBuffer direct_result{file_size}; 740 { 741 AutoFile verify_direct{test_direct.Open(pos, /*read_only=*/true), obfuscation}; 742 verify_direct.read(direct_result); 743 744 DataBuffer excess_byte{1}; 745 BOOST_CHECK_EXCEPTION(verify_direct.read(excess_byte), std::ios_base::failure, HasReason{"end of file"}); 746 } 747 748 DataBuffer buffered_result{file_size}; 749 { 750 AutoFile verify_buffered{test_buffered.Open(pos, /*read_only=*/true), obfuscation}; 751 verify_buffered.read(buffered_result); 752 753 DataBuffer excess_byte{1}; 754 BOOST_CHECK_EXCEPTION(verify_buffered.read(excess_byte), std::ios_base::failure, HasReason{"end of file"}); 755 } 756 757 BOOST_CHECK_EQUAL_COLLECTIONS( 758 direct_result.begin(), direct_result.end(), 759 buffered_result.begin(), buffered_result.end() 760 ); 761 762 fs::remove(test_direct.FileName(pos)); 763 fs::remove(test_buffered.FileName(pos)); 764 } 765 766 BOOST_AUTO_TEST_CASE(buffered_writer_reader) 767 { 768 const uint32_t v1{m_rng.rand32()}, v2{m_rng.rand32()}, v3{m_rng.rand32()}; 769 const fs::path test_file{m_args.GetDataDirBase() / "test_buffered_write_read.bin"}; 770 771 // Write out the values through a precisely sized BufferedWriter 772 AutoFile file{fsbridge::fopen(test_file, "w+b")}; 773 { 774 BufferedWriter f(file, sizeof(v1) + sizeof(v2) + sizeof(v3)); 775 f << v1 << v2; 776 f.write(std::as_bytes(std::span{&v3, 1})); 777 } 778 BOOST_REQUIRE_EQUAL(file.fclose(), 0); 779 780 // Read back and verify using BufferedReader 781 { 782 uint32_t _v1{0}, _v2{0}, _v3{0}; 783 AutoFile file{fsbridge::fopen(test_file, "rb")}; 784 BufferedReader f(std::move(file), sizeof(v1) + sizeof(v2) + sizeof(v3)); 785 f >> _v1 >> _v2; 786 f.read(std::as_writable_bytes(std::span{&_v3, 1})); 787 BOOST_CHECK_EQUAL(_v1, v1); 788 BOOST_CHECK_EQUAL(_v2, v2); 789 BOOST_CHECK_EQUAL(_v3, v3); 790 791 DataBuffer excess_byte{1}; 792 BOOST_CHECK_EXCEPTION(f.read(excess_byte), std::ios_base::failure, HasReason{"end of file"}); 793 } 794 795 fs::remove(test_file); 796 } 797 798 BOOST_AUTO_TEST_CASE(streams_hashed) 799 { 800 DataStream stream{}; 801 HashedSourceWriter hash_writer{stream}; 802 const std::string data{"bitcoin"}; 803 hash_writer << data; 804 805 HashVerifier hash_verifier{stream}; 806 std::string result; 807 hash_verifier >> result; 808 BOOST_CHECK_EQUAL(data, result); 809 BOOST_CHECK_EQUAL(hash_writer.GetHash(), hash_verifier.GetHash()); 810 } 811 812 BOOST_AUTO_TEST_CASE(size_preserves_position) 813 { 814 const fs::path path = m_args.GetDataDirBase() / "size_pos_test.bin"; 815 AutoFile f{fsbridge::fopen(path, "w+b")}; 816 for (uint8_t j = 0; j < 10; ++j) { 817 f << j; 818 } 819 820 // Test that usage of size() does not change the current position 821 // 822 // Case: Pos at beginning of the file 823 f.seek(0, SEEK_SET); 824 (void)f.size(); 825 uint8_t first{}; 826 f >> first; 827 BOOST_CHECK_EQUAL(first, 0); 828 829 // Case: Pos at middle of the file 830 f.seek(0, SEEK_SET); 831 // Move pos to middle 832 f.ignore(4); 833 (void)f.size(); 834 uint8_t middle{}; 835 f >> middle; 836 // Pos still at 4 837 BOOST_CHECK_EQUAL(middle, 4); 838 839 // Case: Pos at EOF 840 f.seek(0, SEEK_END); 841 (void)f.size(); 842 uint8_t end{}; 843 BOOST_CHECK_EXCEPTION(f >> end, std::ios_base::failure, HasReason{"AutoFile::read: end of file"}); 844 845 BOOST_REQUIRE_EQUAL(f.fclose(), 0); 846 fs::remove(path); 847 } 848 849 BOOST_AUTO_TEST_SUITE_END()