streams_tests.cpp
1 // Copyright (c) 2012-present The Bitcoin Core developers 2 // Distributed under the MIT software license, see the accompanying 3 // file COPYING or http://www.opensource.org/licenses/mit-license.php. 4 5 #include <flatfile.h> 6 #include <node/blockstorage.h> 7 #include <streams.h> 8 #include <test/util/common.h> 9 #include <test/util/random.h> 10 #include <test/util/setup_common.h> 11 #include <util/fs.h> 12 #include <util/obfuscation.h> 13 #include <util/strencodings.h> 14 15 #include <boost/test/unit_test.hpp> 16 17 using namespace std::string_literals; 18 using namespace util::hex_literals; 19 20 BOOST_FIXTURE_TEST_SUITE(streams_tests, BasicTestingSetup) 21 22 // Check optimized obfuscation with random offsets and sizes to ensure proper 23 // handling of key wrapping. Also verify it roundtrips. 24 BOOST_AUTO_TEST_CASE(xor_random_chunks) 25 { 26 auto apply_random_xor_chunks{[&](std::span<std::byte> target, const Obfuscation& obfuscation) { 27 for (size_t offset{0}; offset < target.size();) { 28 const size_t chunk_size{1 + m_rng.randrange(target.size() - offset)}; 29 obfuscation(target.subspan(offset, chunk_size), offset); 30 offset += chunk_size; 31 } 32 }}; 33 34 for (size_t test{0}; test < 100; ++test) { 35 const size_t write_size{1 + m_rng.randrange(100U)}; 36 const std::vector original{m_rng.randbytes<std::byte>(write_size)}; 37 std::vector roundtrip{original}; 38 39 const auto key_bytes{m_rng.randbool() ? m_rng.randbytes<Obfuscation::KEY_SIZE>() : std::array<std::byte, Obfuscation::KEY_SIZE>{}}; 40 const Obfuscation obfuscation{key_bytes}; 41 apply_random_xor_chunks(roundtrip, obfuscation); 42 BOOST_CHECK_EQUAL(roundtrip.size(), original.size()); 43 for (size_t i{0}; i < original.size(); ++i) { 44 BOOST_CHECK_EQUAL(roundtrip[i], original[i] ^ key_bytes[i % Obfuscation::KEY_SIZE]); 45 } 46 47 apply_random_xor_chunks(roundtrip, obfuscation); 48 BOOST_CHECK_EQUAL_COLLECTIONS(roundtrip.begin(), roundtrip.end(), original.begin(), original.end()); 49 } 50 } 51 52 BOOST_AUTO_TEST_CASE(obfuscation_hexkey) 53 { 54 const auto key_bytes{m_rng.randbytes<Obfuscation::KEY_SIZE>()}; 55 56 const Obfuscation obfuscation{key_bytes}; 57 BOOST_CHECK_EQUAL(obfuscation.HexKey(), HexStr(key_bytes)); 58 } 59 60 BOOST_AUTO_TEST_CASE(obfuscation_serialize) 61 { 62 Obfuscation obfuscation{}; 63 BOOST_CHECK(!obfuscation); 64 65 // Test loading a key. 66 std::vector key_in{m_rng.randbytes<std::byte>(Obfuscation::KEY_SIZE)}; 67 DataStream ds_in; 68 ds_in << key_in; 69 BOOST_CHECK_EQUAL(ds_in.size(), 1 + Obfuscation::KEY_SIZE); // serialized as a vector 70 ds_in >> obfuscation; 71 72 // Test saving the key. 73 std::vector<std::byte> key_out; 74 DataStream ds_out; 75 ds_out << obfuscation; 76 ds_out >> key_out; 77 78 // Make sure saved key is the same. 79 BOOST_CHECK_EQUAL_COLLECTIONS(key_in.begin(), key_in.end(), key_out.begin(), key_out.end()); 80 } 81 82 BOOST_AUTO_TEST_CASE(obfuscation_empty) 83 { 84 const Obfuscation null_obf{}; 85 BOOST_CHECK(!null_obf); 86 87 const Obfuscation non_null_obf{"ff00ff00ff00ff00"_hex}; 88 BOOST_CHECK(non_null_obf); 89 } 90 91 BOOST_AUTO_TEST_CASE(xor_file) 92 { 93 fs::path xor_path{m_args.GetDataDirBase() / "test_xor.bin"}; 94 auto raw_file{[&](const auto& mode) { return fsbridge::fopen(xor_path, mode); }}; 95 const std::vector<uint8_t> test1{1, 2, 3}; 96 const std::vector<uint8_t> test2{4, 5}; 97 const Obfuscation obfuscation{"ff00ff00ff00ff00"_hex}; 98 99 { 100 // Check errors for missing file 101 AutoFile xor_file{raw_file("rb"), obfuscation}; 102 BOOST_CHECK_EXCEPTION(xor_file << std::byte{}, std::ios_base::failure, HasReason{"AutoFile::write: file handle is nullptr"}); 103 BOOST_CHECK_EXCEPTION(xor_file >> std::byte{}, std::ios_base::failure, HasReason{"AutoFile::read: file handle is nullptr"}); 104 BOOST_CHECK_EXCEPTION(xor_file.ignore(1), std::ios_base::failure, HasReason{"AutoFile::ignore: file handle is nullptr"}); 105 BOOST_CHECK_EXCEPTION(xor_file.size(), std::ios_base::failure, HasReason{"AutoFile::size: file handle is nullptr"}); 106 } 107 { 108 #ifdef __MINGW64__ 109 // Temporary workaround for https://github.com/bitcoin/bitcoin/issues/30210 110 const char* mode = "wb"; 111 #else 112 const char* mode = "wbx"; 113 #endif 114 AutoFile xor_file{raw_file(mode), obfuscation}; 115 xor_file << test1 << test2; 116 BOOST_CHECK_EQUAL(xor_file.size(), 7); 117 BOOST_REQUIRE_EQUAL(xor_file.fclose(), 0); 118 } 119 { 120 // Read raw from disk 121 AutoFile non_xor_file{raw_file("rb")}; 122 std::vector<std::byte> raw(7); 123 non_xor_file >> std::span{raw}; 124 BOOST_CHECK_EQUAL(HexStr(raw), "fc01fd03fd04fa"); 125 // Check that no padding exists 126 BOOST_CHECK_EXCEPTION(non_xor_file.ignore(1), std::ios_base::failure, HasReason{"AutoFile::ignore: end of file"}); 127 BOOST_CHECK_EQUAL(non_xor_file.size(), 7); 128 } 129 { 130 AutoFile xor_file{raw_file("rb"), obfuscation}; 131 std::vector<std::byte> read1, read2; 132 xor_file >> read1 >> read2; 133 BOOST_CHECK_EQUAL(HexStr(read1), HexStr(test1)); 134 BOOST_CHECK_EQUAL(HexStr(read2), HexStr(test2)); 135 // Check that eof was reached 136 BOOST_CHECK_EXCEPTION(xor_file >> std::byte{}, std::ios_base::failure, HasReason{"AutoFile::read: end of file"}); 137 BOOST_CHECK_EQUAL(xor_file.size(), 7); 138 } 139 { 140 AutoFile xor_file{raw_file("rb"), obfuscation}; 141 std::vector<std::byte> read2; 142 // Check that ignore works 143 xor_file.ignore(4); 144 xor_file >> read2; 145 BOOST_CHECK_EQUAL(HexStr(read2), HexStr(test2)); 146 // Check that ignore and read fail now 147 BOOST_CHECK_EXCEPTION(xor_file.ignore(1), std::ios_base::failure, HasReason{"AutoFile::ignore: end of file"}); 148 BOOST_CHECK_EXCEPTION(xor_file >> std::byte{}, std::ios_base::failure, HasReason{"AutoFile::read: end of file"}); 149 BOOST_CHECK_EQUAL(xor_file.size(), 7); 150 } 151 } 152 153 BOOST_AUTO_TEST_CASE(streams_vector_writer) 154 { 155 unsigned char a(1); 156 unsigned char b(2); 157 unsigned char bytes[] = {3, 4, 5, 6}; 158 std::vector<unsigned char> vch; 159 160 // Each test runs twice. Serializing a second time at the same starting 161 // point should yield the same results, even if the first test grew the 162 // vector. 163 164 VectorWriter{vch, 0, a, b}; 165 BOOST_CHECK((vch == std::vector<unsigned char>{{1, 2}})); 166 VectorWriter{vch, 0, a, b}; 167 BOOST_CHECK((vch == std::vector<unsigned char>{{1, 2}})); 168 vch.clear(); 169 170 VectorWriter{vch, 2, a, b}; 171 BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 1, 2}})); 172 VectorWriter{vch, 2, a, b}; 173 BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 1, 2}})); 174 vch.clear(); 175 176 vch.resize(5, 0); 177 VectorWriter{vch, 2, a, b}; 178 BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 1, 2, 0}})); 179 VectorWriter{vch, 2, a, b}; 180 BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 1, 2, 0}})); 181 vch.clear(); 182 183 vch.resize(4, 0); 184 VectorWriter{vch, 3, a, b}; 185 BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 0, 1, 2}})); 186 VectorWriter{vch, 3, a, b}; 187 BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 0, 1, 2}})); 188 vch.clear(); 189 190 vch.resize(4, 0); 191 VectorWriter{vch, 4, a, b}; 192 BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 0, 0, 1, 2}})); 193 VectorWriter{vch, 4, a, b}; 194 BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 0, 0, 1, 2}})); 195 vch.clear(); 196 197 VectorWriter{vch, 0, bytes}; 198 BOOST_CHECK((vch == std::vector<unsigned char>{{3, 4, 5, 6}})); 199 VectorWriter{vch, 0, bytes}; 200 BOOST_CHECK((vch == std::vector<unsigned char>{{3, 4, 5, 6}})); 201 vch.clear(); 202 203 vch.resize(4, 8); 204 VectorWriter{vch, 2, a, bytes, b}; 205 BOOST_CHECK((vch == std::vector<unsigned char>{{8, 8, 1, 3, 4, 5, 6, 2}})); 206 VectorWriter{vch, 2, a, bytes, b}; 207 BOOST_CHECK((vch == std::vector<unsigned char>{{8, 8, 1, 3, 4, 5, 6, 2}})); 208 vch.clear(); 209 } 210 211 BOOST_AUTO_TEST_CASE(streams_vector_reader) 212 { 213 std::vector<unsigned char> vch = {1, 255, 3, 4, 5, 6}; 214 215 SpanReader reader{vch}; 216 BOOST_CHECK_EQUAL(reader.size(), 6U); 217 BOOST_CHECK(!reader.empty()); 218 219 // Read a single byte as an unsigned char. 220 unsigned char a; 221 reader >> a; 222 BOOST_CHECK_EQUAL(a, 1); 223 BOOST_CHECK_EQUAL(reader.size(), 5U); 224 BOOST_CHECK(!reader.empty()); 225 226 // Read a single byte as a int8_t. 227 int8_t b; 228 reader >> b; 229 BOOST_CHECK_EQUAL(b, -1); 230 BOOST_CHECK_EQUAL(reader.size(), 4U); 231 BOOST_CHECK(!reader.empty()); 232 233 // Read a 4 bytes as an unsigned int. 234 unsigned int c; 235 reader >> c; 236 BOOST_CHECK_EQUAL(c, 100992003U); // 3,4,5,6 in little-endian base-256 237 BOOST_CHECK_EQUAL(reader.size(), 0U); 238 BOOST_CHECK(reader.empty()); 239 240 // Reading after end of byte vector throws an error. 241 signed int d; 242 BOOST_CHECK_THROW(reader >> d, std::ios_base::failure); 243 244 // Read a 4 bytes as a signed int from the beginning of the buffer. 245 SpanReader new_reader{vch}; 246 new_reader >> d; 247 BOOST_CHECK_EQUAL(d, 67370753); // 1,255,3,4 in little-endian base-256 248 BOOST_CHECK_EQUAL(new_reader.size(), 2U); 249 BOOST_CHECK(!new_reader.empty()); 250 251 // Reading after end of byte vector throws an error even if the reader is 252 // not totally empty. 253 BOOST_CHECK_THROW(new_reader >> d, std::ios_base::failure); 254 } 255 256 BOOST_AUTO_TEST_CASE(streams_vector_reader_rvalue) 257 { 258 std::vector<uint8_t> data{0x82, 0xa7, 0x31}; 259 SpanReader reader{data}; 260 uint32_t varint = 0; 261 // Deserialize into r-value 262 reader >> VARINT(varint); 263 BOOST_CHECK_EQUAL(varint, 54321U); 264 BOOST_CHECK(reader.empty()); 265 } 266 267 BOOST_AUTO_TEST_CASE(bitstream_reader_writer) 268 { 269 DataStream data{}; 270 271 BitStreamWriter bit_writer{data}; 272 bit_writer.Write(0, 1); 273 bit_writer.Write(2, 2); 274 bit_writer.Write(6, 3); 275 bit_writer.Write(11, 4); 276 bit_writer.Write(1, 5); 277 bit_writer.Write(32, 6); 278 bit_writer.Write(7, 7); 279 bit_writer.Write(30497, 16); 280 bit_writer.Flush(); 281 282 DataStream data_copy{data}; 283 uint32_t serialized_int1; 284 data >> serialized_int1; 285 BOOST_CHECK_EQUAL(serialized_int1, uint32_t{0x7700C35A}); // NOTE: Serialized as LE 286 uint16_t serialized_int2; 287 data >> serialized_int2; 288 BOOST_CHECK_EQUAL(serialized_int2, uint16_t{0x1072}); // NOTE: Serialized as LE 289 290 BitStreamReader bit_reader{data_copy}; 291 BOOST_CHECK_EQUAL(bit_reader.Read(1), 0U); 292 BOOST_CHECK_EQUAL(bit_reader.Read(2), 2U); 293 BOOST_CHECK_EQUAL(bit_reader.Read(3), 6U); 294 BOOST_CHECK_EQUAL(bit_reader.Read(4), 11U); 295 BOOST_CHECK_EQUAL(bit_reader.Read(5), 1U); 296 BOOST_CHECK_EQUAL(bit_reader.Read(6), 32U); 297 BOOST_CHECK_EQUAL(bit_reader.Read(7), 7U); 298 BOOST_CHECK_EQUAL(bit_reader.Read(16), 30497U); 299 BOOST_CHECK_THROW(bit_reader.Read(8), std::ios_base::failure); 300 } 301 302 BOOST_AUTO_TEST_CASE(streams_serializedata_xor) 303 { 304 // Degenerate case 305 { 306 DataStream ds{}; 307 Obfuscation{}(ds); 308 BOOST_CHECK_EQUAL(""s, ds.str()); 309 } 310 311 { 312 const Obfuscation obfuscation{"ffffffffffffffff"_hex}; 313 314 DataStream ds{"0ff0"_hex}; 315 obfuscation(ds); 316 BOOST_CHECK_EQUAL("\xf0\x0f"s, ds.str()); 317 } 318 319 { 320 const Obfuscation obfuscation{"ff0fff0fff0fff0f"_hex}; 321 322 DataStream ds{"f00f"_hex}; 323 obfuscation(ds); 324 BOOST_CHECK_EQUAL("\x0f\x00"s, ds.str()); 325 } 326 } 327 328 BOOST_AUTO_TEST_CASE(streams_buffered_file) 329 { 330 fs::path streams_test_filename = m_args.GetDataDirBase() / "streams_test_tmp"; 331 AutoFile file{fsbridge::fopen(streams_test_filename, "w+b")}; 332 333 // The value at each offset is the offset. 334 for (uint8_t j = 0; j < 40; ++j) { 335 file << j; 336 } 337 file.seek(0, SEEK_SET); 338 339 // The buffer size (second arg) must be greater than the rewind 340 // amount (third arg). 341 try { 342 BufferedFile bfbad{file, 25, 25}; 343 BOOST_CHECK(false); 344 } catch (const std::exception& e) { 345 BOOST_CHECK(strstr(e.what(), 346 "Rewind limit must be less than buffer size") != nullptr); 347 } 348 349 // The buffer is 25 bytes, allow rewinding 10 bytes. 350 BufferedFile bf{file, 25, 10}; 351 BOOST_CHECK(!bf.eof()); 352 353 uint8_t i; 354 bf >> i; 355 BOOST_CHECK_EQUAL(i, 0); 356 bf >> i; 357 BOOST_CHECK_EQUAL(i, 1); 358 359 // After reading bytes 0 and 1, we're positioned at 2. 360 BOOST_CHECK_EQUAL(bf.GetPos(), 2U); 361 362 // Rewind to offset 0, ok (within the 10 byte window). 363 BOOST_CHECK(bf.SetPos(0)); 364 bf >> i; 365 BOOST_CHECK_EQUAL(i, 0); 366 367 // We can go forward to where we've been, but beyond may fail. 368 BOOST_CHECK(bf.SetPos(2)); 369 bf >> i; 370 BOOST_CHECK_EQUAL(i, 2); 371 372 // If you know the maximum number of bytes that should be 373 // read to deserialize the variable, you can limit the read 374 // extent. The current file offset is 3, so the following 375 // SetLimit() allows zero bytes to be read. 376 BOOST_CHECK(bf.SetLimit(3)); 377 try { 378 bf >> i; 379 BOOST_CHECK(false); 380 } catch (const std::exception& e) { 381 BOOST_CHECK(strstr(e.what(), 382 "Attempt to position past buffer limit") != nullptr); 383 } 384 // The default argument removes the limit completely. 385 BOOST_CHECK(bf.SetLimit()); 386 // The read position should still be at 3 (no change). 387 BOOST_CHECK_EQUAL(bf.GetPos(), 3U); 388 389 // Read from current offset, 3, forward until position 10. 390 for (uint8_t j = 3; j < 10; ++j) { 391 bf >> i; 392 BOOST_CHECK_EQUAL(i, j); 393 } 394 BOOST_CHECK_EQUAL(bf.GetPos(), 10U); 395 396 // We're guaranteed (just barely) to be able to rewind to zero. 397 BOOST_CHECK(bf.SetPos(0)); 398 BOOST_CHECK_EQUAL(bf.GetPos(), 0U); 399 bf >> i; 400 BOOST_CHECK_EQUAL(i, 0); 401 402 // We can set the position forward again up to the farthest 403 // into the stream we've been, but no farther. (Attempting 404 // to go farther may succeed, but it's not guaranteed.) 405 BOOST_CHECK(bf.SetPos(10)); 406 bf >> i; 407 BOOST_CHECK_EQUAL(i, 10); 408 BOOST_CHECK_EQUAL(bf.GetPos(), 11U); 409 410 // Now it's only guaranteed that we can rewind to offset 1 411 // (current read position, 11, minus rewind amount, 10). 412 BOOST_CHECK(bf.SetPos(1)); 413 BOOST_CHECK_EQUAL(bf.GetPos(), 1U); 414 bf >> i; 415 BOOST_CHECK_EQUAL(i, 1); 416 417 // We can stream into large variables, even larger than 418 // the buffer size. 419 BOOST_CHECK(bf.SetPos(11)); 420 { 421 uint8_t a[40 - 11]; 422 bf >> a; 423 for (uint8_t j = 0; j < sizeof(a); ++j) { 424 BOOST_CHECK_EQUAL(a[j], 11 + j); 425 } 426 } 427 BOOST_CHECK_EQUAL(bf.GetPos(), 40U); 428 429 // We've read the entire file, the next read should throw. 430 try { 431 bf >> i; 432 BOOST_CHECK(false); 433 } catch (const std::exception& e) { 434 BOOST_CHECK(strstr(e.what(), 435 "BufferedFile::Fill: end of file") != nullptr); 436 } 437 // Attempting to read beyond the end sets the EOF indicator. 438 BOOST_CHECK(bf.eof()); 439 440 // Still at offset 40, we can go back 10, to 30. 441 BOOST_CHECK_EQUAL(bf.GetPos(), 40U); 442 BOOST_CHECK(bf.SetPos(30)); 443 bf >> i; 444 BOOST_CHECK_EQUAL(i, 30); 445 BOOST_CHECK_EQUAL(bf.GetPos(), 31U); 446 447 // We're too far to rewind to position zero. 448 BOOST_CHECK(!bf.SetPos(0)); 449 // But we should now be positioned at least as far back as allowed 450 // by the rewind window (relative to our farthest read position, 40). 451 BOOST_CHECK(bf.GetPos() <= 30U); 452 453 BOOST_REQUIRE_EQUAL(file.fclose(), 0); 454 455 fs::remove(streams_test_filename); 456 } 457 458 BOOST_AUTO_TEST_CASE(streams_buffered_file_skip) 459 { 460 fs::path streams_test_filename = m_args.GetDataDirBase() / "streams_test_tmp"; 461 AutoFile file{fsbridge::fopen(streams_test_filename, "w+b")}; 462 // The value at each offset is the byte offset (e.g. byte 1 in the file has the value 0x01). 463 for (uint8_t j = 0; j < 40; ++j) { 464 file << j; 465 } 466 file.seek(0, SEEK_SET); 467 468 // The buffer is 25 bytes, allow rewinding 10 bytes. 469 BufferedFile bf{file, 25, 10}; 470 471 uint8_t i; 472 // This is like bf >> (7-byte-variable), in that it will cause data 473 // to be read from the file into memory, but it's not copied to us. 474 bf.SkipTo(7); 475 BOOST_CHECK_EQUAL(bf.GetPos(), 7U); 476 bf >> i; 477 BOOST_CHECK_EQUAL(i, 7); 478 479 // The bytes in the buffer up to offset 7 are valid and can be read. 480 BOOST_CHECK(bf.SetPos(0)); 481 bf >> i; 482 BOOST_CHECK_EQUAL(i, 0); 483 bf >> i; 484 BOOST_CHECK_EQUAL(i, 1); 485 486 bf.SkipTo(11); 487 bf >> i; 488 BOOST_CHECK_EQUAL(i, 11); 489 490 // SkipTo() honors the transfer limit; we can't position beyond the limit. 491 bf.SetLimit(13); 492 try { 493 bf.SkipTo(14); 494 BOOST_CHECK(false); 495 } catch (const std::exception& e) { 496 BOOST_CHECK(strstr(e.what(), "Attempt to position past buffer limit") != nullptr); 497 } 498 499 // We can position exactly to the transfer limit. 500 bf.SkipTo(13); 501 BOOST_CHECK_EQUAL(bf.GetPos(), 13U); 502 503 BOOST_REQUIRE_EQUAL(file.fclose(), 0); 504 fs::remove(streams_test_filename); 505 } 506 507 BOOST_AUTO_TEST_CASE(streams_buffered_file_rand) 508 { 509 // Make this test deterministic. 510 SeedRandomForTest(SeedRand::ZEROS); 511 512 fs::path streams_test_filename = m_args.GetDataDirBase() / "streams_test_tmp"; 513 for (int rep = 0; rep < 50; ++rep) { 514 AutoFile file{fsbridge::fopen(streams_test_filename, "w+b")}; 515 size_t fileSize = m_rng.randrange(256); 516 for (uint8_t i = 0; i < fileSize; ++i) { 517 file << i; 518 } 519 file.seek(0, SEEK_SET); 520 521 size_t bufSize = m_rng.randrange(300) + 1; 522 size_t rewindSize = m_rng.randrange(bufSize); 523 BufferedFile bf{file, bufSize, rewindSize}; 524 size_t currentPos = 0; 525 size_t maxPos = 0; 526 for (int step = 0; step < 100; ++step) { 527 if (currentPos >= fileSize) 528 break; 529 530 // We haven't read to the end of the file yet. 531 BOOST_CHECK(!bf.eof()); 532 BOOST_CHECK_EQUAL(bf.GetPos(), currentPos); 533 534 // Pretend the file consists of a series of objects of varying 535 // sizes; the boundaries of the objects can interact arbitrarily 536 // with the CBufferFile's internal buffer. These first three 537 // cases simulate objects of various sizes (1, 2, 5 bytes). 538 switch (m_rng.randrange(6)) { 539 case 0: { 540 uint8_t a[1]; 541 if (currentPos + 1 > fileSize) 542 continue; 543 bf.SetLimit(currentPos + 1); 544 bf >> a; 545 for (uint8_t i = 0; i < 1; ++i) { 546 BOOST_CHECK_EQUAL(a[i], currentPos); 547 currentPos++; 548 } 549 break; 550 } 551 case 1: { 552 uint8_t a[2]; 553 if (currentPos + 2 > fileSize) 554 continue; 555 bf.SetLimit(currentPos + 2); 556 bf >> a; 557 for (uint8_t i = 0; i < 2; ++i) { 558 BOOST_CHECK_EQUAL(a[i], currentPos); 559 currentPos++; 560 } 561 break; 562 } 563 case 2: { 564 uint8_t a[5]; 565 if (currentPos + 5 > fileSize) 566 continue; 567 bf.SetLimit(currentPos + 5); 568 bf >> a; 569 for (uint8_t i = 0; i < 5; ++i) { 570 BOOST_CHECK_EQUAL(a[i], currentPos); 571 currentPos++; 572 } 573 break; 574 } 575 case 3: { 576 // SkipTo is similar to the "read" cases above, except 577 // we don't receive the data. 578 size_t skip_length{static_cast<size_t>(m_rng.randrange(5))}; 579 if (currentPos + skip_length > fileSize) continue; 580 bf.SetLimit(currentPos + skip_length); 581 bf.SkipTo(currentPos + skip_length); 582 currentPos += skip_length; 583 break; 584 } 585 case 4: { 586 // Find a byte value (that is at or ahead of the current position). 587 size_t find = currentPos + m_rng.randrange(8); 588 if (find >= fileSize) 589 find = fileSize - 1; 590 bf.FindByte(std::byte(find)); 591 // The value at each offset is the offset. 592 BOOST_CHECK_EQUAL(bf.GetPos(), find); 593 currentPos = find; 594 595 bf.SetLimit(currentPos + 1); 596 uint8_t i; 597 bf >> i; 598 BOOST_CHECK_EQUAL(i, currentPos); 599 currentPos++; 600 break; 601 } 602 case 5: { 603 size_t requestPos = m_rng.randrange(maxPos + 4); 604 bool okay = bf.SetPos(requestPos); 605 // The new position may differ from the requested position 606 // because we may not be able to rewind beyond the rewind 607 // window, and we may not be able to move forward beyond the 608 // farthest position we've reached so far. 609 currentPos = bf.GetPos(); 610 BOOST_CHECK_EQUAL(okay, currentPos == requestPos); 611 // Check that we can position within the rewind window. 612 if (requestPos <= maxPos && 613 maxPos > rewindSize && 614 requestPos >= maxPos - rewindSize) { 615 // We requested a position within the rewind window. 616 BOOST_CHECK(okay); 617 } 618 break; 619 } 620 } 621 if (maxPos < currentPos) 622 maxPos = currentPos; 623 } 624 BOOST_REQUIRE_EQUAL(file.fclose(), 0); 625 } 626 fs::remove(streams_test_filename); 627 } 628 629 BOOST_AUTO_TEST_CASE(buffered_reader_matches_autofile_random_content) 630 { 631 const size_t file_size{1 + m_rng.randrange<size_t>(1 << 17)}; 632 const size_t buf_size{1 + m_rng.randrange(file_size)}; 633 const FlatFilePos pos{0, 0}; 634 635 const FlatFileSeq test_file{m_args.GetDataDirBase(), "buffered_file_test_random", node::BLOCKFILE_CHUNK_SIZE}; 636 const Obfuscation obfuscation{m_rng.randbytes<Obfuscation::KEY_SIZE>()}; 637 638 // Write out the file with random content 639 { 640 AutoFile f{test_file.Open(pos, /*read_only=*/false), obfuscation}; 641 f.write(m_rng.randbytes<std::byte>(file_size)); 642 BOOST_REQUIRE_EQUAL(f.fclose(), 0); 643 } 644 BOOST_CHECK_EQUAL(fs::file_size(test_file.FileName(pos)), file_size); 645 646 { 647 AutoFile direct_file{test_file.Open(pos, /*read_only=*/true), obfuscation}; 648 649 AutoFile buffered_file{test_file.Open(pos, /*read_only=*/true), obfuscation}; 650 BufferedReader buffered_reader{std::move(buffered_file), buf_size}; 651 652 for (size_t total_read{0}; total_read < file_size;) { 653 const size_t read{Assert(std::min(1 + m_rng.randrange(m_rng.randbool() ? buf_size : 2 * buf_size), file_size - total_read))}; 654 655 DataBuffer direct_file_buffer{read}; 656 direct_file.read(direct_file_buffer); 657 658 DataBuffer buffered_buffer{read}; 659 buffered_reader.read(buffered_buffer); 660 661 BOOST_CHECK_EQUAL_COLLECTIONS( 662 direct_file_buffer.begin(), direct_file_buffer.end(), 663 buffered_buffer.begin(), buffered_buffer.end() 664 ); 665 666 total_read += read; 667 } 668 669 { 670 DataBuffer excess_byte{1}; 671 BOOST_CHECK_EXCEPTION(direct_file.read(excess_byte), std::ios_base::failure, HasReason{"end of file"}); 672 } 673 674 { 675 DataBuffer excess_byte{1}; 676 BOOST_CHECK_EXCEPTION(buffered_reader.read(excess_byte), std::ios_base::failure, HasReason{"end of file"}); 677 } 678 } 679 680 fs::remove(test_file.FileName(pos)); 681 } 682 683 BOOST_AUTO_TEST_CASE(buffered_writer_matches_autofile_random_content) 684 { 685 const size_t file_size{1 + m_rng.randrange<size_t>(1 << 17)}; 686 const size_t buf_size{1 + m_rng.randrange(file_size)}; 687 const FlatFilePos pos{0, 0}; 688 689 const FlatFileSeq test_buffered{m_args.GetDataDirBase(), "buffered_write_test", node::BLOCKFILE_CHUNK_SIZE}; 690 const FlatFileSeq test_direct{m_args.GetDataDirBase(), "direct_write_test", node::BLOCKFILE_CHUNK_SIZE}; 691 const Obfuscation obfuscation{m_rng.randbytes<Obfuscation::KEY_SIZE>()}; 692 693 { 694 DataBuffer test_data{m_rng.randbytes<std::byte>(file_size)}; 695 696 AutoFile direct_file{test_direct.Open(pos, /*read_only=*/false), obfuscation}; 697 698 AutoFile buffered_file{test_buffered.Open(pos, /*read_only=*/false), obfuscation}; 699 { 700 BufferedWriter buffered{buffered_file, buf_size}; 701 702 for (size_t total_written{0}; total_written < file_size;) { 703 const size_t write_size{Assert(std::min(1 + m_rng.randrange(m_rng.randbool() ? buf_size : 2 * buf_size), file_size - total_written))}; 704 705 auto current_span = std::span{test_data}.subspan(total_written, write_size); 706 direct_file.write(current_span); 707 buffered.write(current_span); 708 709 total_written += write_size; 710 } 711 } 712 BOOST_REQUIRE_EQUAL(buffered_file.fclose(), 0); 713 BOOST_REQUIRE_EQUAL(direct_file.fclose(), 0); 714 } 715 716 // Compare the resulting files 717 DataBuffer direct_result{file_size}; 718 { 719 AutoFile verify_direct{test_direct.Open(pos, /*read_only=*/true), obfuscation}; 720 verify_direct.read(direct_result); 721 722 DataBuffer excess_byte{1}; 723 BOOST_CHECK_EXCEPTION(verify_direct.read(excess_byte), std::ios_base::failure, HasReason{"end of file"}); 724 } 725 726 DataBuffer buffered_result{file_size}; 727 { 728 AutoFile verify_buffered{test_buffered.Open(pos, /*read_only=*/true), obfuscation}; 729 verify_buffered.read(buffered_result); 730 731 DataBuffer excess_byte{1}; 732 BOOST_CHECK_EXCEPTION(verify_buffered.read(excess_byte), std::ios_base::failure, HasReason{"end of file"}); 733 } 734 735 BOOST_CHECK_EQUAL_COLLECTIONS( 736 direct_result.begin(), direct_result.end(), 737 buffered_result.begin(), buffered_result.end() 738 ); 739 740 fs::remove(test_direct.FileName(pos)); 741 fs::remove(test_buffered.FileName(pos)); 742 } 743 744 BOOST_AUTO_TEST_CASE(buffered_writer_reader) 745 { 746 const uint32_t v1{m_rng.rand32()}, v2{m_rng.rand32()}, v3{m_rng.rand32()}; 747 const fs::path test_file{m_args.GetDataDirBase() / "test_buffered_write_read.bin"}; 748 749 // Write out the values through a precisely sized BufferedWriter 750 AutoFile file{fsbridge::fopen(test_file, "w+b")}; 751 { 752 BufferedWriter f(file, sizeof(v1) + sizeof(v2) + sizeof(v3)); 753 f << v1 << v2; 754 f.write(std::as_bytes(std::span{&v3, 1})); 755 } 756 BOOST_REQUIRE_EQUAL(file.fclose(), 0); 757 758 // Read back and verify using BufferedReader 759 { 760 uint32_t _v1{0}, _v2{0}, _v3{0}; 761 AutoFile file{fsbridge::fopen(test_file, "rb")}; 762 BufferedReader f(std::move(file), sizeof(v1) + sizeof(v2) + sizeof(v3)); 763 f >> _v1 >> _v2; 764 f.read(std::as_writable_bytes(std::span{&_v3, 1})); 765 BOOST_CHECK_EQUAL(_v1, v1); 766 BOOST_CHECK_EQUAL(_v2, v2); 767 BOOST_CHECK_EQUAL(_v3, v3); 768 769 DataBuffer excess_byte{1}; 770 BOOST_CHECK_EXCEPTION(f.read(excess_byte), std::ios_base::failure, HasReason{"end of file"}); 771 } 772 773 fs::remove(test_file); 774 } 775 776 BOOST_AUTO_TEST_CASE(streams_hashed) 777 { 778 DataStream stream{}; 779 HashedSourceWriter hash_writer{stream}; 780 const std::string data{"bitcoin"}; 781 hash_writer << data; 782 783 HashVerifier hash_verifier{stream}; 784 std::string result; 785 hash_verifier >> result; 786 BOOST_CHECK_EQUAL(data, result); 787 BOOST_CHECK_EQUAL(hash_writer.GetHash(), hash_verifier.GetHash()); 788 } 789 790 BOOST_AUTO_TEST_CASE(size_preserves_position) 791 { 792 const fs::path path = m_args.GetDataDirBase() / "size_pos_test.bin"; 793 AutoFile f{fsbridge::fopen(path, "w+b")}; 794 for (uint8_t j = 0; j < 10; ++j) { 795 f << j; 796 } 797 798 // Test that usage of size() does not change the current position 799 // 800 // Case: Pos at beginning of the file 801 f.seek(0, SEEK_SET); 802 (void)f.size(); 803 uint8_t first{}; 804 f >> first; 805 BOOST_CHECK_EQUAL(first, 0); 806 807 // Case: Pos at middle of the file 808 f.seek(0, SEEK_SET); 809 // Move pos to middle 810 f.ignore(4); 811 (void)f.size(); 812 uint8_t middle{}; 813 f >> middle; 814 // Pos still at 4 815 BOOST_CHECK_EQUAL(middle, 4); 816 817 // Case: Pos at EOF 818 f.seek(0, SEEK_END); 819 (void)f.size(); 820 uint8_t end{}; 821 BOOST_CHECK_EXCEPTION(f >> end, std::ios_base::failure, HasReason{"AutoFile::read: end of file"}); 822 823 BOOST_REQUIRE_EQUAL(f.fclose(), 0); 824 fs::remove(path); 825 } 826 827 BOOST_AUTO_TEST_SUITE_END()