serialize.h
1 // Copyright (c) 2009-2010 Satoshi Nakamoto 2 // Copyright (c) 2009-present The Bitcoin Core developers 3 // Distributed under the MIT software license, see the accompanying 4 // file COPYING or http://www.opensource.org/licenses/mit-license.php. 5 6 #ifndef BITCOIN_SERIALIZE_H 7 #define BITCOIN_SERIALIZE_H 8 9 #include <attributes.h> 10 #include <compat/assumptions.h> // IWYU pragma: keep 11 #include <compat/endian.h> 12 #include <prevector.h> 13 #include <span.h> 14 15 #include <algorithm> 16 #include <concepts> 17 #include <cstdint> 18 #include <cstring> 19 #include <ios> 20 #include <limits> 21 #include <map> 22 #include <memory> 23 #include <set> 24 #include <string> 25 #include <utility> 26 #include <vector> 27 28 /** 29 * The maximum size of a serialized object in bytes or number of elements 30 * (for eg vectors) when the size is encoded as CompactSize. 31 */ 32 static constexpr uint64_t MAX_SIZE = 0x02000000; 33 34 /** Maximum amount of memory (in bytes) to allocate at once when deserializing vectors. */ 35 static const unsigned int MAX_VECTOR_ALLOCATE = 5000000; 36 37 /** 38 * Dummy data type to identify deserializing constructors. 39 * 40 * By convention, a constructor of a type T with signature 41 * 42 * template <typename Stream> T::T(deserialize_type, Stream& s) 43 * 44 * is a deserializing constructor, which builds the type by 45 * deserializing it from s. If T contains const fields, this 46 * is likely the only way to do so. 47 */ 48 struct deserialize_type {}; 49 constexpr deserialize_type deserialize {}; 50 51 /* 52 * Lowest-level serialization and conversion. 53 */ 54 template<typename Stream> inline void ser_writedata8(Stream &s, uint8_t obj) 55 { 56 s.write(std::as_bytes(std::span{&obj, 1})); 57 } 58 template<typename Stream> inline void ser_writedata16(Stream &s, uint16_t obj) 59 { 60 obj = htole16_internal(obj); 61 s.write(std::as_bytes(std::span{&obj, 1})); 62 } 63 template<typename Stream> inline void ser_writedata32(Stream &s, uint32_t obj) 64 { 65 obj = htole32_internal(obj); 66 s.write(std::as_bytes(std::span{&obj, 1})); 67 } 68 template<typename Stream> inline void ser_writedata32be(Stream &s, uint32_t obj) 69 { 70 obj = htobe32_internal(obj); 71 s.write(std::as_bytes(std::span{&obj, 1})); 72 } 73 template<typename Stream> inline void ser_writedata64(Stream &s, uint64_t obj) 74 { 75 obj = htole64_internal(obj); 76 s.write(std::as_bytes(std::span{&obj, 1})); 77 } 78 template<typename Stream> inline uint8_t ser_readdata8(Stream &s) 79 { 80 uint8_t obj; 81 s.read(std::as_writable_bytes(std::span{&obj, 1})); 82 return obj; 83 } 84 template<typename Stream> inline uint16_t ser_readdata16(Stream &s) 85 { 86 uint16_t obj; 87 s.read(std::as_writable_bytes(std::span{&obj, 1})); 88 return le16toh_internal(obj); 89 } 90 template<typename Stream> inline uint32_t ser_readdata32(Stream &s) 91 { 92 uint32_t obj; 93 s.read(std::as_writable_bytes(std::span{&obj, 1})); 94 return le32toh_internal(obj); 95 } 96 template<typename Stream> inline uint32_t ser_readdata32be(Stream &s) 97 { 98 uint32_t obj; 99 s.read(std::as_writable_bytes(std::span{&obj, 1})); 100 return be32toh_internal(obj); 101 } 102 template<typename Stream> inline uint64_t ser_readdata64(Stream &s) 103 { 104 uint64_t obj; 105 s.read(std::as_writable_bytes(std::span{&obj, 1})); 106 return le64toh_internal(obj); 107 } 108 109 110 class SizeComputer; 111 112 /** 113 * Convert any argument to a reference to X, maintaining constness. 114 * 115 * This can be used in serialization code to invoke a base class's 116 * serialization routines. 117 * 118 * Example use: 119 * class Base { ... }; 120 * class Child : public Base { 121 * int m_data; 122 * public: 123 * SERIALIZE_METHODS(Child, obj) { 124 * READWRITE(AsBase<Base>(obj), obj.m_data); 125 * } 126 * }; 127 * 128 * static_cast cannot easily be used here, as the type of Obj will be const Child& 129 * during serialization and Child& during deserialization. AsBase will convert to 130 * const Base& and Base& appropriately. 131 */ 132 template <class Out, class In> 133 Out& AsBase(In& x) 134 { 135 static_assert(std::is_base_of_v<Out, In>); 136 return x; 137 } 138 template <class Out, class In> 139 const Out& AsBase(const In& x) 140 { 141 static_assert(std::is_base_of_v<Out, In>); 142 return x; 143 } 144 145 #define READWRITE(...) (ser_action.SerReadWriteMany(s, __VA_ARGS__)) 146 #define SER_READ(obj, code) ser_action.SerRead(s, obj, [&](Stream& s, std::remove_const_t<Type>& obj) { code; }) 147 #define SER_WRITE(obj, code) ser_action.SerWrite(s, obj, [&](Stream& s, const Type& obj) { code; }) 148 149 /** 150 * Implement the Ser and Unser methods needed for implementing a formatter (see Using below). 151 * 152 * Both Ser and Unser are delegated to a single static method SerializationOps, which is polymorphic 153 * in the serialized/deserialized type (allowing it to be const when serializing, and non-const when 154 * deserializing). 155 * 156 * Example use: 157 * struct FooFormatter { 158 * FORMATTER_METHODS(Class, obj) { READWRITE(obj.val1, VARINT(obj.val2)); } 159 * } 160 * would define a class FooFormatter that defines a serialization of Class objects consisting 161 * of serializing its val1 member using the default serialization, and its val2 member using 162 * VARINT serialization. That FooFormatter can then be used in statements like 163 * READWRITE(Using<FooFormatter>(obj.bla)). 164 */ 165 #define FORMATTER_METHODS(cls, obj) \ 166 template<typename Stream> \ 167 static void Ser(Stream& s, const cls& obj) { SerializationOps(obj, s, ActionSerialize{}); } \ 168 template<typename Stream> \ 169 static void Unser(Stream& s, cls& obj) { SerializationOps(obj, s, ActionUnserialize{}); } \ 170 template<typename Stream, typename Type, typename Operation> \ 171 static void SerializationOps(Type& obj, Stream& s, Operation ser_action) 172 173 /** 174 * Formatter methods can retrieve parameters attached to a stream using the 175 * SER_PARAMS(type) macro as long as the stream is created directly or 176 * indirectly with a parameter of that type. This permits making serialization 177 * depend on run-time context in a type-safe way. 178 * 179 * Example use: 180 * struct BarParameter { bool fancy; ... }; 181 * struct Bar { ... }; 182 * struct FooFormatter { 183 * FORMATTER_METHODS(Bar, obj) { 184 * auto& param = SER_PARAMS(BarParameter); 185 * if (param.fancy) { 186 * READWRITE(VARINT(obj.value)); 187 * } else { 188 * READWRITE(obj.value); 189 * } 190 * } 191 * }; 192 * which would then be invoked as 193 * READWRITE(BarParameter{...}(Using<FooFormatter>(obj.foo))) 194 * 195 * parameter(obj) can be invoked anywhere in the call stack; it is 196 * passed down recursively into all serialization code, until another 197 * serialization parameter overrides it. 198 * 199 * Parameters will be implicitly converted where appropriate. This means that 200 * "parent" serialization code can use a parameter that derives from, or is 201 * convertible to, a "child" formatter's parameter type. 202 * 203 * Compilation will fail in any context where serialization is invoked but 204 * no parameter of a type convertible to BarParameter is provided. 205 */ 206 #define SER_PARAMS(type) (s.template GetParams<type>()) 207 208 #define BASE_SERIALIZE_METHODS(cls) \ 209 template <typename Stream> \ 210 void Serialize(Stream& s) const \ 211 { \ 212 static_assert(std::is_same_v<const cls&, decltype(*this)>, "Serialize type mismatch"); \ 213 Ser(s, *this); \ 214 } \ 215 template <typename Stream> \ 216 void Unserialize(Stream& s) \ 217 { \ 218 static_assert(std::is_same_v<cls&, decltype(*this)>, "Unserialize type mismatch"); \ 219 Unser(s, *this); \ 220 } 221 222 /** 223 * Implement the Serialize and Unserialize methods by delegating to a single templated 224 * static method that takes the to-be-(de)serialized object as a parameter. This approach 225 * has the advantage that the constness of the object becomes a template parameter, and 226 * thus allows a single implementation that sees the object as const for serializing 227 * and non-const for deserializing, without casts. 228 */ 229 #define SERIALIZE_METHODS(cls, obj) \ 230 BASE_SERIALIZE_METHODS(cls) \ 231 FORMATTER_METHODS(cls, obj) 232 233 // Templates for serializing to anything that looks like a stream, 234 // i.e. anything that supports .read(std::span<std::byte>) and .write(std::span<const std::byte>) 235 // 236 237 // Typically int8_t and char are distinct types, but some systems may define int8_t 238 // in terms of char. Forbid serialization of char in the typical case, but allow it if 239 // it's the only way to describe an int8_t. 240 template<class T> 241 concept CharNotInt8 = std::same_as<T, char> && !std::same_as<T, int8_t>; 242 243 // clang-format off 244 template <typename Stream, CharNotInt8 V> void Serialize(Stream&, V) = delete; // char serialization forbidden. Use uint8_t or int8_t 245 template <typename Stream> void Serialize(Stream& s, std::byte a) { ser_writedata8(s, uint8_t(a)); } 246 template <typename Stream> void Serialize(Stream& s, int8_t a) { ser_writedata8(s, uint8_t(a)); } 247 template <typename Stream> void Serialize(Stream& s, uint8_t a) { ser_writedata8(s, a); } 248 template <typename Stream> void Serialize(Stream& s, int16_t a) { ser_writedata16(s, uint16_t(a)); } 249 template <typename Stream> void Serialize(Stream& s, uint16_t a) { ser_writedata16(s, a); } 250 template <typename Stream> void Serialize(Stream& s, int32_t a) { ser_writedata32(s, uint32_t(a)); } 251 template <typename Stream> void Serialize(Stream& s, uint32_t a) { ser_writedata32(s, a); } 252 template <typename Stream> void Serialize(Stream& s, int64_t a) { ser_writedata64(s, uint64_t(a)); } 253 template <typename Stream> void Serialize(Stream& s, uint64_t a) { ser_writedata64(s, a); } 254 255 template <typename Stream, BasicByte B, size_t N> void Serialize(Stream& s, const B (&a)[N]) { s.write(MakeByteSpan(a)); } 256 template <typename Stream, BasicByte B, size_t N> void Serialize(Stream& s, const std::array<B, N>& a) { s.write(MakeByteSpan(a)); } 257 template <typename Stream, BasicByte B, size_t N> void Serialize(Stream& s, std::span<B, N> span) { s.write(std::as_bytes(span)); } 258 template <typename Stream, BasicByte B> void Serialize(Stream& s, std::span<B> span) { s.write(std::as_bytes(span)); } 259 260 template <typename Stream, CharNotInt8 V> void Unserialize(Stream&, V) = delete; // char serialization forbidden. Use uint8_t or int8_t 261 template <typename Stream> void Unserialize(Stream& s, std::byte& a) { a = std::byte(ser_readdata8(s)); } 262 template <typename Stream> void Unserialize(Stream& s, int8_t& a) { a = int8_t(ser_readdata8(s)); } 263 template <typename Stream> void Unserialize(Stream& s, uint8_t& a) { a = ser_readdata8(s); } 264 template <typename Stream> void Unserialize(Stream& s, int16_t& a) { a = int16_t(ser_readdata16(s)); } 265 template <typename Stream> void Unserialize(Stream& s, uint16_t& a) { a = ser_readdata16(s); } 266 template <typename Stream> void Unserialize(Stream& s, int32_t& a) { a = int32_t(ser_readdata32(s)); } 267 template <typename Stream> void Unserialize(Stream& s, uint32_t& a) { a = ser_readdata32(s); } 268 template <typename Stream> void Unserialize(Stream& s, int64_t& a) { a = int64_t(ser_readdata64(s)); } 269 template <typename Stream> void Unserialize(Stream& s, uint64_t& a) { a = ser_readdata64(s); } 270 271 template <typename Stream, BasicByte B, size_t N> void Unserialize(Stream& s, B (&a)[N]) { s.read(MakeWritableByteSpan(a)); } 272 template <typename Stream, BasicByte B, size_t N> void Unserialize(Stream& s, std::array<B, N>& a) { s.read(MakeWritableByteSpan(a)); } 273 template <typename Stream, BasicByte B, size_t N> void Unserialize(Stream& s, std::span<B, N> span) { s.read(std::as_writable_bytes(span)); } 274 template <typename Stream, BasicByte B> void Unserialize(Stream& s, std::span<B> span) { s.read(std::as_writable_bytes(span)); } 275 276 template <typename Stream> void Serialize(Stream& s, bool a) { uint8_t f = a; ser_writedata8(s, f); } 277 template <typename Stream> void Unserialize(Stream& s, bool& a) { uint8_t f = ser_readdata8(s); a = f; } 278 // clang-format on 279 280 281 /** 282 * Compact Size 283 * size < 253 -- 1 byte 284 * size <= USHRT_MAX -- 3 bytes (253 + 2 bytes) 285 * size <= UINT_MAX -- 5 bytes (254 + 4 bytes) 286 * size > UINT_MAX -- 9 bytes (255 + 8 bytes) 287 */ 288 constexpr inline unsigned int GetSizeOfCompactSize(uint64_t nSize) 289 { 290 if (nSize < 253) return sizeof(unsigned char); 291 else if (nSize <= std::numeric_limits<uint16_t>::max()) return sizeof(unsigned char) + sizeof(uint16_t); 292 else if (nSize <= std::numeric_limits<unsigned int>::max()) return sizeof(unsigned char) + sizeof(unsigned int); 293 else return sizeof(unsigned char) + sizeof(uint64_t); 294 } 295 296 inline void WriteCompactSize(SizeComputer& os, uint64_t nSize); 297 298 template<typename Stream> 299 void WriteCompactSize(Stream& os, uint64_t nSize) 300 { 301 if (nSize < 253) 302 { 303 ser_writedata8(os, nSize); 304 } 305 else if (nSize <= std::numeric_limits<uint16_t>::max()) 306 { 307 ser_writedata8(os, 253); 308 ser_writedata16(os, nSize); 309 } 310 else if (nSize <= std::numeric_limits<unsigned int>::max()) 311 { 312 ser_writedata8(os, 254); 313 ser_writedata32(os, nSize); 314 } 315 else 316 { 317 ser_writedata8(os, 255); 318 ser_writedata64(os, nSize); 319 } 320 return; 321 } 322 323 /** 324 * Decode a CompactSize-encoded variable-length integer. 325 * 326 * As these are primarily used to encode the size of vector-like serializations, by default a range 327 * check is performed. When used as a generic number encoding, range_check should be set to false. 328 */ 329 template<typename Stream> 330 uint64_t ReadCompactSize(Stream& is, bool range_check = true) 331 { 332 uint8_t chSize = ser_readdata8(is); 333 uint64_t nSizeRet = 0; 334 if (chSize < 253) 335 { 336 nSizeRet = chSize; 337 } 338 else if (chSize == 253) 339 { 340 nSizeRet = ser_readdata16(is); 341 if (nSizeRet < 253) 342 throw std::ios_base::failure("non-canonical ReadCompactSize()"); 343 } 344 else if (chSize == 254) 345 { 346 nSizeRet = ser_readdata32(is); 347 if (nSizeRet < 0x10000u) 348 throw std::ios_base::failure("non-canonical ReadCompactSize()"); 349 } 350 else 351 { 352 nSizeRet = ser_readdata64(is); 353 if (nSizeRet < 0x100000000ULL) 354 throw std::ios_base::failure("non-canonical ReadCompactSize()"); 355 } 356 if (range_check && nSizeRet > MAX_SIZE) { 357 throw std::ios_base::failure("ReadCompactSize(): size too large"); 358 } 359 return nSizeRet; 360 } 361 362 /** 363 * Variable-length integers: bytes are a MSB base-128 encoding of the number. 364 * The high bit in each byte signifies whether another digit follows. To make 365 * sure the encoding is one-to-one, one is subtracted from all but the last digit. 366 * Thus, the byte sequence a[] with length len, where all but the last byte 367 * has bit 128 set, encodes the number: 368 * 369 * (a[len-1] & 0x7F) + sum(i=1..len-1, 128^i*((a[len-i-1] & 0x7F)+1)) 370 * 371 * Properties: 372 * * Very small (0-127: 1 byte, 128-16511: 2 bytes, 16512-2113663: 3 bytes) 373 * * Every integer has exactly one encoding 374 * * Encoding does not depend on size of original integer type 375 * * No redundancy: every (infinite) byte sequence corresponds to a list 376 * of encoded integers. 377 * 378 * 0: [0x00] 256: [0x81 0x00] 379 * 1: [0x01] 16383: [0xFE 0x7F] 380 * 127: [0x7F] 16384: [0xFF 0x00] 381 * 128: [0x80 0x00] 16511: [0xFF 0x7F] 382 * 255: [0x80 0x7F] 65535: [0x82 0xFE 0x7F] 383 * 2^32: [0x8E 0xFE 0xFE 0xFF 0x00] 384 */ 385 386 /** 387 * Mode for encoding VarInts. 388 * 389 * Currently there is no support for signed encodings. The default mode will not 390 * compile with signed values, and the legacy "nonnegative signed" mode will 391 * accept signed values, but improperly encode and decode them if they are 392 * negative. In the future, the DEFAULT mode could be extended to support 393 * negative numbers in a backwards compatible way, and additional modes could be 394 * added to support different varint formats (e.g. zigzag encoding). 395 */ 396 enum class VarIntMode { DEFAULT, NONNEGATIVE_SIGNED }; 397 398 template <VarIntMode Mode, typename I> 399 struct CheckVarIntMode { 400 constexpr CheckVarIntMode() 401 { 402 static_assert(Mode != VarIntMode::DEFAULT || std::is_unsigned_v<I>, "Unsigned type required with mode DEFAULT."); 403 static_assert(Mode != VarIntMode::NONNEGATIVE_SIGNED || std::is_signed_v<I>, "Signed type required with mode NONNEGATIVE_SIGNED."); 404 } 405 }; 406 407 template<VarIntMode Mode, typename I> 408 inline unsigned int GetSizeOfVarInt(I n) 409 { 410 CheckVarIntMode<Mode, I>(); 411 int nRet = 0; 412 while(true) { 413 nRet++; 414 if (n <= 0x7F) 415 break; 416 n = (n >> 7) - 1; 417 } 418 return nRet; 419 } 420 421 template<typename I> 422 inline void WriteVarInt(SizeComputer& os, I n); 423 424 template<typename Stream, VarIntMode Mode, typename I> 425 void WriteVarInt(Stream& os, I n) 426 { 427 CheckVarIntMode<Mode, I>(); 428 unsigned char tmp[(sizeof(n)*8+6)/7]; 429 int len=0; 430 while(true) { 431 tmp[len] = (n & 0x7F) | (len ? 0x80 : 0x00); 432 if (n <= 0x7F) 433 break; 434 n = (n >> 7) - 1; 435 len++; 436 } 437 do { 438 ser_writedata8(os, tmp[len]); 439 } while(len--); 440 } 441 442 template<typename Stream, VarIntMode Mode, typename I> 443 I ReadVarInt(Stream& is) 444 { 445 CheckVarIntMode<Mode, I>(); 446 I n = 0; 447 while(true) { 448 unsigned char chData = ser_readdata8(is); 449 if (n > (std::numeric_limits<I>::max() >> 7)) { 450 throw std::ios_base::failure("ReadVarInt(): size too large"); 451 } 452 n = (n << 7) | (chData & 0x7F); 453 if (chData & 0x80) { 454 if (n == std::numeric_limits<I>::max()) { 455 throw std::ios_base::failure("ReadVarInt(): size too large"); 456 } 457 n++; 458 } else { 459 return n; 460 } 461 } 462 } 463 464 /** Simple wrapper class to serialize objects using a formatter; used by Using(). */ 465 template<typename Formatter, typename T> 466 class Wrapper 467 { 468 static_assert(std::is_lvalue_reference_v<T>, "Wrapper needs an lvalue reference type T"); 469 protected: 470 T m_object; 471 public: 472 explicit Wrapper(T obj) : m_object(obj) {} 473 template<typename Stream> void Serialize(Stream &s) const { Formatter().Ser(s, m_object); } 474 template<typename Stream> void Unserialize(Stream &s) { Formatter().Unser(s, m_object); } 475 }; 476 477 /** Cause serialization/deserialization of an object to be done using a specified formatter class. 478 * 479 * To use this, you need a class Formatter that has public functions Ser(stream, const object&) for 480 * serialization, and Unser(stream, object&) for deserialization. Serialization routines (inside 481 * READWRITE, or directly with << and >> operators), can then use Using<Formatter>(object). 482 * 483 * This works by constructing a Wrapper<Formatter, T>-wrapped version of object, where T is 484 * const during serialization, and non-const during deserialization, which maintains const 485 * correctness. 486 */ 487 template<typename Formatter, typename T> 488 static inline Wrapper<Formatter, T&> Using(T&& t) { return Wrapper<Formatter, T&>(t); } 489 490 #define VARINT_MODE(obj, mode) Using<VarIntFormatter<mode>>(obj) 491 #define VARINT(obj) Using<VarIntFormatter<VarIntMode::DEFAULT>>(obj) 492 #define COMPACTSIZE(obj) Using<CompactSizeFormatter<true>>(obj) 493 #define LIMITED_STRING(obj,n) Using<LimitedStringFormatter<n>>(obj) 494 495 /** Serialization wrapper class for integers in VarInt format. */ 496 template<VarIntMode Mode> 497 struct VarIntFormatter 498 { 499 template<typename Stream, typename I> void Ser(Stream &s, I v) 500 { 501 WriteVarInt<Stream,Mode, std::remove_cv_t<I>>(s, v); 502 } 503 504 template<typename Stream, typename I> void Unser(Stream& s, I& v) 505 { 506 v = ReadVarInt<Stream,Mode, std::remove_cv_t<I>>(s); 507 } 508 }; 509 510 /** Serialization wrapper class for custom integers and enums. 511 * 512 * It permits specifying the serialized size (1 to 8 bytes) and endianness. 513 * 514 * Use the big endian mode for values that are stored in memory in native 515 * byte order, but serialized in big endian notation. This is only intended 516 * to implement serializers that are compatible with existing formats, and 517 * its use is not recommended for new data structures. 518 */ 519 template<int Bytes, bool BigEndian = false> 520 struct CustomUintFormatter 521 { 522 static_assert(Bytes > 0 && Bytes <= 8, "CustomUintFormatter Bytes out of range"); 523 static constexpr uint64_t MAX = 0xffffffffffffffff >> (8 * (8 - Bytes)); 524 525 template <typename Stream, typename I> void Ser(Stream& s, I v) 526 { 527 if (v < 0 || v > MAX) throw std::ios_base::failure("CustomUintFormatter value out of range"); 528 if (BigEndian) { 529 uint64_t raw = htobe64_internal(v); 530 s.write(std::as_bytes(std::span{&raw, 1}).last(Bytes)); 531 } else { 532 uint64_t raw = htole64_internal(v); 533 s.write(std::as_bytes(std::span{&raw, 1}).first(Bytes)); 534 } 535 } 536 537 template <typename Stream, typename I> void Unser(Stream& s, I& v) 538 { 539 using U = typename std::conditional_t<std::is_enum_v<I>, std::underlying_type<I>, std::common_type<I>>::type; 540 static_assert(std::numeric_limits<U>::max() >= MAX && std::numeric_limits<U>::min() <= 0, "Assigned type too small"); 541 uint64_t raw = 0; 542 if (BigEndian) { 543 s.read(std::as_writable_bytes(std::span{&raw, 1}).last(Bytes)); 544 v = static_cast<I>(be64toh_internal(raw)); 545 } else { 546 s.read(std::as_writable_bytes(std::span{&raw, 1}).first(Bytes)); 547 v = static_cast<I>(le64toh_internal(raw)); 548 } 549 } 550 }; 551 552 template<int Bytes> using BigEndianFormatter = CustomUintFormatter<Bytes, true>; 553 554 /** Formatter for integers in CompactSize format. */ 555 template<bool RangeCheck> 556 struct CompactSizeFormatter 557 { 558 template<typename Stream, typename I> 559 void Unser(Stream& s, I& v) 560 { 561 uint64_t n = ReadCompactSize<Stream>(s, RangeCheck); 562 if (n < std::numeric_limits<I>::min() || n > std::numeric_limits<I>::max()) { 563 throw std::ios_base::failure("CompactSize exceeds limit of type"); 564 } 565 v = n; 566 } 567 568 template<typename Stream, typename I> 569 void Ser(Stream& s, I v) 570 { 571 static_assert(std::is_unsigned_v<I>, "CompactSize only supported for unsigned integers"); 572 static_assert(std::numeric_limits<I>::max() <= std::numeric_limits<uint64_t>::max(), "CompactSize only supports 64-bit integers and below"); 573 574 WriteCompactSize<Stream>(s, v); 575 } 576 }; 577 578 template <typename U, bool LOSSY = false> 579 struct ChronoFormatter { 580 template <typename Stream, typename Tp> 581 void Unser(Stream& s, Tp& tp) 582 { 583 U u; 584 s >> u; 585 // Lossy deserialization does not make sense, so force Wnarrowing 586 tp = Tp{typename Tp::duration{typename Tp::duration::rep{u}}}; 587 } 588 template <typename Stream, typename Tp> 589 void Ser(Stream& s, Tp tp) 590 { 591 if constexpr (LOSSY) { 592 s << U(tp.time_since_epoch().count()); 593 } else { 594 s << U{tp.time_since_epoch().count()}; 595 } 596 } 597 }; 598 template <typename U> 599 using LossyChronoFormatter = ChronoFormatter<U, true>; 600 601 class CompactSizeWriter 602 { 603 protected: 604 uint64_t n; 605 public: 606 explicit CompactSizeWriter(uint64_t n_in) : n(n_in) { } 607 608 template<typename Stream> 609 void Serialize(Stream &s) const { 610 WriteCompactSize<Stream>(s, n); 611 } 612 }; 613 614 template<size_t Limit> 615 struct LimitedStringFormatter 616 { 617 template<typename Stream> 618 void Unser(Stream& s, std::string& v) 619 { 620 size_t size = ReadCompactSize(s); 621 if (size > Limit) { 622 throw std::ios_base::failure("String length limit exceeded"); 623 } 624 v.resize(size); 625 if (size != 0) s.read(MakeWritableByteSpan(v)); 626 } 627 628 template<typename Stream> 629 void Ser(Stream& s, const std::string& v) 630 { 631 s << v; 632 } 633 }; 634 635 /** Formatter to serialize/deserialize vector elements using another formatter 636 * 637 * Example: 638 * struct X { 639 * std::vector<uint64_t> v; 640 * SERIALIZE_METHODS(X, obj) { READWRITE(Using<VectorFormatter<VarInt>>(obj.v)); } 641 * }; 642 * will define a struct that contains a vector of uint64_t, which is serialized 643 * as a vector of VarInt-encoded integers. 644 * 645 * V is not required to be an std::vector type. It works for any class that 646 * exposes a value_type, size, reserve, emplace_back, back, and const iterators. 647 */ 648 template<class Formatter> 649 struct VectorFormatter 650 { 651 template<typename Stream, typename V> 652 void Ser(Stream& s, const V& v) 653 { 654 Formatter formatter; 655 WriteCompactSize(s, v.size()); 656 for (const typename V::value_type& elem : v) { 657 formatter.Ser(s, elem); 658 } 659 } 660 661 template<typename Stream, typename V> 662 void Unser(Stream& s, V& v) 663 { 664 Formatter formatter; 665 v.clear(); 666 size_t size = ReadCompactSize(s); 667 size_t allocated = 0; 668 while (allocated < size) { 669 // For DoS prevention, do not blindly allocate as much as the stream claims to contain. 670 // Instead, allocate in 5MiB batches, so that an attacker actually needs to provide 671 // X MiB of data to make us allocate X+5 Mib. 672 static_assert(sizeof(typename V::value_type) <= MAX_VECTOR_ALLOCATE, "Vector element size too large"); 673 allocated = std::min(size, allocated + MAX_VECTOR_ALLOCATE / sizeof(typename V::value_type)); 674 v.reserve(allocated); 675 while (v.size() < allocated) { 676 v.emplace_back(); 677 formatter.Unser(s, v.back()); 678 } 679 } 680 }; 681 }; 682 683 /** 684 * Forward declarations 685 */ 686 687 /** 688 * string 689 */ 690 template<typename Stream, typename C> void Serialize(Stream& os, const std::basic_string<C>& str); 691 template<typename Stream, typename C> void Unserialize(Stream& is, std::basic_string<C>& str); 692 693 /** 694 * prevector 695 */ 696 template<typename Stream, unsigned int N, typename T> inline void Serialize(Stream& os, const prevector<N, T>& v); 697 template<typename Stream, unsigned int N, typename T> inline void Unserialize(Stream& is, prevector<N, T>& v); 698 699 /** 700 * vector 701 */ 702 template<typename Stream, typename T, typename A> inline void Serialize(Stream& os, const std::vector<T, A>& v); 703 template<typename Stream, typename T, typename A> inline void Unserialize(Stream& is, std::vector<T, A>& v); 704 705 /** 706 * pair 707 */ 708 template<typename Stream, typename K, typename T> void Serialize(Stream& os, const std::pair<K, T>& item); 709 template<typename Stream, typename K, typename T> void Unserialize(Stream& is, std::pair<K, T>& item); 710 711 /** 712 * map 713 */ 714 template<typename Stream, typename K, typename T, typename Pred, typename A> void Serialize(Stream& os, const std::map<K, T, Pred, A>& m); 715 template<typename Stream, typename K, typename T, typename Pred, typename A> void Unserialize(Stream& is, std::map<K, T, Pred, A>& m); 716 717 /** 718 * set 719 */ 720 template<typename Stream, typename K, typename Pred, typename A> void Serialize(Stream& os, const std::set<K, Pred, A>& m); 721 template<typename Stream, typename K, typename Pred, typename A> void Unserialize(Stream& is, std::set<K, Pred, A>& m); 722 723 /** 724 * shared_ptr 725 */ 726 template<typename Stream, typename T> void Serialize(Stream& os, const std::shared_ptr<const T>& p); 727 template<typename Stream, typename T> void Unserialize(Stream& os, std::shared_ptr<const T>& p); 728 729 /** 730 * unique_ptr 731 */ 732 template<typename Stream, typename T> void Serialize(Stream& os, const std::unique_ptr<const T>& p); 733 template<typename Stream, typename T> void Unserialize(Stream& os, std::unique_ptr<const T>& p); 734 735 736 /** 737 * If none of the specialized versions above matched, default to calling member function. 738 */ 739 template <class T, class Stream> 740 concept Serializable = requires(T a, Stream s) { a.Serialize(s); }; 741 template <typename Stream, typename T> 742 requires Serializable<T, Stream> 743 void Serialize(Stream& os, const T& a) 744 { 745 a.Serialize(os); 746 } 747 748 template <class T, class Stream> 749 concept Unserializable = requires(T a, Stream s) { a.Unserialize(s); }; 750 template <typename Stream, typename T> 751 requires Unserializable<T, Stream> 752 void Unserialize(Stream& is, T&& a) 753 { 754 a.Unserialize(is); 755 } 756 757 /** Default formatter. Serializes objects as themselves. 758 * 759 * The vector/prevector serialization code passes this to VectorFormatter 760 * to enable reusing that logic. It shouldn't be needed elsewhere. 761 */ 762 struct DefaultFormatter 763 { 764 template<typename Stream, typename T> 765 static void Ser(Stream& s, const T& t) { Serialize(s, t); } 766 767 template<typename Stream, typename T> 768 static void Unser(Stream& s, T& t) { Unserialize(s, t); } 769 }; 770 771 772 773 774 775 /** 776 * string 777 */ 778 template<typename Stream, typename C> 779 void Serialize(Stream& os, const std::basic_string<C>& str) 780 { 781 WriteCompactSize(os, str.size()); 782 if (!str.empty()) 783 os.write(MakeByteSpan(str)); 784 } 785 786 template<typename Stream, typename C> 787 void Unserialize(Stream& is, std::basic_string<C>& str) 788 { 789 unsigned int nSize = ReadCompactSize(is); 790 str.resize(nSize); 791 if (nSize != 0) 792 is.read(MakeWritableByteSpan(str)); 793 } 794 795 796 797 /** 798 * prevector 799 */ 800 template <typename Stream, unsigned int N, typename T> 801 void Serialize(Stream& os, const prevector<N, T>& v) 802 { 803 if constexpr (BasicByte<T>) { // Use optimized version for unformatted basic bytes 804 WriteCompactSize(os, v.size()); 805 if (!v.empty()) os.write(MakeByteSpan(v)); 806 } else { 807 Serialize(os, Using<VectorFormatter<DefaultFormatter>>(v)); 808 } 809 } 810 811 812 template <typename Stream, unsigned int N, typename T> 813 void Unserialize(Stream& is, prevector<N, T>& v) 814 { 815 if constexpr (BasicByte<T>) { // Use optimized version for unformatted basic bytes 816 // Limit size per read so bogus size value won't cause out of memory 817 v.clear(); 818 unsigned int nSize = ReadCompactSize(is); 819 unsigned int i = 0; 820 while (i < nSize) { 821 unsigned int blk = std::min(nSize - i, (unsigned int)(1 + 4999999 / sizeof(T))); 822 v.resize_uninitialized(i + blk); 823 is.read(std::as_writable_bytes(std::span{&v[i], blk})); 824 i += blk; 825 } 826 } else { 827 Unserialize(is, Using<VectorFormatter<DefaultFormatter>>(v)); 828 } 829 } 830 831 832 /** 833 * vector 834 */ 835 template <typename Stream, typename T, typename A> 836 void Serialize(Stream& os, const std::vector<T, A>& v) 837 { 838 if constexpr (BasicByte<T>) { // Use optimized version for unformatted basic bytes 839 WriteCompactSize(os, v.size()); 840 if (!v.empty()) os.write(MakeByteSpan(v)); 841 } else if constexpr (std::is_same_v<T, bool>) { 842 // A special case for std::vector<bool>, as dereferencing 843 // std::vector<bool>::const_iterator does not result in a const bool& 844 // due to std::vector's special casing for bool arguments. 845 WriteCompactSize(os, v.size()); 846 for (bool elem : v) { 847 ::Serialize(os, elem); 848 } 849 } else { 850 Serialize(os, Using<VectorFormatter<DefaultFormatter>>(v)); 851 } 852 } 853 854 855 template <typename Stream, typename T, typename A> 856 void Unserialize(Stream& is, std::vector<T, A>& v) 857 { 858 if constexpr (BasicByte<T>) { // Use optimized version for unformatted basic bytes 859 // Limit size per read so bogus size value won't cause out of memory 860 v.clear(); 861 unsigned int nSize = ReadCompactSize(is); 862 unsigned int i = 0; 863 while (i < nSize) { 864 unsigned int blk = std::min(nSize - i, (unsigned int)(1 + 4999999 / sizeof(T))); 865 v.resize(i + blk); 866 is.read(std::as_writable_bytes(std::span{&v[i], blk})); 867 i += blk; 868 } 869 } else { 870 Unserialize(is, Using<VectorFormatter<DefaultFormatter>>(v)); 871 } 872 } 873 874 875 /** 876 * pair 877 */ 878 template<typename Stream, typename K, typename T> 879 void Serialize(Stream& os, const std::pair<K, T>& item) 880 { 881 Serialize(os, item.first); 882 Serialize(os, item.second); 883 } 884 885 template<typename Stream, typename K, typename T> 886 void Unserialize(Stream& is, std::pair<K, T>& item) 887 { 888 Unserialize(is, item.first); 889 Unserialize(is, item.second); 890 } 891 892 893 894 /** 895 * map 896 */ 897 template<typename Stream, typename K, typename T, typename Pred, typename A> 898 void Serialize(Stream& os, const std::map<K, T, Pred, A>& m) 899 { 900 WriteCompactSize(os, m.size()); 901 for (const auto& entry : m) 902 Serialize(os, entry); 903 } 904 905 template<typename Stream, typename K, typename T, typename Pred, typename A> 906 void Unserialize(Stream& is, std::map<K, T, Pred, A>& m) 907 { 908 m.clear(); 909 unsigned int nSize = ReadCompactSize(is); 910 typename std::map<K, T, Pred, A>::iterator mi = m.begin(); 911 for (unsigned int i = 0; i < nSize; i++) 912 { 913 std::pair<K, T> item; 914 Unserialize(is, item); 915 mi = m.insert(mi, item); 916 } 917 } 918 919 920 921 /** 922 * set 923 */ 924 template<typename Stream, typename K, typename Pred, typename A> 925 void Serialize(Stream& os, const std::set<K, Pred, A>& m) 926 { 927 WriteCompactSize(os, m.size()); 928 for (typename std::set<K, Pred, A>::const_iterator it = m.begin(); it != m.end(); ++it) 929 Serialize(os, (*it)); 930 } 931 932 template<typename Stream, typename K, typename Pred, typename A> 933 void Unserialize(Stream& is, std::set<K, Pred, A>& m) 934 { 935 m.clear(); 936 unsigned int nSize = ReadCompactSize(is); 937 typename std::set<K, Pred, A>::iterator it = m.begin(); 938 for (unsigned int i = 0; i < nSize; i++) 939 { 940 K key; 941 Unserialize(is, key); 942 it = m.insert(it, key); 943 } 944 } 945 946 947 948 /** 949 * unique_ptr 950 */ 951 template<typename Stream, typename T> void 952 Serialize(Stream& os, const std::unique_ptr<const T>& p) 953 { 954 Serialize(os, *p); 955 } 956 957 template<typename Stream, typename T> 958 void Unserialize(Stream& is, std::unique_ptr<const T>& p) 959 { 960 p.reset(new T(deserialize, is)); 961 } 962 963 964 965 /** 966 * shared_ptr 967 */ 968 template<typename Stream, typename T> void 969 Serialize(Stream& os, const std::shared_ptr<const T>& p) 970 { 971 Serialize(os, *p); 972 } 973 974 template<typename Stream, typename T> 975 void Unserialize(Stream& is, std::shared_ptr<const T>& p) 976 { 977 p = std::make_shared<const T>(deserialize, is); 978 } 979 980 /** 981 * Support for (un)serializing many things at once 982 */ 983 984 template <typename Stream, typename... Args> 985 void SerializeMany(Stream& s, const Args&... args) 986 { 987 (::Serialize(s, args), ...); 988 } 989 990 template <typename Stream, typename... Args> 991 inline void UnserializeMany(Stream& s, Args&&... args) 992 { 993 (::Unserialize(s, args), ...); 994 } 995 996 /** 997 * Support for all macros providing or using the ser_action parameter of the SerializationOps method. 998 */ 999 struct ActionSerialize { 1000 static constexpr bool ForRead() { return false; } 1001 1002 template<typename Stream, typename... Args> 1003 static void SerReadWriteMany(Stream& s, const Args&... args) 1004 { 1005 ::SerializeMany(s, args...); 1006 } 1007 1008 template<typename Stream, typename Type, typename Fn> 1009 static void SerRead(Stream& s, Type&&, Fn&&) 1010 { 1011 } 1012 1013 template<typename Stream, typename Type, typename Fn> 1014 static void SerWrite(Stream& s, Type&& obj, Fn&& fn) 1015 { 1016 fn(s, std::forward<Type>(obj)); 1017 } 1018 }; 1019 struct ActionUnserialize { 1020 static constexpr bool ForRead() { return true; } 1021 1022 template<typename Stream, typename... Args> 1023 static void SerReadWriteMany(Stream& s, Args&&... args) 1024 { 1025 ::UnserializeMany(s, args...); 1026 } 1027 1028 template<typename Stream, typename Type, typename Fn> 1029 static void SerRead(Stream& s, Type&& obj, Fn&& fn) 1030 { 1031 fn(s, std::forward<Type>(obj)); 1032 } 1033 1034 template<typename Stream, typename Type, typename Fn> 1035 static void SerWrite(Stream& s, Type&&, Fn&&) 1036 { 1037 } 1038 }; 1039 1040 /* ::GetSerializeSize implementations 1041 * 1042 * Computing the serialized size of objects is done through a special stream 1043 * object of type SizeComputer, which only records the number of bytes written 1044 * to it. 1045 * 1046 * If your Serialize or SerializationOp method has non-trivial overhead for 1047 * serialization, it may be worthwhile to implement a specialized version for 1048 * SizeComputer, which uses the s.seek() method to record bytes that would 1049 * be written instead. 1050 */ 1051 class SizeComputer 1052 { 1053 protected: 1054 uint64_t m_size{0}; 1055 1056 public: 1057 SizeComputer() = default; 1058 1059 void write(std::span<const std::byte> src) 1060 { 1061 m_size += src.size(); 1062 } 1063 1064 /** Pretend this many bytes are written, without specifying them. */ 1065 void seek(uint64_t num) 1066 { 1067 m_size += num; 1068 } 1069 1070 template <typename T> 1071 SizeComputer& operator<<(const T& obj) 1072 { 1073 ::Serialize(*this, obj); 1074 return *this; 1075 } 1076 1077 uint64_t size() const 1078 { 1079 return m_size; 1080 } 1081 }; 1082 1083 template<typename I> 1084 inline void WriteVarInt(SizeComputer &s, I n) 1085 { 1086 s.seek(GetSizeOfVarInt<I>(n)); 1087 } 1088 1089 inline void WriteCompactSize(SizeComputer &s, uint64_t nSize) 1090 { 1091 s.seek(GetSizeOfCompactSize(nSize)); 1092 } 1093 1094 template <typename T> 1095 uint64_t GetSerializeSize(const T& t) 1096 { 1097 return (SizeComputer() << t).size(); 1098 } 1099 1100 //! Check if type contains a stream by seeing if has a GetStream() method. 1101 template<typename T> 1102 concept ContainsStream = requires(T t) { t.GetStream(); }; 1103 1104 /** Wrapper that overrides the GetParams() function of a stream. */ 1105 template <typename SubStream, typename Params> 1106 class ParamsStream 1107 { 1108 const Params& m_params; 1109 // If ParamsStream constructor is passed an lvalue argument, Substream will 1110 // be a reference type, and m_substream will reference that argument. 1111 // Otherwise m_substream will be a substream instance and move from the 1112 // argument. Letting ParamsStream contain a substream instance instead of 1113 // just a reference is useful to make the ParamsStream object self contained 1114 // and let it do cleanup when destroyed, for example by closing files if 1115 // SubStream is a file stream. 1116 SubStream m_substream; 1117 1118 public: 1119 ParamsStream(SubStream&& substream, const Params& params LIFETIMEBOUND) : m_params{params}, m_substream{std::forward<SubStream>(substream)} {} 1120 1121 template <typename NestedSubstream, typename Params1, typename Params2, typename... NestedParams> 1122 ParamsStream(NestedSubstream&& s, const Params1& params1 LIFETIMEBOUND, const Params2& params2 LIFETIMEBOUND, const NestedParams&... params LIFETIMEBOUND) 1123 : ParamsStream{::ParamsStream{std::forward<NestedSubstream>(s), params2, params...}, params1} {} 1124 1125 template <typename U> ParamsStream& operator<<(const U& obj) { ::Serialize(*this, obj); return *this; } 1126 template <typename U> ParamsStream& operator>>(U&& obj) { ::Unserialize(*this, obj); return *this; } 1127 void write(std::span<const std::byte> src) { GetStream().write(src); } 1128 void read(std::span<std::byte> dst) { GetStream().read(dst); } 1129 void ignore(size_t num) { GetStream().ignore(num); } 1130 bool eof() const { return GetStream().eof(); } 1131 size_t size() const { return GetStream().size(); } 1132 1133 //! Get reference to stream parameters. 1134 template <typename P> 1135 const auto& GetParams() const 1136 { 1137 if constexpr (std::is_convertible_v<Params, P>) { 1138 return m_params; 1139 } else { 1140 return m_substream.template GetParams<P>(); 1141 } 1142 } 1143 1144 //! Get reference to underlying stream. 1145 auto& GetStream() 1146 { 1147 if constexpr (ContainsStream<SubStream>) { 1148 return m_substream.GetStream(); 1149 } else { 1150 return m_substream; 1151 } 1152 } 1153 const auto& GetStream() const 1154 { 1155 if constexpr (ContainsStream<SubStream>) { 1156 return m_substream.GetStream(); 1157 } else { 1158 return m_substream; 1159 } 1160 } 1161 }; 1162 1163 /** 1164 * Explicit template deduction guide is required for single-parameter 1165 * constructor so Substream&& is treated as a forwarding reference, and 1166 * SubStream is deduced as reference type for lvalue arguments. 1167 */ 1168 template <typename Substream, typename Params> 1169 ParamsStream(Substream&&, const Params&) -> ParamsStream<Substream, Params>; 1170 1171 /** 1172 * Template deduction guide for multiple params arguments that creates a nested 1173 * ParamsStream. 1174 */ 1175 template <typename Substream, typename Params1, typename Params2, typename... Params> 1176 ParamsStream(Substream&& s, const Params1& params1, const Params2& params2, const Params&... params) -> 1177 ParamsStream<decltype(ParamsStream{std::forward<Substream>(s), params2, params...}), Params1>; 1178 1179 /** Wrapper that serializes objects with the specified parameters. */ 1180 template <typename Params, typename T> 1181 class ParamsWrapper 1182 { 1183 const Params& m_params; 1184 T& m_object; 1185 1186 public: 1187 explicit ParamsWrapper(const Params& params, T& obj) : m_params{params}, m_object{obj} {} 1188 1189 template <typename Stream> 1190 void Serialize(Stream& s) const 1191 { 1192 ParamsStream ss{s, m_params}; 1193 ::Serialize(ss, m_object); 1194 } 1195 template <typename Stream> 1196 void Unserialize(Stream& s) 1197 { 1198 ParamsStream ss{s, m_params}; 1199 ::Unserialize(ss, m_object); 1200 } 1201 }; 1202 1203 /** 1204 * Helper macro for SerParams structs 1205 * 1206 * Allows you define SerParams instances and then apply them directly 1207 * to an object via function call syntax, eg: 1208 * 1209 * constexpr SerParams FOO{....}; 1210 * ss << FOO(obj); 1211 */ 1212 #define SER_PARAMS_OPFUNC \ 1213 /** \ 1214 * Return a wrapper around t that (de)serializes it with specified parameter params. \ 1215 * \ 1216 * See SER_PARAMS for more information on serialization parameters. \ 1217 */ \ 1218 template <typename T> \ 1219 auto operator()(T&& t) const \ 1220 { \ 1221 return ParamsWrapper{*this, t}; \ 1222 } 1223 1224 #endif // BITCOIN_SERIALIZE_H