serialize.h
1 // Copyright (c) 2009-2010 Satoshi Nakamoto 2 // Copyright (c) 2009-present The Bitcoin Core developers 3 // Distributed under the MIT software license, see the accompanying 4 // file COPYING or http://www.opensource.org/licenses/mit-license.php. 5 6 #ifndef BITCOIN_SERIALIZE_H 7 #define BITCOIN_SERIALIZE_H 8 9 #include <attributes.h> 10 #include <compat/assumptions.h> // IWYU pragma: keep 11 #include <compat/endian.h> 12 #include <prevector.h> 13 #include <span.h> 14 #include <util/overflow.h> 15 16 #include <algorithm> 17 #include <concepts> 18 #include <cstdint> 19 #include <cstring> 20 #include <ios> 21 #include <limits> 22 #include <map> 23 #include <memory> 24 #include <set> 25 #include <span> 26 #include <string> 27 #include <utility> 28 #include <vector> 29 30 /** 31 * The maximum size of a serialized object in bytes or number of elements 32 * (for eg vectors) when the size is encoded as CompactSize. 33 */ 34 static constexpr uint64_t MAX_SIZE = 0x02000000; 35 36 /** Maximum amount of memory (in bytes) to allocate at once when deserializing vectors. */ 37 static const unsigned int MAX_VECTOR_ALLOCATE = 5000000; 38 39 /** 40 * Dummy data type to identify deserializing constructors. 41 * 42 * By convention, a constructor of a type T with signature 43 * 44 * template <typename Stream> T::T(deserialize_type, Stream& s) 45 * 46 * is a deserializing constructor, which builds the type by 47 * deserializing it from s. If T contains const fields, this 48 * is likely the only way to do so. 49 */ 50 struct deserialize_type {}; 51 constexpr deserialize_type deserialize {}; 52 53 /* 54 * Lowest-level serialization and conversion. 55 */ 56 template<typename Stream> inline void ser_writedata8(Stream &s, uint8_t obj) 57 { 58 s.write(std::as_bytes(std::span{&obj, 1})); 59 } 60 template<typename Stream> inline void ser_writedata16(Stream &s, uint16_t obj) 61 { 62 obj = htole16_internal(obj); 63 s.write(std::as_bytes(std::span{&obj, 1})); 64 } 65 template<typename Stream> inline void ser_writedata32(Stream &s, uint32_t obj) 66 { 67 obj = htole32_internal(obj); 68 s.write(std::as_bytes(std::span{&obj, 1})); 69 } 70 template<typename Stream> inline void ser_writedata32be(Stream &s, uint32_t obj) 71 { 72 obj = htobe32_internal(obj); 73 s.write(std::as_bytes(std::span{&obj, 1})); 74 } 75 template<typename Stream> inline void ser_writedata64(Stream &s, uint64_t obj) 76 { 77 obj = htole64_internal(obj); 78 s.write(std::as_bytes(std::span{&obj, 1})); 79 } 80 template<typename Stream> inline uint8_t ser_readdata8(Stream &s) 81 { 82 uint8_t obj; 83 s.read(std::as_writable_bytes(std::span{&obj, 1})); 84 return obj; 85 } 86 template<typename Stream> inline uint16_t ser_readdata16(Stream &s) 87 { 88 uint16_t obj; 89 s.read(std::as_writable_bytes(std::span{&obj, 1})); 90 return le16toh_internal(obj); 91 } 92 template<typename Stream> inline uint32_t ser_readdata32(Stream &s) 93 { 94 uint32_t obj; 95 s.read(std::as_writable_bytes(std::span{&obj, 1})); 96 return le32toh_internal(obj); 97 } 98 template<typename Stream> inline uint32_t ser_readdata32be(Stream &s) 99 { 100 uint32_t obj; 101 s.read(std::as_writable_bytes(std::span{&obj, 1})); 102 return be32toh_internal(obj); 103 } 104 template<typename Stream> inline uint64_t ser_readdata64(Stream &s) 105 { 106 uint64_t obj; 107 s.read(std::as_writable_bytes(std::span{&obj, 1})); 108 return le64toh_internal(obj); 109 } 110 111 112 class SizeComputer; 113 114 /** 115 * Convert any argument to a reference to X, maintaining constness. 116 * 117 * This can be used in serialization code to invoke a base class's 118 * serialization routines. 119 * 120 * Example use: 121 * class Base { ... }; 122 * class Child : public Base { 123 * int m_data; 124 * public: 125 * SERIALIZE_METHODS(Child, obj) { 126 * READWRITE(AsBase<Base>(obj), obj.m_data); 127 * } 128 * }; 129 * 130 * static_cast cannot easily be used here, as the type of Obj will be const Child& 131 * during serialization and Child& during deserialization. AsBase will convert to 132 * const Base& and Base& appropriately. 133 */ 134 template <class Out, class In> 135 Out& AsBase(In& x) 136 { 137 static_assert(std::is_base_of_v<Out, In>); 138 return x; 139 } 140 template <class Out, class In> 141 const Out& AsBase(const In& x) 142 { 143 static_assert(std::is_base_of_v<Out, In>); 144 return x; 145 } 146 147 #define READWRITE(...) (ser_action.SerReadWriteMany(s, __VA_ARGS__)) 148 #define SER_READ(obj, code) ser_action.SerRead(s, obj, [&](Stream& s, std::remove_const_t<Type>& obj) { code; }) 149 #define SER_WRITE(obj, code) ser_action.SerWrite(s, obj, [&](Stream& s, const Type& obj) { code; }) 150 151 /** 152 * Implement the Ser and Unser methods needed for implementing a formatter (see Using below). 153 * 154 * Both Ser and Unser are delegated to a single static method SerializationOps, which is polymorphic 155 * in the serialized/deserialized type (allowing it to be const when serializing, and non-const when 156 * deserializing). 157 * 158 * Example use: 159 * struct FooFormatter { 160 * FORMATTER_METHODS(Class, obj) { READWRITE(obj.val1, VARINT(obj.val2)); } 161 * } 162 * would define a class FooFormatter that defines a serialization of Class objects consisting 163 * of serializing its val1 member using the default serialization, and its val2 member using 164 * VARINT serialization. That FooFormatter can then be used in statements like 165 * READWRITE(Using<FooFormatter>(obj.bla)). 166 */ 167 #define FORMATTER_METHODS(cls, obj) \ 168 template<typename Stream> \ 169 static void Ser(Stream& s, const cls& obj) { SerializationOps(obj, s, ActionSerialize{}); } \ 170 template<typename Stream> \ 171 static void Unser(Stream& s, cls& obj) { SerializationOps(obj, s, ActionUnserialize{}); } \ 172 template<typename Stream, typename Type, typename Operation> \ 173 static void SerializationOps(Type& obj, Stream& s, Operation ser_action) 174 175 /** 176 * Formatter methods can retrieve parameters attached to a stream using the 177 * SER_PARAMS(type) macro as long as the stream is created directly or 178 * indirectly with a parameter of that type. This permits making serialization 179 * depend on run-time context in a type-safe way. 180 * 181 * Example use: 182 * struct BarParameter { bool fancy; ... }; 183 * struct Bar { ... }; 184 * struct FooFormatter { 185 * FORMATTER_METHODS(Bar, obj) { 186 * auto& param = SER_PARAMS(BarParameter); 187 * if (param.fancy) { 188 * READWRITE(VARINT(obj.value)); 189 * } else { 190 * READWRITE(obj.value); 191 * } 192 * } 193 * }; 194 * which would then be invoked as 195 * READWRITE(BarParameter{...}(Using<FooFormatter>(obj.foo))) 196 * 197 * parameter(obj) can be invoked anywhere in the call stack; it is 198 * passed down recursively into all serialization code, until another 199 * serialization parameter overrides it. 200 * 201 * Parameters will be implicitly converted where appropriate. This means that 202 * "parent" serialization code can use a parameter that derives from, or is 203 * convertible to, a "child" formatter's parameter type. 204 * 205 * Compilation will fail in any context where serialization is invoked but 206 * no parameter of a type convertible to BarParameter is provided. 207 */ 208 #define SER_PARAMS(type) (s.template GetParams<type>()) 209 210 #define BASE_SERIALIZE_METHODS(cls) \ 211 template <typename Stream> \ 212 void Serialize(Stream& s) const \ 213 { \ 214 static_assert(std::is_same_v<const cls&, decltype(*this)>, "Serialize type mismatch"); \ 215 Ser(s, *this); \ 216 } \ 217 template <typename Stream> \ 218 void Unserialize(Stream& s) \ 219 { \ 220 static_assert(std::is_same_v<cls&, decltype(*this)>, "Unserialize type mismatch"); \ 221 Unser(s, *this); \ 222 } 223 224 /** 225 * Implement the Serialize and Unserialize methods by delegating to a single templated 226 * static method that takes the to-be-(de)serialized object as a parameter. This approach 227 * has the advantage that the constness of the object becomes a template parameter, and 228 * thus allows a single implementation that sees the object as const for serializing 229 * and non-const for deserializing, without casts. 230 */ 231 #define SERIALIZE_METHODS(cls, obj) \ 232 BASE_SERIALIZE_METHODS(cls) \ 233 FORMATTER_METHODS(cls, obj) 234 235 // Templates for serializing to anything that looks like a stream, 236 // i.e. anything that supports .read(std::span<std::byte>) and .write(std::span<const std::byte>) 237 // 238 239 // Typically int8_t and char are distinct types, but some systems may define int8_t 240 // in terms of char. Forbid serialization of char in the typical case, but allow it if 241 // it's the only way to describe an int8_t. 242 template<class T> 243 concept CharNotInt8 = std::same_as<T, char> && !std::same_as<T, int8_t>; 244 245 // clang-format off 246 template <typename Stream, CharNotInt8 V> void Serialize(Stream&, V) = delete; // char serialization forbidden. Use uint8_t or int8_t 247 template <typename Stream> void Serialize(Stream& s, std::byte a) { ser_writedata8(s, uint8_t(a)); } 248 template <typename Stream> void Serialize(Stream& s, int8_t a) { ser_writedata8(s, uint8_t(a)); } 249 template <typename Stream> void Serialize(Stream& s, uint8_t a) { ser_writedata8(s, a); } 250 template <typename Stream> void Serialize(Stream& s, int16_t a) { ser_writedata16(s, uint16_t(a)); } 251 template <typename Stream> void Serialize(Stream& s, uint16_t a) { ser_writedata16(s, a); } 252 template <typename Stream> void Serialize(Stream& s, int32_t a) { ser_writedata32(s, uint32_t(a)); } 253 template <typename Stream> void Serialize(Stream& s, uint32_t a) { ser_writedata32(s, a); } 254 template <typename Stream> void Serialize(Stream& s, int64_t a) { ser_writedata64(s, uint64_t(a)); } 255 template <typename Stream> void Serialize(Stream& s, uint64_t a) { ser_writedata64(s, a); } 256 257 template <typename Stream, BasicByte B, size_t N> void Serialize(Stream& s, const B (&a)[N]) { s.write(MakeByteSpan(a)); } 258 template <typename Stream, BasicByte B, size_t N> void Serialize(Stream& s, const std::array<B, N>& a) { s.write(MakeByteSpan(a)); } 259 template <typename Stream, BasicByte B, size_t N> void Serialize(Stream& s, std::span<B, N> span) { s.write(std::as_bytes(span)); } 260 template <typename Stream, BasicByte B> void Serialize(Stream& s, std::span<B> span) { s.write(std::as_bytes(span)); } 261 262 template <typename Stream, CharNotInt8 V> void Unserialize(Stream&, V) = delete; // char serialization forbidden. Use uint8_t or int8_t 263 template <typename Stream> void Unserialize(Stream& s, std::byte& a) { a = std::byte(ser_readdata8(s)); } 264 template <typename Stream> void Unserialize(Stream& s, int8_t& a) { a = int8_t(ser_readdata8(s)); } 265 template <typename Stream> void Unserialize(Stream& s, uint8_t& a) { a = ser_readdata8(s); } 266 template <typename Stream> void Unserialize(Stream& s, int16_t& a) { a = int16_t(ser_readdata16(s)); } 267 template <typename Stream> void Unserialize(Stream& s, uint16_t& a) { a = ser_readdata16(s); } 268 template <typename Stream> void Unserialize(Stream& s, int32_t& a) { a = int32_t(ser_readdata32(s)); } 269 template <typename Stream> void Unserialize(Stream& s, uint32_t& a) { a = ser_readdata32(s); } 270 template <typename Stream> void Unserialize(Stream& s, int64_t& a) { a = int64_t(ser_readdata64(s)); } 271 template <typename Stream> void Unserialize(Stream& s, uint64_t& a) { a = ser_readdata64(s); } 272 273 template <typename Stream, BasicByte B, size_t N> void Unserialize(Stream& s, B (&a)[N]) { s.read(MakeWritableByteSpan(a)); } 274 template <typename Stream, BasicByte B, size_t N> void Unserialize(Stream& s, std::array<B, N>& a) { s.read(MakeWritableByteSpan(a)); } 275 template <typename Stream, BasicByte B, size_t N> void Unserialize(Stream& s, std::span<B, N> span) { s.read(std::as_writable_bytes(span)); } 276 template <typename Stream, BasicByte B> void Unserialize(Stream& s, std::span<B> span) { s.read(std::as_writable_bytes(span)); } 277 278 template <typename Stream> void Serialize(Stream& s, bool a) { uint8_t f = a; ser_writedata8(s, f); } 279 template <typename Stream> void Unserialize(Stream& s, bool& a) { uint8_t f = ser_readdata8(s); a = f; } 280 // clang-format on 281 282 283 /** 284 * Compact Size 285 * size < 253 -- 1 byte 286 * size <= USHRT_MAX -- 3 bytes (253 + 2 bytes) 287 * size <= UINT_MAX -- 5 bytes (254 + 4 bytes) 288 * size > UINT_MAX -- 9 bytes (255 + 8 bytes) 289 */ 290 constexpr inline unsigned int GetSizeOfCompactSize(uint64_t nSize) 291 { 292 if (nSize < 253) return sizeof(unsigned char); 293 else if (nSize <= std::numeric_limits<uint16_t>::max()) return sizeof(unsigned char) + sizeof(uint16_t); 294 else if (nSize <= std::numeric_limits<unsigned int>::max()) return sizeof(unsigned char) + sizeof(unsigned int); 295 else return sizeof(unsigned char) + sizeof(uint64_t); 296 } 297 298 inline void WriteCompactSize(SizeComputer& os, uint64_t nSize); 299 300 template<typename Stream> 301 void WriteCompactSize(Stream& os, uint64_t nSize) 302 { 303 if (nSize < 253) 304 { 305 ser_writedata8(os, nSize); 306 } 307 else if (nSize <= std::numeric_limits<uint16_t>::max()) 308 { 309 ser_writedata8(os, 253); 310 ser_writedata16(os, nSize); 311 } 312 else if (nSize <= std::numeric_limits<unsigned int>::max()) 313 { 314 ser_writedata8(os, 254); 315 ser_writedata32(os, nSize); 316 } 317 else 318 { 319 ser_writedata8(os, 255); 320 ser_writedata64(os, nSize); 321 } 322 return; 323 } 324 325 /** 326 * Decode a CompactSize-encoded variable-length integer. 327 * 328 * As these are primarily used to encode the size of vector-like serializations, by default a range 329 * check is performed. When used as a generic number encoding, range_check should be set to false. 330 */ 331 template<typename Stream> 332 uint64_t ReadCompactSize(Stream& is, bool range_check = true) 333 { 334 uint8_t chSize = ser_readdata8(is); 335 uint64_t nSizeRet = 0; 336 if (chSize < 253) 337 { 338 nSizeRet = chSize; 339 } 340 else if (chSize == 253) 341 { 342 nSizeRet = ser_readdata16(is); 343 if (nSizeRet < 253) 344 throw std::ios_base::failure("non-canonical ReadCompactSize()"); 345 } 346 else if (chSize == 254) 347 { 348 nSizeRet = ser_readdata32(is); 349 if (nSizeRet < 0x10000u) 350 throw std::ios_base::failure("non-canonical ReadCompactSize()"); 351 } 352 else 353 { 354 nSizeRet = ser_readdata64(is); 355 if (nSizeRet < 0x100000000ULL) 356 throw std::ios_base::failure("non-canonical ReadCompactSize()"); 357 } 358 if (range_check && nSizeRet > MAX_SIZE) { 359 throw std::ios_base::failure("ReadCompactSize(): size too large"); 360 } 361 return nSizeRet; 362 } 363 364 /** 365 * Variable-length integers: bytes are a MSB base-128 encoding of the number. 366 * The high bit in each byte signifies whether another digit follows. To make 367 * sure the encoding is one-to-one, one is subtracted from all but the last digit. 368 * Thus, the byte sequence a[] with length len, where all but the last byte 369 * has bit 128 set, encodes the number: 370 * 371 * (a[len-1] & 0x7F) + sum(i=1..len-1, 128^i*((a[len-i-1] & 0x7F)+1)) 372 * 373 * Properties: 374 * * Very small (0-127: 1 byte, 128-16511: 2 bytes, 16512-2113663: 3 bytes) 375 * * Every integer has exactly one encoding 376 * * Encoding does not depend on size of original integer type 377 * * No redundancy: every (infinite) byte sequence corresponds to a list 378 * of encoded integers. 379 * 380 * 0: [0x00] 256: [0x81 0x00] 381 * 1: [0x01] 16383: [0xFE 0x7F] 382 * 127: [0x7F] 16384: [0xFF 0x00] 383 * 128: [0x80 0x00] 16511: [0xFF 0x7F] 384 * 255: [0x80 0x7F] 65535: [0x82 0xFE 0x7F] 385 * 2^32: [0x8E 0xFE 0xFE 0xFF 0x00] 386 */ 387 388 /** 389 * Mode for encoding VarInts. 390 * 391 * Currently there is no support for signed encodings. The default mode will not 392 * compile with signed values, and the legacy "nonnegative signed" mode will 393 * accept signed values, but improperly encode and decode them if they are 394 * negative. In the future, the DEFAULT mode could be extended to support 395 * negative numbers in a backwards compatible way, and additional modes could be 396 * added to support different varint formats (e.g. zigzag encoding). 397 */ 398 enum class VarIntMode { DEFAULT, NONNEGATIVE_SIGNED }; 399 400 template <VarIntMode Mode, typename I> 401 struct CheckVarIntMode { 402 constexpr CheckVarIntMode() 403 { 404 static_assert(Mode != VarIntMode::DEFAULT || std::is_unsigned_v<I>, "Unsigned type required with mode DEFAULT."); 405 static_assert(Mode != VarIntMode::NONNEGATIVE_SIGNED || std::is_signed_v<I>, "Signed type required with mode NONNEGATIVE_SIGNED."); 406 } 407 }; 408 409 template<VarIntMode Mode, typename I> 410 inline unsigned int GetSizeOfVarInt(I n) 411 { 412 CheckVarIntMode<Mode, I>(); 413 int nRet = 0; 414 while(true) { 415 nRet++; 416 if (n <= 0x7F) 417 break; 418 n = (n >> 7) - 1; 419 } 420 return nRet; 421 } 422 423 template<typename I> 424 inline void WriteVarInt(SizeComputer& os, I n); 425 426 template<typename Stream, VarIntMode Mode, typename I> 427 void WriteVarInt(Stream& os, I n) 428 { 429 CheckVarIntMode<Mode, I>(); 430 unsigned char tmp[CeilDiv(sizeof(n) * 8, 7u)]; 431 int len=0; 432 while(true) { 433 tmp[len] = (n & 0x7F) | (len ? 0x80 : 0x00); 434 if (n <= 0x7F) 435 break; 436 n = (n >> 7) - 1; 437 len++; 438 } 439 do { 440 ser_writedata8(os, tmp[len]); 441 } while(len--); 442 } 443 444 template<typename Stream, VarIntMode Mode, typename I> 445 I ReadVarInt(Stream& is) 446 { 447 CheckVarIntMode<Mode, I>(); 448 I n = 0; 449 while(true) { 450 unsigned char chData = ser_readdata8(is); 451 if (n > (std::numeric_limits<I>::max() >> 7)) { 452 throw std::ios_base::failure("ReadVarInt(): size too large"); 453 } 454 n = (n << 7) | (chData & 0x7F); 455 if (chData & 0x80) { 456 if (n == std::numeric_limits<I>::max()) { 457 throw std::ios_base::failure("ReadVarInt(): size too large"); 458 } 459 n++; 460 } else { 461 return n; 462 } 463 } 464 } 465 466 /** Simple wrapper class to serialize objects using a formatter; used by Using(). */ 467 template<typename Formatter, typename T> 468 class Wrapper 469 { 470 static_assert(std::is_lvalue_reference_v<T>, "Wrapper needs an lvalue reference type T"); 471 protected: 472 T m_object; 473 public: 474 explicit Wrapper(T obj) : m_object(obj) {} 475 template<typename Stream> void Serialize(Stream &s) const { Formatter().Ser(s, m_object); } 476 template<typename Stream> void Unserialize(Stream &s) { Formatter().Unser(s, m_object); } 477 }; 478 479 /** Cause serialization/deserialization of an object to be done using a specified formatter class. 480 * 481 * To use this, you need a class Formatter that has public functions Ser(stream, const object&) for 482 * serialization, and Unser(stream, object&) for deserialization. Serialization routines (inside 483 * READWRITE, or directly with << and >> operators), can then use Using<Formatter>(object). 484 * 485 * This works by constructing a Wrapper<Formatter, T>-wrapped version of object, where T is 486 * const during serialization, and non-const during deserialization, which maintains const 487 * correctness. 488 */ 489 template<typename Formatter, typename T> 490 static inline Wrapper<Formatter, T&> Using(T&& t) { return Wrapper<Formatter, T&>(t); } 491 492 #define VARINT_MODE(obj, mode) Using<VarIntFormatter<mode>>(obj) 493 #define VARINT(obj) Using<VarIntFormatter<VarIntMode::DEFAULT>>(obj) 494 #define COMPACTSIZE(obj) Using<CompactSizeFormatter<true>>(obj) 495 #define LIMITED_STRING(obj,n) Using<LimitedStringFormatter<n>>(obj) 496 497 /** Serialization wrapper class for integers in VarInt format. */ 498 template<VarIntMode Mode> 499 struct VarIntFormatter 500 { 501 template<typename Stream, typename I> void Ser(Stream &s, I v) 502 { 503 WriteVarInt<Stream,Mode, std::remove_cv_t<I>>(s, v); 504 } 505 506 template<typename Stream, typename I> void Unser(Stream& s, I& v) 507 { 508 v = ReadVarInt<Stream,Mode, std::remove_cv_t<I>>(s); 509 } 510 }; 511 512 /** Serialization wrapper class for custom integers and enums. 513 * 514 * It permits specifying the serialized size (1 to 8 bytes) and endianness. 515 * 516 * Use the big endian mode for values that are stored in memory in native 517 * byte order, but serialized in big endian notation. This is only intended 518 * to implement serializers that are compatible with existing formats, and 519 * its use is not recommended for new data structures. 520 */ 521 template<int Bytes, bool BigEndian = false> 522 struct CustomUintFormatter 523 { 524 static_assert(Bytes > 0 && Bytes <= 8, "CustomUintFormatter Bytes out of range"); 525 static constexpr uint64_t MAX = 0xffffffffffffffff >> (8 * (8 - Bytes)); 526 527 template <typename Stream, typename I> void Ser(Stream& s, I v) 528 { 529 if (v < 0 || v > MAX) throw std::ios_base::failure("CustomUintFormatter value out of range"); 530 if (BigEndian) { 531 uint64_t raw = htobe64_internal(v); 532 s.write(std::as_bytes(std::span{&raw, 1}).last(Bytes)); 533 } else { 534 uint64_t raw = htole64_internal(v); 535 s.write(std::as_bytes(std::span{&raw, 1}).first(Bytes)); 536 } 537 } 538 539 template <typename Stream, typename I> void Unser(Stream& s, I& v) 540 { 541 using U = typename std::conditional_t<std::is_enum_v<I>, std::underlying_type<I>, std::common_type<I>>::type; 542 static_assert(std::numeric_limits<U>::max() >= MAX && std::numeric_limits<U>::min() <= 0, "Assigned type too small"); 543 uint64_t raw = 0; 544 if (BigEndian) { 545 s.read(std::as_writable_bytes(std::span{&raw, 1}).last(Bytes)); 546 v = static_cast<I>(be64toh_internal(raw)); 547 } else { 548 s.read(std::as_writable_bytes(std::span{&raw, 1}).first(Bytes)); 549 v = static_cast<I>(le64toh_internal(raw)); 550 } 551 } 552 }; 553 554 template<int Bytes> using BigEndianFormatter = CustomUintFormatter<Bytes, true>; 555 556 /** Formatter for integers in CompactSize format. */ 557 template<bool RangeCheck> 558 struct CompactSizeFormatter 559 { 560 template<typename Stream, typename I> 561 void Unser(Stream& s, I& v) 562 { 563 uint64_t n = ReadCompactSize<Stream>(s, RangeCheck); 564 if (n < std::numeric_limits<I>::min() || n > std::numeric_limits<I>::max()) { 565 throw std::ios_base::failure("CompactSize exceeds limit of type"); 566 } 567 v = n; 568 } 569 570 template<typename Stream, typename I> 571 void Ser(Stream& s, I v) 572 { 573 static_assert(std::is_unsigned_v<I>, "CompactSize only supported for unsigned integers"); 574 static_assert(std::numeric_limits<I>::max() <= std::numeric_limits<uint64_t>::max(), "CompactSize only supports 64-bit integers and below"); 575 576 WriteCompactSize<Stream>(s, v); 577 } 578 }; 579 580 template <typename U, bool LOSSY = false> 581 struct ChronoFormatter { 582 template <typename Stream, typename Tp> 583 void Unser(Stream& s, Tp& tp) 584 { 585 U u; 586 s >> u; 587 // Lossy deserialization does not make sense, so force Wnarrowing 588 tp = Tp{typename Tp::duration{typename Tp::duration::rep{u}}}; 589 } 590 template <typename Stream, typename Tp> 591 void Ser(Stream& s, Tp tp) 592 { 593 if constexpr (LOSSY) { 594 s << U(tp.time_since_epoch().count()); 595 } else { 596 s << U{tp.time_since_epoch().count()}; 597 } 598 } 599 }; 600 template <typename U> 601 using LossyChronoFormatter = ChronoFormatter<U, true>; 602 603 class CompactSizeWriter 604 { 605 protected: 606 uint64_t n; 607 public: 608 explicit CompactSizeWriter(uint64_t n_in) : n(n_in) { } 609 610 template<typename Stream> 611 void Serialize(Stream &s) const { 612 WriteCompactSize<Stream>(s, n); 613 } 614 }; 615 616 template<size_t Limit> 617 struct LimitedStringFormatter 618 { 619 template<typename Stream> 620 void Unser(Stream& s, std::string& v) 621 { 622 size_t size = ReadCompactSize(s); 623 if (size > Limit) { 624 throw std::ios_base::failure("String length limit exceeded"); 625 } 626 v.resize(size); 627 if (size != 0) s.read(MakeWritableByteSpan(v)); 628 } 629 630 template<typename Stream> 631 void Ser(Stream& s, const std::string& v) 632 { 633 s << v; 634 } 635 }; 636 637 /** Formatter to serialize/deserialize vector elements using another formatter 638 * 639 * Example: 640 * struct X { 641 * std::vector<uint64_t> v; 642 * SERIALIZE_METHODS(X, obj) { READWRITE(Using<VectorFormatter<VarInt>>(obj.v)); } 643 * }; 644 * will define a struct that contains a vector of uint64_t, which is serialized 645 * as a vector of VarInt-encoded integers. 646 * 647 * V is not required to be an std::vector type. It works for any class that 648 * exposes a value_type, size, reserve, emplace_back, back, and const iterators. 649 */ 650 template<class Formatter> 651 struct VectorFormatter 652 { 653 template<typename Stream, typename V> 654 void Ser(Stream& s, const V& v) 655 { 656 Formatter formatter; 657 WriteCompactSize(s, v.size()); 658 for (const typename V::value_type& elem : v) { 659 formatter.Ser(s, elem); 660 } 661 } 662 663 template<typename Stream, typename V> 664 void Unser(Stream& s, V& v) 665 { 666 Formatter formatter; 667 v.clear(); 668 size_t size = ReadCompactSize(s); 669 size_t allocated = 0; 670 while (allocated < size) { 671 // For DoS prevention, do not blindly allocate as much as the stream claims to contain. 672 // Instead, allocate in 5MiB batches, so that an attacker actually needs to provide 673 // X MiB of data to make us allocate X+5 Mib. 674 static_assert(sizeof(typename V::value_type) <= MAX_VECTOR_ALLOCATE, "Vector element size too large"); 675 allocated = std::min(size, allocated + MAX_VECTOR_ALLOCATE / sizeof(typename V::value_type)); 676 v.reserve(allocated); 677 while (v.size() < allocated) { 678 v.emplace_back(); 679 formatter.Unser(s, v.back()); 680 } 681 } 682 }; 683 }; 684 685 /** 686 * Forward declarations 687 */ 688 689 /** 690 * string 691 */ 692 template<typename Stream, typename C> void Serialize(Stream& os, const std::basic_string<C>& str); 693 template<typename Stream, typename C> void Unserialize(Stream& is, std::basic_string<C>& str); 694 695 /** 696 * prevector 697 */ 698 template<typename Stream, unsigned int N, typename T> inline void Serialize(Stream& os, const prevector<N, T>& v); 699 template<typename Stream, unsigned int N, typename T> inline void Unserialize(Stream& is, prevector<N, T>& v); 700 701 /** 702 * vector 703 */ 704 template<typename Stream, typename T, typename A> inline void Serialize(Stream& os, const std::vector<T, A>& v); 705 template<typename Stream, typename T, typename A> inline void Unserialize(Stream& is, std::vector<T, A>& v); 706 707 /** 708 * pair 709 */ 710 template<typename Stream, typename K, typename T> void Serialize(Stream& os, const std::pair<K, T>& item); 711 template<typename Stream, typename K, typename T> void Unserialize(Stream& is, std::pair<K, T>& item); 712 713 /** 714 * map 715 */ 716 template<typename Stream, typename K, typename T, typename Pred, typename A> void Serialize(Stream& os, const std::map<K, T, Pred, A>& m); 717 template<typename Stream, typename K, typename T, typename Pred, typename A> void Unserialize(Stream& is, std::map<K, T, Pred, A>& m); 718 719 /** 720 * set 721 */ 722 template<typename Stream, typename K, typename Pred, typename A> void Serialize(Stream& os, const std::set<K, Pred, A>& m); 723 template<typename Stream, typename K, typename Pred, typename A> void Unserialize(Stream& is, std::set<K, Pred, A>& m); 724 725 /** 726 * shared_ptr 727 */ 728 template<typename Stream, typename T> void Serialize(Stream& os, const std::shared_ptr<const T>& p); 729 template<typename Stream, typename T> void Unserialize(Stream& os, std::shared_ptr<const T>& p); 730 731 /** 732 * unique_ptr 733 */ 734 template<typename Stream, typename T> void Serialize(Stream& os, const std::unique_ptr<const T>& p); 735 template<typename Stream, typename T> void Unserialize(Stream& os, std::unique_ptr<const T>& p); 736 737 738 /** 739 * If none of the specialized versions above matched, default to calling member function. 740 */ 741 template <class T, class Stream> 742 concept Serializable = requires(T a, Stream s) { a.Serialize(s); }; 743 template <typename Stream, typename T> 744 requires Serializable<T, Stream> 745 void Serialize(Stream& os, const T& a) 746 { 747 a.Serialize(os); 748 } 749 750 template <class T, class Stream> 751 concept Unserializable = requires(T a, Stream s) { a.Unserialize(s); }; 752 template <typename Stream, typename T> 753 requires Unserializable<T, Stream> 754 void Unserialize(Stream& is, T&& a) 755 { 756 a.Unserialize(is); 757 } 758 759 /** Default formatter. Serializes objects as themselves. 760 * 761 * The vector/prevector serialization code passes this to VectorFormatter 762 * to enable reusing that logic. It shouldn't be needed elsewhere. 763 */ 764 struct DefaultFormatter 765 { 766 template<typename Stream, typename T> 767 static void Ser(Stream& s, const T& t) { Serialize(s, t); } 768 769 template<typename Stream, typename T> 770 static void Unser(Stream& s, T& t) { Unserialize(s, t); } 771 }; 772 773 774 775 776 777 /** 778 * string 779 */ 780 template<typename Stream, typename C> 781 void Serialize(Stream& os, const std::basic_string<C>& str) 782 { 783 WriteCompactSize(os, str.size()); 784 if (!str.empty()) 785 os.write(MakeByteSpan(str)); 786 } 787 788 template<typename Stream, typename C> 789 void Unserialize(Stream& is, std::basic_string<C>& str) 790 { 791 unsigned int nSize = ReadCompactSize(is); 792 str.resize(nSize); 793 if (nSize != 0) 794 is.read(MakeWritableByteSpan(str)); 795 } 796 797 798 799 /** 800 * prevector 801 */ 802 template <typename Stream, unsigned int N, typename T> 803 void Serialize(Stream& os, const prevector<N, T>& v) 804 { 805 if constexpr (BasicByte<T>) { // Use optimized version for unformatted basic bytes 806 WriteCompactSize(os, v.size()); 807 if (!v.empty()) os.write(MakeByteSpan(v)); 808 } else { 809 Serialize(os, Using<VectorFormatter<DefaultFormatter>>(v)); 810 } 811 } 812 813 814 template <typename Stream, unsigned int N, typename T> 815 void Unserialize(Stream& is, prevector<N, T>& v) 816 { 817 if constexpr (BasicByte<T>) { // Use optimized version for unformatted basic bytes 818 // Limit size per read so bogus size value won't cause out of memory 819 v.clear(); 820 unsigned int nSize = ReadCompactSize(is); 821 unsigned int i = 0; 822 while (i < nSize) { 823 unsigned int blk = std::min(nSize - i, (unsigned int)(1 + 4999999 / sizeof(T))); 824 v.resize_uninitialized(i + blk); 825 is.read(std::as_writable_bytes(std::span{&v[i], blk})); 826 i += blk; 827 } 828 } else { 829 Unserialize(is, Using<VectorFormatter<DefaultFormatter>>(v)); 830 } 831 } 832 833 834 /** 835 * vector 836 */ 837 template <typename Stream, typename T, typename A> 838 void Serialize(Stream& os, const std::vector<T, A>& v) 839 { 840 if constexpr (BasicByte<T>) { // Use optimized version for unformatted basic bytes 841 WriteCompactSize(os, v.size()); 842 if (!v.empty()) os.write(MakeByteSpan(v)); 843 } else if constexpr (std::is_same_v<T, bool>) { 844 // A special case for std::vector<bool>, as dereferencing 845 // std::vector<bool>::const_iterator does not result in a const bool& 846 // due to std::vector's special casing for bool arguments. 847 WriteCompactSize(os, v.size()); 848 for (bool elem : v) { 849 ::Serialize(os, elem); 850 } 851 } else { 852 Serialize(os, Using<VectorFormatter<DefaultFormatter>>(v)); 853 } 854 } 855 856 857 template <typename Stream, typename T, typename A> 858 void Unserialize(Stream& is, std::vector<T, A>& v) 859 { 860 if constexpr (BasicByte<T>) { // Use optimized version for unformatted basic bytes 861 // Limit size per read so bogus size value won't cause out of memory 862 v.clear(); 863 unsigned int nSize = ReadCompactSize(is); 864 unsigned int i = 0; 865 while (i < nSize) { 866 unsigned int blk = std::min(nSize - i, (unsigned int)(1 + 4999999 / sizeof(T))); 867 v.resize(i + blk); 868 is.read(std::as_writable_bytes(std::span{&v[i], blk})); 869 i += blk; 870 } 871 } else { 872 Unserialize(is, Using<VectorFormatter<DefaultFormatter>>(v)); 873 } 874 } 875 876 877 /** 878 * pair 879 */ 880 template<typename Stream, typename K, typename T> 881 void Serialize(Stream& os, const std::pair<K, T>& item) 882 { 883 Serialize(os, item.first); 884 Serialize(os, item.second); 885 } 886 887 template<typename Stream, typename K, typename T> 888 void Unserialize(Stream& is, std::pair<K, T>& item) 889 { 890 Unserialize(is, item.first); 891 Unserialize(is, item.second); 892 } 893 894 895 896 /** 897 * map 898 */ 899 template<typename Stream, typename K, typename T, typename Pred, typename A> 900 void Serialize(Stream& os, const std::map<K, T, Pred, A>& m) 901 { 902 WriteCompactSize(os, m.size()); 903 for (const auto& entry : m) 904 Serialize(os, entry); 905 } 906 907 template<typename Stream, typename K, typename T, typename Pred, typename A> 908 void Unserialize(Stream& is, std::map<K, T, Pred, A>& m) 909 { 910 m.clear(); 911 unsigned int nSize = ReadCompactSize(is); 912 typename std::map<K, T, Pred, A>::iterator mi = m.begin(); 913 for (unsigned int i = 0; i < nSize; i++) 914 { 915 std::pair<K, T> item; 916 Unserialize(is, item); 917 mi = m.insert(mi, item); 918 } 919 } 920 921 922 923 /** 924 * set 925 */ 926 template<typename Stream, typename K, typename Pred, typename A> 927 void Serialize(Stream& os, const std::set<K, Pred, A>& m) 928 { 929 WriteCompactSize(os, m.size()); 930 for (typename std::set<K, Pred, A>::const_iterator it = m.begin(); it != m.end(); ++it) 931 Serialize(os, (*it)); 932 } 933 934 template<typename Stream, typename K, typename Pred, typename A> 935 void Unserialize(Stream& is, std::set<K, Pred, A>& m) 936 { 937 m.clear(); 938 unsigned int nSize = ReadCompactSize(is); 939 typename std::set<K, Pred, A>::iterator it = m.begin(); 940 for (unsigned int i = 0; i < nSize; i++) 941 { 942 K key; 943 Unserialize(is, key); 944 it = m.insert(it, key); 945 } 946 } 947 948 949 950 /** 951 * unique_ptr 952 */ 953 template<typename Stream, typename T> void 954 Serialize(Stream& os, const std::unique_ptr<const T>& p) 955 { 956 Serialize(os, *p); 957 } 958 959 template<typename Stream, typename T> 960 void Unserialize(Stream& is, std::unique_ptr<const T>& p) 961 { 962 p.reset(new T(deserialize, is)); 963 } 964 965 966 967 /** 968 * shared_ptr 969 */ 970 template<typename Stream, typename T> void 971 Serialize(Stream& os, const std::shared_ptr<const T>& p) 972 { 973 Serialize(os, *p); 974 } 975 976 template<typename Stream, typename T> 977 void Unserialize(Stream& is, std::shared_ptr<const T>& p) 978 { 979 p = std::make_shared<const T>(deserialize, is); 980 } 981 982 /** 983 * Support for (un)serializing many things at once 984 */ 985 986 template <typename Stream, typename... Args> 987 void SerializeMany(Stream& s, const Args&... args) 988 { 989 (::Serialize(s, args), ...); 990 } 991 992 template <typename Stream, typename... Args> 993 inline void UnserializeMany(Stream& s, Args&&... args) 994 { 995 (::Unserialize(s, args), ...); 996 } 997 998 /** 999 * Support for all macros providing or using the ser_action parameter of the SerializationOps method. 1000 */ 1001 struct ActionSerialize { 1002 static constexpr bool ForRead() { return false; } 1003 1004 template<typename Stream, typename... Args> 1005 static void SerReadWriteMany(Stream& s, const Args&... args) 1006 { 1007 ::SerializeMany(s, args...); 1008 } 1009 1010 template<typename Stream, typename Type, typename Fn> 1011 static void SerRead(Stream& s, Type&&, Fn&&) 1012 { 1013 } 1014 1015 template<typename Stream, typename Type, typename Fn> 1016 static void SerWrite(Stream& s, Type&& obj, Fn&& fn) 1017 { 1018 fn(s, std::forward<Type>(obj)); 1019 } 1020 }; 1021 struct ActionUnserialize { 1022 static constexpr bool ForRead() { return true; } 1023 1024 template<typename Stream, typename... Args> 1025 static void SerReadWriteMany(Stream& s, Args&&... args) 1026 { 1027 ::UnserializeMany(s, args...); 1028 } 1029 1030 template<typename Stream, typename Type, typename Fn> 1031 static void SerRead(Stream& s, Type&& obj, Fn&& fn) 1032 { 1033 fn(s, std::forward<Type>(obj)); 1034 } 1035 1036 template<typename Stream, typename Type, typename Fn> 1037 static void SerWrite(Stream& s, Type&&, Fn&&) 1038 { 1039 } 1040 }; 1041 1042 /* ::GetSerializeSize implementations 1043 * 1044 * Computing the serialized size of objects is done through a special stream 1045 * object of type SizeComputer, which only records the number of bytes written 1046 * to it. 1047 * 1048 * If your Serialize or SerializationOp method has non-trivial overhead for 1049 * serialization, it may be worthwhile to implement a specialized version for 1050 * SizeComputer, which uses the s.seek() method to record bytes that would 1051 * be written instead. 1052 */ 1053 class SizeComputer 1054 { 1055 protected: 1056 uint64_t m_size{0}; 1057 1058 public: 1059 SizeComputer() = default; 1060 1061 void write(std::span<const std::byte> src) 1062 { 1063 m_size += src.size(); 1064 } 1065 1066 /** Pretend this many bytes are written, without specifying them. */ 1067 void seek(uint64_t num) 1068 { 1069 m_size += num; 1070 } 1071 1072 template <typename T> 1073 SizeComputer& operator<<(const T& obj) 1074 { 1075 ::Serialize(*this, obj); 1076 return *this; 1077 } 1078 1079 uint64_t size() const 1080 { 1081 return m_size; 1082 } 1083 }; 1084 1085 template<typename I> 1086 inline void WriteVarInt(SizeComputer &s, I n) 1087 { 1088 s.seek(GetSizeOfVarInt<I>(n)); 1089 } 1090 1091 inline void WriteCompactSize(SizeComputer &s, uint64_t nSize) 1092 { 1093 s.seek(GetSizeOfCompactSize(nSize)); 1094 } 1095 1096 template <typename T> 1097 uint64_t GetSerializeSize(const T& t) 1098 { 1099 return (SizeComputer() << t).size(); 1100 } 1101 1102 //! Check if type contains a stream by seeing if has a GetStream() method. 1103 template<typename T> 1104 concept ContainsStream = requires(T t) { t.GetStream(); }; 1105 1106 /** Wrapper that overrides the GetParams() function of a stream. */ 1107 template <typename SubStream, typename Params> 1108 class ParamsStream 1109 { 1110 const Params& m_params; 1111 // If ParamsStream constructor is passed an lvalue argument, Substream will 1112 // be a reference type, and m_substream will reference that argument. 1113 // Otherwise m_substream will be a substream instance and move from the 1114 // argument. Letting ParamsStream contain a substream instance instead of 1115 // just a reference is useful to make the ParamsStream object self contained 1116 // and let it do cleanup when destroyed, for example by closing files if 1117 // SubStream is a file stream. 1118 SubStream m_substream; 1119 1120 public: 1121 ParamsStream(SubStream&& substream, const Params& params LIFETIMEBOUND) : m_params{params}, m_substream{std::forward<SubStream>(substream)} {} 1122 1123 template <typename NestedSubstream, typename Params1, typename Params2, typename... NestedParams> 1124 ParamsStream(NestedSubstream&& s, const Params1& params1 LIFETIMEBOUND, const Params2& params2 LIFETIMEBOUND, const NestedParams&... params LIFETIMEBOUND) 1125 : ParamsStream{::ParamsStream{std::forward<NestedSubstream>(s), params2, params...}, params1} {} 1126 1127 template <typename U> ParamsStream& operator<<(const U& obj) { ::Serialize(*this, obj); return *this; } 1128 template <typename U> ParamsStream& operator>>(U&& obj) { ::Unserialize(*this, obj); return *this; } 1129 void write(std::span<const std::byte> src) { GetStream().write(src); } 1130 void read(std::span<std::byte> dst) { GetStream().read(dst); } 1131 void ignore(size_t num) { GetStream().ignore(num); } 1132 bool empty() const { return GetStream().empty(); } 1133 size_t size() const { return GetStream().size(); } 1134 1135 //! Get reference to stream parameters. 1136 template <typename P> 1137 const auto& GetParams() const 1138 { 1139 if constexpr (std::is_convertible_v<Params, P>) { 1140 return m_params; 1141 } else { 1142 return m_substream.template GetParams<P>(); 1143 } 1144 } 1145 1146 //! Get reference to underlying stream. 1147 auto& GetStream() 1148 { 1149 if constexpr (ContainsStream<SubStream>) { 1150 return m_substream.GetStream(); 1151 } else { 1152 return m_substream; 1153 } 1154 } 1155 const auto& GetStream() const 1156 { 1157 if constexpr (ContainsStream<SubStream>) { 1158 return m_substream.GetStream(); 1159 } else { 1160 return m_substream; 1161 } 1162 } 1163 }; 1164 1165 /** 1166 * Explicit template deduction guide is required for single-parameter 1167 * constructor so Substream&& is treated as a forwarding reference, and 1168 * SubStream is deduced as reference type for lvalue arguments. 1169 */ 1170 template <typename Substream, typename Params> 1171 ParamsStream(Substream&&, const Params&) -> ParamsStream<Substream, Params>; 1172 1173 /** 1174 * Template deduction guide for multiple params arguments that creates a nested 1175 * ParamsStream. 1176 */ 1177 template <typename Substream, typename Params1, typename Params2, typename... Params> 1178 ParamsStream(Substream&& s, const Params1& params1, const Params2& params2, const Params&... params) -> 1179 ParamsStream<decltype(ParamsStream{std::forward<Substream>(s), params2, params...}), Params1>; 1180 1181 /** Wrapper that serializes objects with the specified parameters. */ 1182 template <typename Params, typename T> 1183 class ParamsWrapper 1184 { 1185 const Params& m_params; 1186 T& m_object; 1187 1188 public: 1189 explicit ParamsWrapper(const Params& params, T& obj) : m_params{params}, m_object{obj} {} 1190 1191 template <typename Stream> 1192 void Serialize(Stream& s) const 1193 { 1194 ParamsStream ss{s, m_params}; 1195 ::Serialize(ss, m_object); 1196 } 1197 template <typename Stream> 1198 void Unserialize(Stream& s) 1199 { 1200 ParamsStream ss{s, m_params}; 1201 ::Unserialize(ss, m_object); 1202 } 1203 }; 1204 1205 /** 1206 * Helper macro for SerParams structs 1207 * 1208 * Allows you define SerParams instances and then apply them directly 1209 * to an object via function call syntax, eg: 1210 * 1211 * constexpr SerParams FOO{....}; 1212 * ss << FOO(obj); 1213 */ 1214 #define SER_PARAMS_OPFUNC \ 1215 /** \ 1216 * Return a wrapper around t that (de)serializes it with specified parameter params. \ 1217 * \ 1218 * See SER_PARAMS for more information on serialization parameters. \ 1219 */ \ 1220 template <typename T> \ 1221 auto operator()(T&& t) const \ 1222 { \ 1223 return ParamsWrapper{*this, t}; \ 1224 } 1225 1226 #endif // BITCOIN_SERIALIZE_H