Reserve output capacity up front for binary serialization The vector-returning to_cbor/to_msgpack/to_ubjson/to_bjdata/to_bson grew the output buffer purely by geometric reallocation. Reserving an estimate up front avoids the early reallocations, which is the dominant per-byte cost for array/object-heavy output. The estimate (binary_reserve_hint) is deliberately conservative and safe against untrusted input: it consults only the top-level element count (O(1), no walk of the DOM), guards the multiplication against overflow, and clamps the result to a fixed 1 MiB ceiling, so a large or hostile DOM can never force an oversized allocation here. The buffer still grows geometrically past the hint, so an underestimate only costs a few later reallocations; scalars/strings/binary are written in one shot and get no hint. Reserving capacity does not change the bytes produced. Throughput (g++/clang -O3, vs the previous commit): cbor int array +10% / +13% cbor object array +20% / +38% Output is byte-for-byte identical to develop across the binary differential corpus. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01XAYM1qhSA2FDaDcGfPW3fG Signed-off-by: Niels Lohmann <mail@nlohmann.me>
diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index 3065068..27122a2 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp
@@ -40,6 +40,35 @@ /////////////////// /*! +@brief conservative capacity hint for binary serialization into a std::vector + +Returns an approximate number of bytes to reserve up front so that serializing +an array/object of many elements does not repeatedly reallocate the output +buffer. Only the top-level element count is consulted (O(1), no walk of the +DOM), and the result is clamped to a fixed ceiling: a large or untrusted DOM can +therefore never trigger an oversized allocation here, and the multiplication +cannot overflow. The buffer still grows geometrically beyond the hint, so a hint +that is too small only costs a few later reallocations. A single scalar, string, +or binary value is written in one shot and needs no hint. +*/ +template<typename BasicJsonType> +std::size_t binary_reserve_hint(const BasicJsonType& j) +{ + constexpr std::size_t max_hint = static_cast<std::size_t>(1) << 20; // 1 MiB + if (j.is_array() || j.is_object()) + { + const std::size_t elements = j.size(); + // guard the multiplication against overflow and cap the reservation + if (elements > max_hint / 4) + { + return max_hint; + } + return (elements * 4) + 2; + } + return 0; +} + +/*! @brief serialization to CBOR and MessagePack values */ template<typename BasicJsonType, typename CharType, typename OutputSinkType = output_adapter_sink<CharType>>
diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 7eac69d..002399b 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp
@@ -4340,6 +4340,7 @@ static std::vector<std::uint8_t> to_cbor(const basic_json& j) { std::vector<std::uint8_t> result; + result.reserve(detail::binary_reserve_hint(j)); detail::binary_writer<basic_json, std::uint8_t, detail::output_vector_sink<std::uint8_t>>( detail::output_vector_sink<std::uint8_t>(result)).write_cbor(j); return result; @@ -4364,6 +4365,7 @@ static std::vector<std::uint8_t> to_msgpack(const basic_json& j) { std::vector<std::uint8_t> result; + result.reserve(detail::binary_reserve_hint(j)); detail::binary_writer<basic_json, std::uint8_t, detail::output_vector_sink<std::uint8_t>>( detail::output_vector_sink<std::uint8_t>(result)).write_msgpack(j); return result; @@ -4390,6 +4392,7 @@ const bool use_type = false) { std::vector<std::uint8_t> result; + result.reserve(detail::binary_reserve_hint(j)); detail::binary_writer<basic_json, std::uint8_t, detail::output_vector_sink<std::uint8_t>>( detail::output_vector_sink<std::uint8_t>(result)).write_ubjson(j, use_size, use_type); return result; @@ -4419,6 +4422,7 @@ const bjdata_version_t version = bjdata_version_t::draft2) { std::vector<std::uint8_t> result; + result.reserve(detail::binary_reserve_hint(j)); detail::binary_writer<basic_json, std::uint8_t, detail::output_vector_sink<std::uint8_t>>( detail::output_vector_sink<std::uint8_t>(result)).write_ubjson(j, use_size, use_type, true, true, version); return result; @@ -4447,6 +4451,7 @@ static std::vector<std::uint8_t> to_bson(const basic_json& j) { std::vector<std::uint8_t> result; + result.reserve(detail::binary_reserve_hint(j)); detail::binary_writer<basic_json, std::uint8_t, detail::output_vector_sink<std::uint8_t>>( detail::output_vector_sink<std::uint8_t>(result)).write_bson(j); return result;
diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index fc2ac98..c9cbb98 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp
@@ -17050,6 +17050,35 @@ /////////////////// /*! +@brief conservative capacity hint for binary serialization into a std::vector + +Returns an approximate number of bytes to reserve up front so that serializing +an array/object of many elements does not repeatedly reallocate the output +buffer. Only the top-level element count is consulted (O(1), no walk of the +DOM), and the result is clamped to a fixed ceiling: a large or untrusted DOM can +therefore never trigger an oversized allocation here, and the multiplication +cannot overflow. The buffer still grows geometrically beyond the hint, so a hint +that is too small only costs a few later reallocations. A single scalar, string, +or binary value is written in one shot and needs no hint. +*/ +template<typename BasicJsonType> +std::size_t binary_reserve_hint(const BasicJsonType& j) +{ + constexpr std::size_t max_hint = static_cast<std::size_t>(1) << 20; // 1 MiB + if (j.is_array() || j.is_object()) + { + const std::size_t elements = j.size(); + // guard the multiplication against overflow and cap the reservation + if (elements > max_hint / 4) + { + return max_hint; + } + return (elements * 4) + 2; + } + return 0; +} + +/*! @brief serialization to CBOR and MessagePack values */ template<typename BasicJsonType, typename CharType, typename OutputSinkType = output_adapter_sink<CharType>> @@ -25846,6 +25875,7 @@ static std::vector<std::uint8_t> to_cbor(const basic_json& j) { std::vector<std::uint8_t> result; + result.reserve(detail::binary_reserve_hint(j)); detail::binary_writer<basic_json, std::uint8_t, detail::output_vector_sink<std::uint8_t>>( detail::output_vector_sink<std::uint8_t>(result)).write_cbor(j); return result; @@ -25870,6 +25900,7 @@ static std::vector<std::uint8_t> to_msgpack(const basic_json& j) { std::vector<std::uint8_t> result; + result.reserve(detail::binary_reserve_hint(j)); detail::binary_writer<basic_json, std::uint8_t, detail::output_vector_sink<std::uint8_t>>( detail::output_vector_sink<std::uint8_t>(result)).write_msgpack(j); return result; @@ -25896,6 +25927,7 @@ const bool use_type = false) { std::vector<std::uint8_t> result; + result.reserve(detail::binary_reserve_hint(j)); detail::binary_writer<basic_json, std::uint8_t, detail::output_vector_sink<std::uint8_t>>( detail::output_vector_sink<std::uint8_t>(result)).write_ubjson(j, use_size, use_type); return result; @@ -25925,6 +25957,7 @@ const bjdata_version_t version = bjdata_version_t::draft2) { std::vector<std::uint8_t> result; + result.reserve(detail::binary_reserve_hint(j)); detail::binary_writer<basic_json, std::uint8_t, detail::output_vector_sink<std::uint8_t>>( detail::output_vector_sink<std::uint8_t>(result)).write_ubjson(j, use_size, use_type, true, true, version); return result; @@ -25953,6 +25986,7 @@ static std::vector<std::uint8_t> to_bson(const basic_json& j) { std::vector<std::uint8_t> result; + result.reserve(detail::binary_reserve_hint(j)); detail::binary_writer<basic_json, std::uint8_t, detail::output_vector_sink<std::uint8_t>>( detail::output_vector_sink<std::uint8_t>(result)).write_bson(j); return result;