Reserve output capacity up front for binary serialization The vector-returning to_cbor/to_msgpack/to_ubjson/to_bjdata/to_bson grew the output buffer purely by geometric reallocation. Reserving an estimate up front avoids the early reallocations, which is the dominant per-byte cost for array/object-heavy output. The estimate (binary_reserve_hint) is deliberately conservative and safe against untrusted input: it consults only the top-level element count (O(1), no walk of the DOM), guards the multiplication against overflow, and clamps the result to a fixed 1 MiB ceiling, so a large or hostile DOM can never force an oversized allocation here. The buffer still grows geometrically past the hint, so an underestimate only costs a few later reallocations; scalars/strings/binary are written in one shot and get no hint. Reserving capacity does not change the bytes produced. Throughput (g++/clang -O3, vs the previous commit): cbor int array +10% / +13% cbor object array +20% / +38% Output is byte-for-byte identical to develop across the binary differential corpus. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01XAYM1qhSA2FDaDcGfPW3fG Signed-off-by: Niels Lohmann <mail@nlohmann.me>
diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index 4e27471..fe97dc2 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp
@@ -40,6 +40,35 @@ /////////////////// /*! +@brief conservative capacity hint for binary serialization into a std::vector + +Returns an approximate number of bytes to reserve up front so that serializing +an array/object of many elements does not repeatedly reallocate the output +buffer. Only the top-level element count is consulted (O(1), no walk of the +DOM), and the result is clamped to a fixed ceiling: a large or untrusted DOM can +therefore never trigger an oversized allocation here, and the multiplication +cannot overflow. The buffer still grows geometrically beyond the hint, so a hint +that is too small only costs a few later reallocations. A single scalar, string, +or binary value is written in one shot and needs no hint. +*/ +template<typename BasicJsonType> +std::size_t binary_reserve_hint(const BasicJsonType& j) +{ + constexpr std::size_t max_hint = static_cast<std::size_t>(1) << 20; // 1 MiB + if (j.is_array() || j.is_object()) + { + const std::size_t elements = j.size(); + // guard the multiplication against overflow and cap the reservation + if (elements > max_hint / 4) + { + return max_hint; + } + return (elements * 4) + 2; + } + return 0; +} + +/*! @brief serialization to CBOR and MessagePack values */ template<typename BasicJsonType, typename CharType, typename OutputSinkType = output_adapter_sink<CharType>>
diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 551bcde..1eeac50 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp
@@ -4327,6 +4327,7 @@ static std::vector<std::uint8_t> to_cbor(const basic_json& j) { std::vector<std::uint8_t> result; + result.reserve(detail::binary_reserve_hint(j)); detail::binary_writer<basic_json, std::uint8_t, detail::output_vector_sink<std::uint8_t>>( detail::output_vector_sink<std::uint8_t>(result)).write_cbor(j); return result; @@ -4351,6 +4352,7 @@ static std::vector<std::uint8_t> to_msgpack(const basic_json& j) { std::vector<std::uint8_t> result; + result.reserve(detail::binary_reserve_hint(j)); detail::binary_writer<basic_json, std::uint8_t, detail::output_vector_sink<std::uint8_t>>( detail::output_vector_sink<std::uint8_t>(result)).write_msgpack(j); return result; @@ -4377,6 +4379,7 @@ const bool use_type = false) { std::vector<std::uint8_t> result; + result.reserve(detail::binary_reserve_hint(j)); detail::binary_writer<basic_json, std::uint8_t, detail::output_vector_sink<std::uint8_t>>( detail::output_vector_sink<std::uint8_t>(result)).write_ubjson(j, use_size, use_type); return result; @@ -4406,6 +4409,7 @@ const bjdata_version_t version = bjdata_version_t::draft2) { std::vector<std::uint8_t> result; + result.reserve(detail::binary_reserve_hint(j)); detail::binary_writer<basic_json, std::uint8_t, detail::output_vector_sink<std::uint8_t>>( detail::output_vector_sink<std::uint8_t>(result)).write_ubjson(j, use_size, use_type, true, true, version); return result; @@ -4434,6 +4438,7 @@ static std::vector<std::uint8_t> to_bson(const basic_json& j) { std::vector<std::uint8_t> result; + result.reserve(detail::binary_reserve_hint(j)); detail::binary_writer<basic_json, std::uint8_t, detail::output_vector_sink<std::uint8_t>>( detail::output_vector_sink<std::uint8_t>(result)).write_bson(j); return result;
diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index cfffde1..6cfdbfc 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp
@@ -16854,6 +16854,35 @@ /////////////////// /*! +@brief conservative capacity hint for binary serialization into a std::vector + +Returns an approximate number of bytes to reserve up front so that serializing +an array/object of many elements does not repeatedly reallocate the output +buffer. Only the top-level element count is consulted (O(1), no walk of the +DOM), and the result is clamped to a fixed ceiling: a large or untrusted DOM can +therefore never trigger an oversized allocation here, and the multiplication +cannot overflow. The buffer still grows geometrically beyond the hint, so a hint +that is too small only costs a few later reallocations. A single scalar, string, +or binary value is written in one shot and needs no hint. +*/ +template<typename BasicJsonType> +std::size_t binary_reserve_hint(const BasicJsonType& j) +{ + constexpr std::size_t max_hint = static_cast<std::size_t>(1) << 20; // 1 MiB + if (j.is_array() || j.is_object()) + { + const std::size_t elements = j.size(); + // guard the multiplication against overflow and cap the reservation + if (elements > max_hint / 4) + { + return max_hint; + } + return (elements * 4) + 2; + } + return 0; +} + +/*! @brief serialization to CBOR and MessagePack values */ template<typename BasicJsonType, typename CharType, typename OutputSinkType = output_adapter_sink<CharType>> @@ -25571,6 +25600,7 @@ static std::vector<std::uint8_t> to_cbor(const basic_json& j) { std::vector<std::uint8_t> result; + result.reserve(detail::binary_reserve_hint(j)); detail::binary_writer<basic_json, std::uint8_t, detail::output_vector_sink<std::uint8_t>>( detail::output_vector_sink<std::uint8_t>(result)).write_cbor(j); return result; @@ -25595,6 +25625,7 @@ static std::vector<std::uint8_t> to_msgpack(const basic_json& j) { std::vector<std::uint8_t> result; + result.reserve(detail::binary_reserve_hint(j)); detail::binary_writer<basic_json, std::uint8_t, detail::output_vector_sink<std::uint8_t>>( detail::output_vector_sink<std::uint8_t>(result)).write_msgpack(j); return result; @@ -25621,6 +25652,7 @@ const bool use_type = false) { std::vector<std::uint8_t> result; + result.reserve(detail::binary_reserve_hint(j)); detail::binary_writer<basic_json, std::uint8_t, detail::output_vector_sink<std::uint8_t>>( detail::output_vector_sink<std::uint8_t>(result)).write_ubjson(j, use_size, use_type); return result; @@ -25650,6 +25682,7 @@ const bjdata_version_t version = bjdata_version_t::draft2) { std::vector<std::uint8_t> result; + result.reserve(detail::binary_reserve_hint(j)); detail::binary_writer<basic_json, std::uint8_t, detail::output_vector_sink<std::uint8_t>>( detail::output_vector_sink<std::uint8_t>(result)).write_ubjson(j, use_size, use_type, true, true, version); return result; @@ -25678,6 +25711,7 @@ static std::vector<std::uint8_t> to_bson(const basic_json& j) { std::vector<std::uint8_t> result; + result.reserve(detail::binary_reserve_hint(j)); detail::binary_writer<basic_json, std::uint8_t, detail::output_vector_sink<std::uint8_t>>( detail::output_vector_sink<std::uint8_t>(result)).write_bson(j); return result;