From 9ff8686a60ed040f20037f033083735cce35e566 Mon Sep 17 00:00:00 2001 From: slowriot Date: Fri, 24 May 2024 13:24:39 +0100 Subject: [PATCH 01/14] Add support for bson element 11: uint64 --- include/nlohmann/detail/input/binary_reader.hpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index a6e100e761..89c0cd2b53 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -318,6 +318,12 @@ class binary_reader std::int32_t value{}; return get_number(input_format_t::bson, value) && sax->number_integer(value); } + + case 0x11: // uint64 + { + std::uint64_t value{}; + return get_number(input_format_t::bson, value) && sax->number_integer(value); + } case 0x12: // int64 { From 4c4c20998ceccc99b5edeef17bb4a6e40f9be763 Mon Sep 17 00:00:00 2001 From: slowriot Date: Fri, 24 May 2024 13:32:38 +0100 Subject: [PATCH 02/14] Correctly encode unsigned 64bit values in BSON --- include/nlohmann/detail/output/binary_writer.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index f475d57be8..6ccc98e19f 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -1066,7 +1066,7 @@ class binary_writer { return (value <= static_cast((std::numeric_limits::max)())) ? sizeof(std::int32_t) - : sizeof(std::int64_t); + : sizeof(std::uint64_t); } /*! @@ -1080,14 +1080,14 @@ class binary_writer write_bson_entry_header(name, 0x10 /* int32 */); write_number(static_cast(j.m_data.m_value.number_unsigned), true); } - else if (j.m_data.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) + else if (j.m_data.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) { - write_bson_entry_header(name, 0x12 /* int64 */); - write_number(static_cast(j.m_data.m_value.number_unsigned), true); + write_bson_entry_header(name, 0x11 /* uint64 */); + write_number(static_cast(j.m_data.m_value.number_unsigned), true); } else { - JSON_THROW(out_of_range::create(407, concat("integer number ", std::to_string(j.m_data.m_value.number_unsigned), " cannot be represented by BSON as it does not fit int64"), &j)); + JSON_THROW(out_of_range::create(407, concat("unsigned integer number ", std::to_string(j.m_data.m_value.number_unsigned), " cannot be represented by BSON as it does not fit into uint64"), &j)); } } From d56e85ecbbad6d3a2fb5f58fa2eae63cc09294f8 Mon Sep 17 00:00:00 2001 From: slowriot Date: Fri, 24 May 2024 13:51:57 +0100 Subject: [PATCH 03/14] Update tests to expect uint64_t support --- tests/src/unit-bson.cpp | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/tests/src/unit-bson.cpp b/tests/src/unit-bson.cpp index 13216f2f5a..2e4f4189f3 100644 --- a/tests/src/unit-bson.cpp +++ b/tests/src/unit-bson.cpp @@ -331,7 +331,6 @@ TEST_CASE("BSON") SECTION("non-empty object with unsigned integer (64-bit) member") { - // directly encoding uint64 is not supported in bson (only for timestamp values) json const j = { { "entry", std::uint64_t{0x1234567804030201} } @@ -340,7 +339,7 @@ TEST_CASE("BSON") std::vector const expected = { 0x14, 0x00, 0x00, 0x00, // size (little endian) - 0x12, /// entry: int64 + 0x11, /// entry: uint64 'e', 'n', 't', 'r', 'y', '\x00', 0x01, 0x02, 0x03, 0x04, 0x78, 0x56, 0x34, 0x12, 0x00 // end marker @@ -1134,7 +1133,7 @@ TEST_CASE("BSON numerical data") std::vector const expected_bson = { 0x14u, 0x00u, 0x00u, 0x00u, // size (little endian) - 0x12u, /// entry: int64 + 0x11u, /// entry: uint64 'e', 'n', 't', 'r', 'y', '\x00', static_cast((iu >> (8u * 0u)) & 0xffu), static_cast((iu >> (8u * 1u)) & 0xffu), @@ -1184,7 +1183,7 @@ TEST_CASE("BSON numerical data") std::vector const expected_bson = { 0x14u, 0x00u, 0x00u, 0x00u, // size (little endian) - 0x12u, /// entry: int64 + 0x11u, /// entry: uint64 'e', 'n', 't', 'r', 'y', '\x00', static_cast((iu >> (8u * 0u)) & 0xffu), static_cast((iu >> (8u * 1u)) & 0xffu), @@ -1197,12 +1196,15 @@ TEST_CASE("BSON numerical data") 0x00u // end marker }; - CHECK_THROWS_AS(json::to_bson(j), json::out_of_range&); -#if JSON_DIAGNOSTICS - CHECK_THROWS_WITH_STD_STR(json::to_bson(j), "[json.exception.out_of_range.407] (/entry) integer number " + std::to_string(i) + " cannot be represented by BSON as it does not fit int64"); -#else - CHECK_THROWS_WITH_STD_STR(json::to_bson(j), "[json.exception.out_of_range.407] integer number " + std::to_string(i) + " cannot be represented by BSON as it does not fit int64"); -#endif + const auto bson = json::to_bson(j); + CHECK(bson == expected_bson); + + auto j_roundtrip = json::from_bson(bson); + + CHECK(j.at("entry").is_number_unsigned()); + CHECK(j_roundtrip.at("entry").is_number_integer()); + CHECK(j_roundtrip == j); + CHECK(json::from_bson(bson, true, false) == j); } } From 31863bc3e777d45da203535f76bcdd8b3db5045d Mon Sep 17 00:00:00 2001 From: slowriot Date: Fri, 24 May 2024 13:54:52 +0100 Subject: [PATCH 04/14] make pretty --- include/nlohmann/detail/input/binary_reader.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index 89c0cd2b53..e12d3165a0 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -318,7 +318,7 @@ class binary_reader std::int32_t value{}; return get_number(input_format_t::bson, value) && sax->number_integer(value); } - + case 0x11: // uint64 { std::uint64_t value{}; From 7c616a24f5bff374d15ed578b7bb1bf00bbf26b3 Mon Sep 17 00:00:00 2001 From: slowriot Date: Fri, 24 May 2024 13:55:49 +0100 Subject: [PATCH 05/14] Amalgamate --- single_include/nlohmann/json.hpp | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index a858728c4c..6f74a4a526 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -9465,6 +9465,12 @@ class binary_reader return get_number(input_format_t::bson, value) && sax->number_integer(value); } + case 0x11: // uint64 + { + std::uint64_t value{}; + return get_number(input_format_t::bson, value) && sax->number_integer(value); + } + case 0x12: // int64 { std::int64_t value{}; @@ -16106,7 +16112,7 @@ class binary_writer { return (value <= static_cast((std::numeric_limits::max)())) ? sizeof(std::int32_t) - : sizeof(std::int64_t); + : sizeof(std::uint64_t); } /*! @@ -16120,14 +16126,14 @@ class binary_writer write_bson_entry_header(name, 0x10 /* int32 */); write_number(static_cast(j.m_data.m_value.number_unsigned), true); } - else if (j.m_data.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) + else if (j.m_data.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) { - write_bson_entry_header(name, 0x12 /* int64 */); - write_number(static_cast(j.m_data.m_value.number_unsigned), true); + write_bson_entry_header(name, 0x11 /* uint64 */); + write_number(static_cast(j.m_data.m_value.number_unsigned), true); } else { - JSON_THROW(out_of_range::create(407, concat("integer number ", std::to_string(j.m_data.m_value.number_unsigned), " cannot be represented by BSON as it does not fit int64"), &j)); + JSON_THROW(out_of_range::create(407, concat("unsigned integer number ", std::to_string(j.m_data.m_value.number_unsigned), " cannot be represented by BSON as it does not fit into uint64"), &j)); } } From 3c4c28adaec1f78a5e3fd33a9b0489eaeb3352be Mon Sep 17 00:00:00 2001 From: slowriot Date: Fri, 24 May 2024 17:58:40 +0100 Subject: [PATCH 06/14] Remove unnecessary cast --- include/nlohmann/detail/output/binary_writer.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index 6ccc98e19f..bc508ac4f1 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -1080,7 +1080,7 @@ class binary_writer write_bson_entry_header(name, 0x10 /* int32 */); write_number(static_cast(j.m_data.m_value.number_unsigned), true); } - else if (j.m_data.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) + else if (j.m_data.m_value.number_unsigned <= std::numeric_limits::max()) { write_bson_entry_header(name, 0x11 /* uint64 */); write_number(static_cast(j.m_data.m_value.number_unsigned), true); From d14286388f27e0e2a81dfb7d9edf2d2533f380ef Mon Sep 17 00:00:00 2001 From: slowriot Date: Thu, 7 Nov 2024 13:43:16 +0000 Subject: [PATCH 07/14] Remove BSON integer limit --- docs/mkdocs/docs/home/exceptions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/mkdocs/docs/home/exceptions.md b/docs/mkdocs/docs/home/exceptions.md index a0fee9e333..1df26d2820 100644 --- a/docs/mkdocs/docs/home/exceptions.md +++ b/docs/mkdocs/docs/home/exceptions.md @@ -830,7 +830,7 @@ A parsed number could not be stored as without changing it to NaN or INF. ### json.exception.out_of_range.407 -UBJSON and BSON only support integer numbers up to 9223372036854775807. +UBJSON only supports integer numbers up to 9223372036854775807. !!! failure "Example message" From 8fa46288b750da8977028c928742b9290ed3ed90 Mon Sep 17 00:00:00 2001 From: slowriot Date: Thu, 7 Nov 2024 13:50:16 +0000 Subject: [PATCH 08/14] Update bson mapping tables --- docs/mkdocs/docs/features/binary_formats/bson.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/mkdocs/docs/features/binary_formats/bson.md b/docs/mkdocs/docs/features/binary_formats/bson.md index f3b8cf18d0..e42bade216 100644 --- a/docs/mkdocs/docs/features/binary_formats/bson.md +++ b/docs/mkdocs/docs/features/binary_formats/bson.md @@ -22,6 +22,7 @@ The library uses the following mapping from JSON values types to BSON types: | number_integer | -2147483648..2147483647 | int32 | 0x10 | | number_integer | 2147483648..9223372036854775807 | int64 | 0x12 | | number_unsigned | 0..2147483647 | int32 | 0x10 | +| number_unsigned | 0..18446744073709551615 | uint64 | 0x11 | | number_unsigned | 2147483648..9223372036854775807 | int64 | 0x12 | | number_unsigned | 9223372036854775808..18446744073709551615 | -- | -- | | number_float | *any value* | double | 0x01 | @@ -73,7 +74,7 @@ The library maps BSON record types to JSON value types as follows: | Symbol | 0x0E | *unsupported* | | JavaScript Code | 0x0F | *unsupported* | | int32 | 0x10 | number_integer | -| Timestamp | 0x11 | *unsupported* | +| Timestamp | 0x11 | number_unsigned | | 128-bit decimal float | 0x13 | *unsupported* | | Max Key | 0x7F | *unsupported* | | Min Key | 0xFF | *unsupported* | From 5345c0708380e59f8bdc8fe0b64439990eb0b96c Mon Sep 17 00:00:00 2001 From: slowriot Date: Thu, 7 Nov 2024 13:59:52 +0000 Subject: [PATCH 09/14] Correct serialisation table --- docs/mkdocs/docs/features/binary_formats/bson.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/mkdocs/docs/features/binary_formats/bson.md b/docs/mkdocs/docs/features/binary_formats/bson.md index e42bade216..a162672631 100644 --- a/docs/mkdocs/docs/features/binary_formats/bson.md +++ b/docs/mkdocs/docs/features/binary_formats/bson.md @@ -22,9 +22,7 @@ The library uses the following mapping from JSON values types to BSON types: | number_integer | -2147483648..2147483647 | int32 | 0x10 | | number_integer | 2147483648..9223372036854775807 | int64 | 0x12 | | number_unsigned | 0..2147483647 | int32 | 0x10 | -| number_unsigned | 0..18446744073709551615 | uint64 | 0x11 | -| number_unsigned | 2147483648..9223372036854775807 | int64 | 0x12 | -| number_unsigned | 9223372036854775808..18446744073709551615 | -- | -- | +| number_unsigned | 2147483648..18446744073709551615 | int64 | 0x11 | | number_float | *any value* | double | 0x01 | | string | *any value* | string | 0x02 | | array | *any value* | document | 0x04 | From 36ce98472008b316bd393c9cdc0a59f5f72046f4 Mon Sep 17 00:00:00 2001 From: SlowRiot Date: Fri, 10 Jan 2025 18:54:27 +0000 Subject: [PATCH 10/14] amalgamate --- single_include/nlohmann/json.hpp | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 25d4a44929..2063883c39 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -10003,7 +10003,7 @@ class binary_reader case 0x11: // uint64 { std::uint64_t value{}; - return get_number(input_format_t::bson, value) && sax->number_integer(value); + return get_number(input_format_t::bson, value) && sax->number_unsigned(value); } case 0x12: // int64 @@ -10012,11 +10012,6 @@ class binary_reader return get_number(input_format_t::bson, value) && sax->number_integer(value); } - case 0x11: // uint64 - { - std::uint64_t value{}; - return get_number(input_format_t::bson, value) && sax->number_unsigned(value); - } default: // anything else not supported (yet) { @@ -16738,7 +16733,7 @@ class binary_writer write_bson_entry_header(name, 0x10 /* int32 */); write_number(static_cast(j.m_data.m_value.number_unsigned), true); } - else if (j.m_data.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) + else if (j.m_data.m_value.number_unsigned <= std::numeric_limits::max()) { write_bson_entry_header(name, 0x11 /* uint64 */); write_number(static_cast(j.m_data.m_value.number_unsigned), true); From f24ecf7145a487633008a724c9c2aaea8a11a300 Mon Sep 17 00:00:00 2001 From: SlowRiot Date: Sat, 11 Jan 2025 16:18:05 +0000 Subject: [PATCH 11/14] revert leftover whitespace changes from previous merge conflict resolution --- .../docs/features/binary_formats/bson.md | 4 ++-- docs/mkdocs/docs/home/exceptions.md | 22 +++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/mkdocs/docs/features/binary_formats/bson.md b/docs/mkdocs/docs/features/binary_formats/bson.md index 281bda6f59..3e90a116ab 100644 --- a/docs/mkdocs/docs/features/binary_formats/bson.md +++ b/docs/mkdocs/docs/features/binary_formats/bson.md @@ -8,7 +8,7 @@ representation of data types that are not part of the JSON spec. For example, BS - [BSON Website](http://bsonspec.org) - the main source on BSON - [BSON Specification](http://bsonspec.org/spec.html) - the specification - + ## Serialization @@ -43,7 +43,7 @@ The library uses the following mapping from JSON values types to BSON types: ```cpp --8<-- "examples/to_bson.cpp" ``` - + Output: ```c diff --git a/docs/mkdocs/docs/home/exceptions.md b/docs/mkdocs/docs/home/exceptions.md index 5f14770c7c..1b8d7ef83f 100644 --- a/docs/mkdocs/docs/home/exceptions.md +++ b/docs/mkdocs/docs/home/exceptions.md @@ -47,7 +47,7 @@ Note that [`JSON_THROW_USER`](../api/macros/json_throw_user.md) should leave the ```cpp #include - + #define JSON_TRY_USER if(true) #define JSON_CATCH_USER(exception) if(false) #define JSON_THROW_USER(exception) \ @@ -55,7 +55,7 @@ Note that [`JSON_THROW_USER`](../api/macros/json_throw_user.md) should leave the << " (function " << __FUNCTION__ << ") - " \ << (exception).what() << std::endl; \ std::abort();} - + #include ``` @@ -72,7 +72,7 @@ Exceptions in the library are thrown in the local context of the JSON value they ```cpp --8<-- "examples/diagnostics_standard.cpp" ``` - + Output: ``` @@ -90,7 +90,7 @@ As this global context comes at the price of storing one additional pointer per ```cpp --8<-- "examples/diagnostics_extended.cpp" ``` - + Output: ``` @@ -125,7 +125,7 @@ Exceptions have ids 1xx. ```cpp --8<-- "examples/parse_error.cpp" ``` - + Output: ``` @@ -377,7 +377,7 @@ Exceptions have ids 2xx. ```cpp --8<-- "examples/invalid_iterator.cpp" ``` - + Output: ``` @@ -541,7 +541,7 @@ Exceptions have ids 3xx. ```cpp --8<-- "examples/type_error.cpp" ``` - + Output: ``` @@ -735,7 +735,7 @@ The `dump()` function only works with UTF-8 encoded strings; that is, if you ass - Store the source file with UTF-8 encoding. - Pass an error handler as last parameter to the `dump()` function to avoid this exception: - - `json::error_handler_t::replace` will replace invalid bytes sequences with `U+FFFD` + - `json::error_handler_t::replace` will replace invalid bytes sequences with `U+FFFD` - `json::error_handler_t::ignore` will silently ignore invalid byte sequences ### json.exception.type_error.317 @@ -770,7 +770,7 @@ Exceptions have ids 4xx. ```cpp --8<-- "examples/out_of_range.cpp" ``` - + Output: ``` @@ -849,7 +849,7 @@ UBJSON only supports integer numbers up to 9223372036854775807. !!! note - Since version 3.9.0, integer numbers beyond int64 are serialized as high-precision UBJSON numbers, and this exception does not further occur. + Since version 3.9.0, integer numbers beyond int64 are serialized as high-precision UBJSON numbers, and this exception does not further occur. ### json.exception.out_of_range.408 @@ -885,7 +885,7 @@ Exceptions have ids 5xx. ```cpp --8<-- "examples/other_error.cpp" ``` - + Output: ``` From 465adfc23bc651b7a6cda35af369cdc22388551e Mon Sep 17 00:00:00 2001 From: SlowRiot Date: Sat, 11 Jan 2025 16:20:18 +0000 Subject: [PATCH 12/14] don't attempt to handle unsigned ints over 64bit --- include/nlohmann/detail/output/binary_writer.hpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index 905e1d06a8..6dff77a8cd 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -1090,15 +1090,11 @@ class binary_writer write_bson_entry_header(name, 0x10 /* int32 */); write_number(static_cast(j.m_data.m_value.number_unsigned), true); } - else if (j.m_data.m_value.number_unsigned <= std::numeric_limits::max()) + else { write_bson_entry_header(name, 0x11 /* uint64 */); write_number(static_cast(j.m_data.m_value.number_unsigned), true); } - else - { - JSON_THROW(out_of_range::create(407, concat("unsigned integer number ", std::to_string(j.m_data.m_value.number_unsigned), " cannot be represented by BSON as it does not fit into uint64"), &j)); - } } /*! From 01215ded0415f2d49b552951328c6b7483aeea59 Mon Sep 17 00:00:00 2001 From: SlowRiot Date: Sat, 11 Jan 2025 16:21:49 +0000 Subject: [PATCH 13/14] amalgamate --- single_include/nlohmann/json.hpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 2063883c39..e000f94a28 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -16733,15 +16733,11 @@ class binary_writer write_bson_entry_header(name, 0x10 /* int32 */); write_number(static_cast(j.m_data.m_value.number_unsigned), true); } - else if (j.m_data.m_value.number_unsigned <= std::numeric_limits::max()) + else { write_bson_entry_header(name, 0x11 /* uint64 */); write_number(static_cast(j.m_data.m_value.number_unsigned), true); } - else - { - JSON_THROW(out_of_range::create(407, concat("unsigned integer number ", std::to_string(j.m_data.m_value.number_unsigned), " cannot be represented by BSON as it does not fit into uint64"), &j)); - } } /*! From 24de2abc39689be9e5efa58151886c170eea3f79 Mon Sep 17 00:00:00 2001 From: SlowRiot Date: Sat, 11 Jan 2025 16:23:43 +0000 Subject: [PATCH 14/14] revert leftover whitespace change from merge --- docs/mkdocs/docs/features/binary_formats/bson.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/mkdocs/docs/features/binary_formats/bson.md b/docs/mkdocs/docs/features/binary_formats/bson.md index 3e90a116ab..3b925dd8db 100644 --- a/docs/mkdocs/docs/features/binary_formats/bson.md +++ b/docs/mkdocs/docs/features/binary_formats/bson.md @@ -97,5 +97,5 @@ The library maps BSON record types to JSON value types as follows: !!! note "Handling of BSON type 0x11" - BSON type 0x11 is used to represent uint64 numbers. This library treats these values purely as uint64 numbers - and does not parse them into date-related formats. + BSON type 0x11 is used to represent uint64 numbers. This library treats these values purely as uint64 numbers + and does not parse them into date-related formats. \ No newline at end of file