From 59e51bf811a5907983b84f3e2ed14e47a7210e75 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Tue, 26 Mar 2024 13:31:10 +0100 Subject: add yaml serialization support (#3) this change adds serialization of payloads as YAML, but not parsing. The implementation is somewhat based on the JSON path, and may be collapsed eventually as it is possible to serialize JSON format using the same code it also separates out the JSON serialization into a separate file for ease of maintenance any HTTP request response may be formatted as yaml by using a `.yaml` suffix or an `Accept: text/yaml` header --- src/zencore/compactbinary.cpp | 781 ------------------------------------------ 1 file changed, 781 deletions(-) (limited to 'src/zencore/compactbinary.cpp') diff --git a/src/zencore/compactbinary.cpp b/src/zencore/compactbinary.cpp index 6677b5a61..0907f8a2e 100644 --- a/src/zencore/compactbinary.cpp +++ b/src/zencore/compactbinary.cpp @@ -24,10 +24,6 @@ # include #endif -ZEN_THIRD_PARTY_INCLUDES_START -#include -ZEN_THIRD_PARTY_INCLUDES_END - namespace zen { const int DaysToMonth[] = {0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334, 365}; @@ -1468,339 +1464,6 @@ SaveCompactBinary(BinaryWriter& Ar, const CbObjectView& Object) ////////////////////////////////////////////////////////////////////////// -class CbJsonWriter -{ -public: - explicit CbJsonWriter(StringBuilderBase& InBuilder) : Builder(InBuilder) { NewLineAndIndent << LINE_TERMINATOR_ANSI; } - - void BeginObject() - { - Builder << '{'; - NewLineAndIndent << '\t'; - NeedsNewLine = true; - } - - void EndObject() - { - NewLineAndIndent.RemoveSuffix(1); - if (NeedsComma) - { - WriteOptionalNewLine(); - } - Builder << '}'; - } - - void BeginArray() - { - Builder << '['; - NewLineAndIndent << '\t'; - NeedsNewLine = true; - } - - void EndArray() - { - NewLineAndIndent.RemoveSuffix(1); - if (NeedsComma) - { - WriteOptionalNewLine(); - } - Builder << ']'; - } - - void WriteField(CbFieldView Field) - { - using namespace std::literals; - - WriteOptionalComma(); - WriteOptionalNewLine(); - - if (std::u8string_view Name = Field.GetU8Name(); !Name.empty()) - { - AppendQuotedString(Name); - Builder << ": "sv; - } - - switch (CbValue Accessor = Field.GetValue(); Accessor.GetType()) - { - case CbFieldType::Null: - Builder << "null"sv; - break; - case CbFieldType::Object: - case CbFieldType::UniformObject: - { - BeginObject(); - for (CbFieldView It : Field) - { - WriteField(It); - } - EndObject(); - } - break; - case CbFieldType::Array: - case CbFieldType::UniformArray: - { - BeginArray(); - for (CbFieldView It : Field) - { - WriteField(It); - } - EndArray(); - } - break; - case CbFieldType::Binary: - AppendBase64String(Accessor.AsBinary()); - break; - case CbFieldType::String: - AppendQuotedString(Accessor.AsU8String()); - break; - case CbFieldType::IntegerPositive: - Builder << Accessor.AsIntegerPositive(); - break; - case CbFieldType::IntegerNegative: - Builder << Accessor.AsIntegerNegative(); - break; - case CbFieldType::Float32: - { - const float Value = Accessor.AsFloat32(); - if (std::isfinite(Value)) - { - Builder.Append(fmt::format("{:.9g}", Value)); - } - else - { - Builder << "null"sv; - } - } - break; - case CbFieldType::Float64: - { - const double Value = Accessor.AsFloat64(); - if (std::isfinite(Value)) - { - Builder.Append(fmt::format("{:.17g}", Value)); - } - else - { - Builder << "null"sv; - } - } - break; - case CbFieldType::BoolFalse: - Builder << "false"sv; - break; - case CbFieldType::BoolTrue: - Builder << "true"sv; - break; - case CbFieldType::ObjectAttachment: - case CbFieldType::BinaryAttachment: - { - Builder << '"'; - Accessor.AsAttachment().ToHexString(Builder); - Builder << '"'; - } - break; - case CbFieldType::Hash: - { - Builder << '"'; - Accessor.AsHash().ToHexString(Builder); - Builder << '"'; - } - break; - case CbFieldType::Uuid: - { - Builder << '"'; - Accessor.AsUuid().ToString(Builder); - Builder << '"'; - } - break; - case CbFieldType::DateTime: - Builder << '"' << DateTime(Accessor.AsDateTimeTicks()).ToIso8601() << '"'; - break; - case CbFieldType::TimeSpan: - { - const TimeSpan Span(Accessor.AsTimeSpanTicks()); - if (Span.GetDays() == 0) - { - Builder << '"' << Span.ToString("%h:%m:%s.%n") << '"'; - } - else - { - Builder << '"' << Span.ToString("%d.%h:%m:%s.%n") << '"'; - } - break; - } - case CbFieldType::ObjectId: - Builder << '"'; - Accessor.AsObjectId().ToString(Builder); - Builder << '"'; - break; - case CbFieldType::CustomById: - { - CbCustomById Custom = Accessor.AsCustomById(); - Builder << "{ \"Id\": "; - Builder << Custom.Id; - Builder << ", \"Data\": "; - AppendBase64String(Custom.Data); - Builder << " }"; - break; - } - case CbFieldType::CustomByName: - { - CbCustomByName Custom = Accessor.AsCustomByName(); - Builder << "{ \"Name\": "; - AppendQuotedString(Custom.Name); - Builder << ", \"Data\": "; - AppendBase64String(Custom.Data); - Builder << " }"; - break; - } - default: - ZEN_ASSERT_FORMAT(false, "invalid field type: {}", uint8_t(Accessor.GetType())); - break; - } - - NeedsComma = true; - NeedsNewLine = true; - } - -private: - void WriteOptionalComma() - { - if (NeedsComma) - { - NeedsComma = false; - Builder << ','; - } - } - - void WriteOptionalNewLine() - { - if (NeedsNewLine) - { - NeedsNewLine = false; - Builder << NewLineAndIndent; - } - } - - void AppendQuotedString(std::u8string_view Value) - { - using namespace std::literals; - - const AsciiSet EscapeSet( - "\\\"\b\f\n\r\t" - "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" - "\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"); - - Builder << '\"'; - while (!Value.empty()) - { - std::u8string_view Verbatim = AsciiSet::FindPrefixWithout(Value, EscapeSet); - Builder << Verbatim; - - Value = Value.substr(Verbatim.size()); - - std::u8string_view Escape = AsciiSet::FindPrefixWith(Value, EscapeSet); - for (char Char : Escape) - { - switch (Char) - { - case '\\': - Builder << "\\\\"sv; - break; - case '\"': - Builder << "\\\""sv; - break; - case '\b': - Builder << "\\b"sv; - break; - case '\f': - Builder << "\\f"sv; - break; - case '\n': - Builder << "\\n"sv; - break; - case '\r': - Builder << "\\r"sv; - break; - case '\t': - Builder << "\\t"sv; - break; - default: - Builder << Char; - break; - } - } - Value = Value.substr(Escape.size()); - } - Builder << '\"'; - } - - void AppendBase64String(MemoryView Value) - { - Builder << '"'; - ZEN_ASSERT(Value.GetSize() <= 512 * 1024 * 1024); - const uint32_t EncodedSize = Base64::GetEncodedDataSize(uint32_t(Value.GetSize())); - const size_t EncodedIndex = Builder.AddUninitialized(size_t(EncodedSize)); - Base64::Encode(static_cast(Value.GetData()), uint32_t(Value.GetSize()), Builder.Data() + EncodedIndex); - } - -private: - StringBuilderBase& Builder; - ExtendableStringBuilder<32> NewLineAndIndent; - bool NeedsComma{false}; - bool NeedsNewLine{false}; -}; - -void -CompactBinaryToJson(const CbObjectView& Object, StringBuilderBase& Builder) -{ - CbJsonWriter Writer(Builder); - Writer.WriteField(Object.AsFieldView()); -} - -void -CompactBinaryToJson(const CbArrayView& Array, StringBuilderBase& Builder) -{ - CbJsonWriter Writer(Builder); - Writer.WriteField(Array.AsFieldView()); -} - -void -CompactBinaryToJson(MemoryView Data, StringBuilderBase& InBuilder) -{ - std::vector Fields = ReadCompactBinaryStream(Data); - CbJsonWriter Writer(InBuilder); - if (!Fields.empty()) - { - if (Fields.size() == 1) - { - Writer.WriteField(Fields[0]); - return; - } - bool UseTopLevelObject = Fields[0].HasName(); - if (UseTopLevelObject) - { - Writer.BeginObject(); - } - else - { - Writer.BeginArray(); - } - for (const CbFieldView& Field : Fields) - { - Writer.WriteField(Field); - } - if (UseTopLevelObject) - { - Writer.EndObject(); - } - else - { - Writer.EndArray(); - } - } -} - std::vector ReadCompactBinaryStream(MemoryView Data) { @@ -1823,225 +1486,6 @@ ReadCompactBinaryStream(MemoryView Data) ////////////////////////////////////////////////////////////////////////// -class CbJsonReader -{ -public: - static CbFieldIterator Read(std::string_view JsonText, std::string& Error) - { - using namespace json11; - - const Json Json = Json::parse(std::string(JsonText), Error); - - if (Error.empty()) - { - CbWriter Writer; - if (ReadField(Writer, Json, std::string_view(), Error)) - { - return Writer.Save(); - } - } - - return CbFieldIterator(); - } - -private: - static bool ReadField(CbWriter& Writer, const json11::Json& Json, const std::string_view FieldName, std::string& Error) - { - using namespace json11; - - switch (Json.type()) - { - case Json::Type::OBJECT: - { - if (FieldName.empty()) - { - Writer.BeginObject(); - } - else - { - Writer.BeginObject(FieldName); - } - - for (const auto& Kv : Json.object_items()) - { - const std::string& Name = Kv.first; - const json11::Json& Item = Kv.second; - - if (ReadField(Writer, Item, Name, Error) == false) - { - return false; - } - } - - Writer.EndObject(); - } - break; - case Json::Type::ARRAY: - { - if (FieldName.empty()) - { - Writer.BeginArray(); - } - else - { - Writer.BeginArray(FieldName); - } - - for (const json11::Json& Item : Json.array_items()) - { - if (ReadField(Writer, Item, std::string_view(), Error) == false) - { - return false; - } - } - - Writer.EndArray(); - } - break; - case Json::Type::NUL: - { - if (FieldName.empty()) - { - Writer.AddNull(); - } - else - { - Writer.AddNull(FieldName); - } - } - break; - case Json::Type::BOOL: - { - if (FieldName.empty()) - { - Writer.AddBool(Json.bool_value()); - } - else - { - Writer.AddBool(FieldName, Json.bool_value()); - } - } - break; - case Json::Type::NUMBER: - { - if (FieldName.empty()) - { - Writer.AddFloat(Json.number_value()); - } - else - { - Writer.AddFloat(FieldName, Json.number_value()); - } - } - break; - case Json::Type::STRING: - { - Oid Id; - if (TryParseObjectId(Json.string_value(), Id)) - { - if (FieldName.empty()) - { - Writer.AddObjectId(Id); - } - else - { - Writer.AddObjectId(FieldName, Id); - } - - return true; - } - - IoHash Hash; - if (TryParseIoHash(Json.string_value(), Hash)) - { - if (FieldName.empty()) - { - Writer.AddHash(Hash); - } - else - { - Writer.AddHash(FieldName, Hash); - } - - return true; - } - - if (FieldName.empty()) - { - Writer.AddString(Json.string_value()); - } - else - { - Writer.AddString(FieldName, Json.string_value()); - } - } - break; - default: - break; - } - - return true; - } - - static constexpr AsciiSet HexCharSet = AsciiSet("0123456789abcdefABCDEF"); - - static bool TryParseObjectId(std::string_view Str, Oid& Id) - { - using namespace std::literals; - - if (Str.size() == Oid::StringLength && AsciiSet::HasOnly(Str, HexCharSet)) - { - Id = Oid::FromHexString(Str); - return true; - } - - if (Str.starts_with("0x"sv)) - { - return TryParseObjectId(Str.substr(2), Id); - } - - return false; - } - - static bool TryParseIoHash(std::string_view Str, IoHash& Hash) - { - using namespace std::literals; - - if (Str.size() == IoHash::StringLength && AsciiSet::HasOnly(Str, HexCharSet)) - { - Hash = IoHash::FromHexString(Str); - return true; - } - - if (Str.starts_with("0x"sv)) - { - return TryParseIoHash(Str.substr(2), Hash); - } - - return false; - } -}; - -CbFieldIterator -LoadCompactBinaryFromJson(std::string_view Json, std::string& Error) -{ - if (Json.empty() == false) - { - return CbJsonReader::Read(Json, Error); - } - - return CbFieldIterator(); -} - -CbFieldIterator -LoadCompactBinaryFromJson(std::string_view Json) -{ - std::string Error; - return LoadCompactBinaryFromJson(Json, Error); -} - -////////////////////////////////////////////////////////////////////////// - #if ZEN_WITH_TESTS void uson_forcelink() @@ -2211,130 +1655,6 @@ TEST_CASE("uson.null") } } -TEST_CASE("uson.json") -{ - using namespace std::literals; - - SUBCASE("string") - { - CbObjectWriter Writer; - Writer << "KeyOne" - << "ValueOne"; - Writer << "KeyTwo" - << "ValueTwo"; - CbObject Obj = Writer.Save(); - - StringBuilder<128> Sb; - const char* JsonText = Obj.ToJson(Sb).Data(); - - std::string JsonError; - json11::Json Json = json11::Json::parse(JsonText, JsonError); - - const std::string ValueOne = Json["KeyOne"].string_value(); - const std::string ValueTwo = Json["KeyTwo"].string_value(); - - CHECK(JsonError.empty()); - CHECK(ValueOne == "ValueOne"); - CHECK(ValueTwo == "ValueTwo"); - } - - SUBCASE("number") - { - const float ExpectedFloatValue = 21.21f; - const double ExpectedDoubleValue = 42.42; - - CbObjectWriter Writer; - Writer << "Float" << ExpectedFloatValue; - Writer << "Double" << ExpectedDoubleValue; - - CbObject Obj = Writer.Save(); - - StringBuilder<128> Sb; - const char* JsonText = Obj.ToJson(Sb).Data(); - - std::string JsonError; - json11::Json Json = json11::Json::parse(JsonText, JsonError); - - const float FloatValue = float(Json["Float"].number_value()); - const double DoubleValue = Json["Double"].number_value(); - - CHECK(JsonError.empty()); - CHECK(FloatValue == Approx(ExpectedFloatValue)); - CHECK(DoubleValue == Approx(ExpectedDoubleValue)); - } - - SUBCASE("number.nan") - { - constexpr float FloatNan = std::numeric_limits::quiet_NaN(); - constexpr double DoubleNan = std::numeric_limits::quiet_NaN(); - - CbObjectWriter Writer; - Writer << "FloatNan" << FloatNan; - Writer << "DoubleNan" << DoubleNan; - - CbObject Obj = Writer.Save(); - - StringBuilder<128> Sb; - const char* JsonText = Obj.ToJson(Sb).Data(); - - std::string JsonError; - json11::Json Json = json11::Json::parse(JsonText, JsonError); - - const double FloatValue = Json["FloatNan"].number_value(); - const double DoubleValue = Json["DoubleNan"].number_value(); - - CHECK(JsonError.empty()); - CHECK(FloatValue == 0); - CHECK(DoubleValue == 0); - } - - SUBCASE("stream") - { - const auto MakeObject = [&](std::string_view Name, const std::vector& Fields) -> CbObject { - CbWriter Writer; - Writer.SetName(Name); - Writer.BeginObject(); - for (const auto& Field : Fields) - { - Writer.AddInteger(fmt::format("{}", Field), Field); - } - Writer.EndObject(); - return Writer.Save().AsObject(); - }; - - std::vector Buffer; - - auto AppendToBuffer = [&](const void* Data, size_t Count) { - const uint8_t* AppendBytes = reinterpret_cast(Data); - Buffer.insert(Buffer.end(), AppendBytes, AppendBytes + Count); - }; - - auto Append = [&](const CbFieldView& Field) { - Field.WriteToStream([&](const void* Data, size_t Count) { - const uint8_t* AppendBytes = reinterpret_cast(Data); - Buffer.insert(Buffer.end(), AppendBytes, AppendBytes + Count); - }); - }; - - CbObject DataObjects[] = {MakeObject("Empty object"sv, {}), - MakeObject("OneField object"sv, {5}), - MakeObject("TwoField object"sv, {-5, 999}), - MakeObject("ThreeField object"sv, {1, 2, -129})}; - for (const CbObject& Object : DataObjects) - { - Object.AsField().WriteToStream(AppendToBuffer); - } - - ExtendableStringBuilder<128> Sb; - CompactBinaryToJson(MemoryView(Buffer.data(), Buffer.size()), Sb); - std::string JsonText = Sb.ToString().c_str(); - std::string JsonError; - json11::Json Json = json11::Json::parse(JsonText, JsonError); - std::string ParsedJsonString = Json.dump(); - CHECK(!ParsedJsonString.empty()); - } -} - TEST_CASE("uson.datetime") { using namespace std::literals; @@ -2362,107 +1682,6 @@ TEST_CASE("uson.datetime") } } -TEST_CASE("json.uson") -{ - using namespace std::literals; - using namespace json11; - - SUBCASE("empty") - { - CbFieldIterator It = LoadCompactBinaryFromJson(""sv); - CHECK(It.HasValue() == false); - } - - SUBCASE("object") - { - const Json JsonObject = Json::object{{"Null", nullptr}, - {"String", "Value1"}, - {"Bool", true}, - {"Number", 46.2}, - {"Array", Json::array{1, 2, 3}}, - {"Object", - Json::object{ - {"String", "Value2"}, - }}}; - - CbObject Cb = LoadCompactBinaryFromJson(JsonObject.dump()).AsObject(); - - CHECK(Cb["Null"].IsNull()); - CHECK(Cb["String"].AsString() == "Value1"sv); - CHECK(Cb["Bool"].AsBool()); - CHECK(Cb["Number"].AsDouble() == 46.2); - CHECK(Cb["Object"].IsObject()); - CbObjectView Object = Cb["Object"].AsObjectView(); - CHECK(Object["String"].AsString() == "Value2"sv); - } - - SUBCASE("array") - { - const Json JsonArray = Json::array{42, 43, 44}; - CbArray Cb = LoadCompactBinaryFromJson(JsonArray.dump()).AsArray(); - - auto It = Cb.CreateIterator(); - CHECK((*It).AsDouble() == 42); - It++; - CHECK((*It).AsDouble() == 43); - It++; - CHECK((*It).AsDouble() == 44); - } - - SUBCASE("objectid") - { - const Oid& Id = Oid::NewOid(); - - StringBuilder<64> Sb; - Id.ToString(Sb); - - Json JsonObject = Json::object{{"value", Sb.ToString()}}; - CbObject Cb = LoadCompactBinaryFromJson(JsonObject.dump()).AsObject(); - - CHECK(Cb["value"sv].IsObjectId()); - CHECK(Cb["value"sv].AsObjectId() == Id); - - Sb.Reset(); - Sb << "0x"; - Id.ToString(Sb); - - JsonObject = Json::object{{"value", Sb.ToString()}}; - Cb = LoadCompactBinaryFromJson(JsonObject.dump()).AsObject(); - - CHECK(Cb["value"sv].IsObjectId()); - CHECK(Cb["value"sv].AsObjectId() == Id); - } - - SUBCASE("iohash") - { - const uint8_t Data[] = { - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - }; - - const IoHash Hash = IoHash::HashBuffer(Data, sizeof(Data)); - - Json JsonObject = Json::object{{"value", Hash.ToHexString()}}; - CbObject Cb = LoadCompactBinaryFromJson(JsonObject.dump()).AsObject(); - - CHECK(Cb["value"sv].IsHash()); - CHECK(Cb["value"sv].AsHash() == Hash); - - JsonObject = Json::object{{"value", "0x" + Hash.ToHexString()}}; - Cb = LoadCompactBinaryFromJson(JsonObject.dump()).AsObject(); - - CHECK(Cb["value"sv].IsHash()); - CHECK(Cb["value"sv].AsHash() == Hash); - } -} - ////////////////////////////////////////////////////////////////////////// TEST_SUITE_BEGIN("core.datetime"); -- cgit v1.2.3 From 6d634ab59c05adf1ba028d95b16031a7f8e8db2a Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 18 Apr 2024 12:44:26 +0200 Subject: improved lock file handling (#50) - Feature: `zen down` - --`data-dir` to specify a data directory to deduce which zen instance to bring down - Feature: `zen attach` - --`data-dir` to specify a data directory to deduce which zen instance to attach to222 - Feature: `zen status` - --`port` filter running zen instances based on port - --`data-dir` filter running zen instances based on information in the data directory - Improvement: Trying to load a compact binary object from an empty file no longer causes access violation --- src/zencore/compactbinary.cpp | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) (limited to 'src/zencore/compactbinary.cpp') diff --git a/src/zencore/compactbinary.cpp b/src/zencore/compactbinary.cpp index 0907f8a2e..029889e85 100644 --- a/src/zencore/compactbinary.cpp +++ b/src/zencore/compactbinary.cpp @@ -1421,25 +1421,43 @@ LoadCompactBinary(BinaryReader& Ar, BufferAllocator Allocator) CbObject LoadCompactBinaryObject(IoBuffer&& Payload) { + if (Payload.GetSize() == 0) + { + return CbObject(); + } return CbObject{SharedBuffer(std::move(Payload))}; } CbObject LoadCompactBinaryObject(const IoBuffer& Payload) { + if (Payload.GetSize() == 0) + { + return CbObject(); + } return CbObject{SharedBuffer(Payload)}; } CbObject LoadCompactBinaryObject(CompressedBuffer&& Payload) { - return CbObject{SharedBuffer(Payload.DecompressToComposite().Flatten())}; + CompositeBuffer Decompressed = std::move(Payload).DecompressToComposite(); + if (Decompressed.GetSize() == 0) + { + return CbObject(); + } + return CbObject{std::move(Decompressed).Flatten()}; } CbObject LoadCompactBinaryObject(const CompressedBuffer& Payload) { - return CbObject{SharedBuffer(Payload.DecompressToComposite().Flatten())}; + CompositeBuffer Decompressed = Payload.DecompressToComposite(); + if (Decompressed.GetSize() == 0) + { + return CbObject(); + } + return CbObject{std::move(Decompressed).Flatten()}; } ////////////////////////////////////////////////////////////////////////// -- cgit v1.2.3 From 54ee1372c4254e185e83c8eb9329ace9704664c6 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 30 Sep 2024 09:27:36 +0200 Subject: optimize startup time (#175) * use tsl::robin_set for BlockIndexSet don't calculate full block location when only block index is needed * don't copy visitor function * reserve space for attachments --- src/zencore/compactbinary.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/zencore/compactbinary.cpp') diff --git a/src/zencore/compactbinary.cpp b/src/zencore/compactbinary.cpp index 029889e85..adccaba70 100644 --- a/src/zencore/compactbinary.cpp +++ b/src/zencore/compactbinary.cpp @@ -407,7 +407,7 @@ CbFieldView::CbFieldView(const void* DataPointer, CbFieldType FieldType) } void -CbFieldView::IterateAttachments(std::function Visitor) const +CbFieldView::IterateAttachments(const std::function& Visitor) const { switch (CbFieldTypeOps::GetType(Type)) { @@ -1170,7 +1170,7 @@ template class TCbFieldIterator; template void -TCbFieldIterator::IterateRangeAttachments(std::function Visitor) const +TCbFieldIterator::IterateRangeAttachments(const std::function& Visitor) const { if (CbFieldTypeOps::HasFieldType(FieldType::GetType())) { -- cgit v1.2.3 From 920120bbcec9f91df3336f62970b3e010a4fa6c2 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Thu, 6 Mar 2025 16:18:32 +0100 Subject: reduced memory churn using fixed_xxx containers (#236) * Added EASTL to help with eliminating memory allocations * Applied EASTL to eliminate memory allocations, primarily by using `fixed_vector` et al to use stack allocations / inline struct allocations Reduces memory events in traces by close to a factor of 10 in test scenario (starting editor for project F) --- src/zencore/compactbinary.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'src/zencore/compactbinary.cpp') diff --git a/src/zencore/compactbinary.cpp b/src/zencore/compactbinary.cpp index adccaba70..b43cc18f1 100644 --- a/src/zencore/compactbinary.cpp +++ b/src/zencore/compactbinary.cpp @@ -15,6 +15,8 @@ #include #include +#include + #include #include @@ -1376,9 +1378,9 @@ TryMeasureCompactBinary(MemoryView View, CbFieldType& OutType, uint64_t& OutSize CbField LoadCompactBinary(BinaryReader& Ar, BufferAllocator Allocator) { - std::vector HeaderBytes; - CbFieldType FieldType; - uint64_t FieldSize = 1; + eastl::fixed_vector HeaderBytes; + CbFieldType FieldType; + uint64_t FieldSize = 1; for (const int64_t StartPos = Ar.CurrentOffset(); FieldSize > 0;) { @@ -1393,7 +1395,7 @@ LoadCompactBinary(BinaryReader& Ar, BufferAllocator Allocator) HeaderBytes.resize(ReadOffset + ReadSize); Ar.Read(HeaderBytes.data() + ReadOffset, ReadSize); - if (TryMeasureCompactBinary(MakeMemoryView(HeaderBytes), FieldType, FieldSize)) + if (TryMeasureCompactBinary(MakeMemoryView(HeaderBytes.data(), HeaderBytes.size()), FieldType, FieldSize)) { if (FieldSize <= uint64_t(Ar.Size() - StartPos)) { -- cgit v1.2.3