From ad464941474173a9ba8d860c88db4689611d50a6 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 21 May 2024 15:05:40 +0200 Subject: iobuffer memstomp fix (#85) * Don't try to validate empty compact binary * Don't try to free fake memory buffer pointer in iobuffer destructor --- src/zencore/compactbinaryvalidation.cpp | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'src/zencore/compactbinaryvalidation.cpp') diff --git a/src/zencore/compactbinaryvalidation.cpp b/src/zencore/compactbinaryvalidation.cpp index 462978f63..db830d250 100644 --- a/src/zencore/compactbinaryvalidation.cpp +++ b/src/zencore/compactbinaryvalidation.cpp @@ -544,10 +544,17 @@ ValidateCompactBinary(MemoryView View, CbValidateMode Mode, CbFieldType Type) CbValidateError Error = CbValidateError::None; if (EnumHasAnyFlags(Mode, CbValidateMode::All)) { - ValidateCbField(View, Mode, Error, Type); - if (!View.IsEmpty() && EnumHasAnyFlags(Mode, CbValidateMode::Padding)) + if (View.IsEmpty()) { - AddError(Error, CbValidateError::Padding); + AddError(Error, CbValidateError::OutOfBounds); + } + else + { + ValidateCbField(View, Mode, Error, Type); + if (!View.IsEmpty() && EnumHasAnyFlags(Mode, CbValidateMode::Padding)) + { + AddError(Error, CbValidateError::Padding); + } } } return Error; -- cgit v1.2.3 From d020b6522b2d962db67f8a66410e74d61cf3da24 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 17 Sep 2024 15:05:40 +0200 Subject: gc performance improvements (#160) * optimized ValidateCbUInt * optimized iohash comparision * replace unordered set/map with tsl/robin set/map in blockstore * increase max buffer size when writing cache bucket sidecar * only store meta data for files < 4Gb * faster ReadAttachmentsFromMetaData * remove memcpy call in BlockStoreDiskLocation * only write cache bucket state to disk if GC deleted anything --- src/zencore/compactbinaryvalidation.cpp | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) (limited to 'src/zencore/compactbinaryvalidation.cpp') diff --git a/src/zencore/compactbinaryvalidation.cpp b/src/zencore/compactbinaryvalidation.cpp index db830d250..82738e678 100644 --- a/src/zencore/compactbinaryvalidation.cpp +++ b/src/zencore/compactbinaryvalidation.cpp @@ -86,23 +86,24 @@ ValidateCbFieldType(MemoryView& View, CbValidateMode Mode, CbValidateError& Erro static uint64_t ValidateCbUInt(MemoryView& View, CbValidateMode Mode, CbValidateError& Error) { - if (View.GetSize() > 0 && View.GetSize() >= MeasureVarUInt(View.GetData())) + size_t ViewSize = View.GetSize(); + if (ViewSize > 0) { - uint32_t ValueByteCount; - const uint64_t Value = ReadVarUInt(View.GetData(), ValueByteCount); - if (EnumHasAnyFlags(Mode, CbValidateMode::Format) && ValueByteCount > MeasureVarUInt(Value)) + uint32_t ValueByteCount = MeasureVarUInt(View.GetData()); + if (ViewSize >= ValueByteCount) { - AddError(Error, CbValidateError::InvalidInteger); + const uint64_t Value = ReadMeasuredVarUInt(View.GetData(), ValueByteCount); + if (EnumHasAnyFlags(Mode, CbValidateMode::Format) && ValueByteCount != MeasureVarUInt(Value)) + { + AddError(Error, CbValidateError::InvalidInteger); + } + View += ValueByteCount; + return Value; } - View += ValueByteCount; - return Value; - } - else - { - AddError(Error, CbValidateError::OutOfBounds); - View.Reset(); - return 0; } + AddError(Error, CbValidateError::OutOfBounds); + View.Reset(); + return 0; } /** -- cgit v1.2.3 From 1e43506a365c91e411c2375ec55a7d5ad918df0e Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 25 Nov 2024 10:26:59 +0100 Subject: split zencore/memory.h -> memoryview.h, memcmp.h (#228) minor clean-up `zencore/memory.h` used to contain a variety of things including `Malloc` support along with `MemoryView` etc since the memory allocator stuff moved into `zencore/memory/memory.h` there was basically only `MemoryView` and `MemCmp` in there which seemed better to split out into separate headers to avoid overloading `memory.h` --- src/zencore/compactbinaryvalidation.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/zencore/compactbinaryvalidation.cpp') diff --git a/src/zencore/compactbinaryvalidation.cpp b/src/zencore/compactbinaryvalidation.cpp index 82738e678..6f53bba69 100644 --- a/src/zencore/compactbinaryvalidation.cpp +++ b/src/zencore/compactbinaryvalidation.cpp @@ -4,7 +4,7 @@ #include #include -#include +#include #include #include -- cgit v1.2.3 From 95fc783ae9914019fee03f12603e4c49f9ab6106 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Fri, 16 May 2025 12:08:05 +0100 Subject: validate custom fields (#399) * implemented validation of compact binary custom fields --- src/zencore/compactbinaryvalidation.cpp | 45 ++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) (limited to 'src/zencore/compactbinaryvalidation.cpp') diff --git a/src/zencore/compactbinaryvalidation.cpp b/src/zencore/compactbinaryvalidation.cpp index 6f53bba69..833649b88 100644 --- a/src/zencore/compactbinaryvalidation.cpp +++ b/src/zencore/compactbinaryvalidation.cpp @@ -134,6 +134,37 @@ ValidateCbFloat64(MemoryView& View, CbValidateMode Mode, CbValidateError& Error) } } +/** + * Validate and read a fixed-size value from the view. + * + * Modifies the view to start at the end of the value, and adds error flags if applicable. + */ +static MemoryView +ValidateCbFixedValue(MemoryView& View, CbValidateMode Mode, CbValidateError& Error, uint64_t Size) +{ + ZEN_UNUSED(Mode); + + const MemoryView Value = View.Left(Size); + View += Size; + if (Value.GetSize() < Size) + { + AddError(Error, CbValidateError::OutOfBounds); + } + return Value; +}; + +/** + * Validate and read a value from the view where the view begins with the value size. + * + * Modifies the view to start at the end of the value, and adds error flags if applicable. + */ +static MemoryView +ValidateCbDynamicValue(MemoryView& View, CbValidateMode Mode, CbValidateError& Error) +{ + const uint64_t ValueSize = ValidateCbUInt(View, Mode, Error); + return ValidateCbFixedValue(View, Mode, Error, ValueSize); +} + /** * Validate and read a string from the view. * @@ -378,8 +409,20 @@ ValidateCbField(MemoryView& View, CbValidateMode Mode, CbValidateError& Error, c ValidateFixedPayload(12); break; case CbFieldType::CustomById: + { + MemoryView Value = ValidateCbDynamicValue(View, Mode, Error); + ValidateCbUInt(Value, Mode, Error); + } + break; case CbFieldType::CustomByName: - ZEN_NOT_IMPLEMENTED(); // TODO: FIX! + { + MemoryView Value = ValidateCbDynamicValue(View, Mode, Error); + const std::string_view TypeName = ValidateCbString(Value, Mode, Error); + if (TypeName.empty() && !EnumHasAnyFlags(Error, CbValidateError::OutOfBounds)) + { + AddError(Error, CbValidateError::InvalidType); + } + } break; } -- cgit v1.2.3 From 6bdaf6ad6e1308aae12845b20bf06a4406ba0c03 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 19 Aug 2025 14:03:02 +0200 Subject: zen print fixes/improvements (#469) - Improvement: `zen print` now allows output of compact binary content even if they are in non-optimal format (Unifom vs Non-Uniform arrays and objects) - Feature: `zen print` now has a `--show-type-info` option to add type information to output of compact binary content - Bugfix: Stats information for Build Store (Zen Store Cache) no longer throws exception and outputs invalid state information --- src/zencore/compactbinaryvalidation.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'src/zencore/compactbinaryvalidation.cpp') diff --git a/src/zencore/compactbinaryvalidation.cpp b/src/zencore/compactbinaryvalidation.cpp index 833649b88..d7292f405 100644 --- a/src/zencore/compactbinaryvalidation.cpp +++ b/src/zencore/compactbinaryvalidation.cpp @@ -705,10 +705,11 @@ ToString(const CbValidateError Error) ExtendableStringBuilder<128> Out; - auto AppendFlag = [&, IsFirst = false](std::string_view FlagString) { + auto AppendFlag = [&, IsFirst = true](std::string_view FlagString) mutable { if (!IsFirst) Out.Append('|'); Out.Append(FlagString); + IsFirst = false; }; #define _ENUM_CASE(V) \ @@ -737,7 +738,11 @@ ToString(const CbValidateError Error) #undef _ENUM_CASE - return "Error"; + if (Out.Size() == 0) + { + return "Error"; + } + return Out.ToString(); } /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -- cgit v1.2.3