diff options
| author | Dan Engelbrecht <[email protected]> | 2025-08-19 14:03:02 +0200 |
|---|---|---|
| committer | GitHub Enterprise <[email protected]> | 2025-08-19 14:03:02 +0200 |
| commit | 6bdaf6ad6e1308aae12845b20bf06a4406ba0c03 (patch) | |
| tree | 718a2bb2016d4e88084c21e2868a3efca34a5926 /src | |
| parent | 5.6.17 (diff) | |
| download | zen-6bdaf6ad6e1308aae12845b20bf06a4406ba0c03.tar.xz zen-6bdaf6ad6e1308aae12845b20bf06a4406ba0c03.zip | |
zen print fixes/improvements (#469)
- Improvement: `zen print` now allows output of compact binary content even if they are in non-optimal format (Unifom vs Non-Uniform arrays and objects)
- Feature: `zen print` now has a `--show-type-info` option to add type information to output of compact binary content
- Bugfix: Stats information for Build Store (Zen Store Cache) no longer throws exception and outputs invalid state information
Diffstat (limited to 'src')
| -rw-r--r-- | src/zen/cmds/builds_cmd.cpp | 4 | ||||
| -rw-r--r-- | src/zen/cmds/print_cmd.cpp | 93 | ||||
| -rw-r--r-- | src/zen/cmds/print_cmd.h | 10 | ||||
| -rw-r--r-- | src/zencore/compactbinaryfile.cpp | 2 | ||||
| -rw-r--r-- | src/zencore/compactbinaryjson.cpp | 128 | ||||
| -rw-r--r-- | src/zencore/compactbinaryvalidation.cpp | 9 | ||||
| -rw-r--r-- | src/zencore/include/zencore/compactbinary.h | 4 | ||||
| -rw-r--r-- | src/zenserver/buildstore/httpbuildstore.cpp | 1 | ||||
| -rw-r--r-- | src/zenserver/projectstore/projectstore.cpp | 6 | ||||
| -rw-r--r-- | src/zenserver/zenserver.cpp | 4 | ||||
| -rw-r--r-- | src/zenstore/cas.cpp | 2 |
11 files changed, 223 insertions, 40 deletions
diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index cf5e8d9da..8692e5941 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -11200,7 +11200,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) /*RequireBucket*/ false); CbObject Response = Storage.BuildStorage->ListNamespaces(m_ListNamespacesRecursive); - ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::All) == CbValidateError::None); + ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None); if (m_ListResultPath.empty()) { ExtendableStringBuilder<1024> SB; @@ -11294,7 +11294,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) /*RequireBucket*/ false); CbObject Response = Storage.BuildStorage->ListBuilds(QueryObject); - ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::All) == CbValidateError::None); + ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None); if (m_ListResultPath.empty()) { ExtendableStringBuilder<1024> SB; diff --git a/src/zen/cmds/print_cmd.cpp b/src/zen/cmds/print_cmd.cpp index 469dddf55..5bc8a8ed8 100644 --- a/src/zen/cmds/print_cmd.cpp +++ b/src/zen/cmds/print_cmd.cpp @@ -15,25 +15,41 @@ using namespace std::literals; namespace zen { static void -PrintCbObject(CbObject Object) +PrintCbObject(CbObject Object, bool AddTypeComment) { ExtendableStringBuilder<1024> ObjStr; - CompactBinaryToJson(Object, ObjStr); + CompactBinaryToJson(Object, ObjStr, AddTypeComment); ZEN_CONSOLE("{}", ObjStr); } static void -PrintCompactBinary(IoBuffer Data) +PrintCompactBinary(IoBuffer Data, bool AddTypeComment) { ExtendableStringBuilder<1024> StreamString; - CompactBinaryToJson(Data.GetView(), StreamString); + CompactBinaryToJson(Data.GetView(), StreamString, AddTypeComment); ZEN_CONSOLE("{}", StreamString); } +static CbValidateError +MakeFilteredResult(CbValidateError Result) +{ + CbValidateError FilteredResult = Result; + EnumRemoveFlags(FilteredResult, + CbValidateError::InvalidString | CbValidateError::InvalidInteger | CbValidateError::InvalidFloat | + CbValidateError::NonUniformObject | CbValidateError::NonUniformArray | CbValidateError::Padding); + return FilteredResult; +} + PrintCommand::PrintCommand() { m_Options.add_options()("h,help", "Print help"); m_Options.add_option("", "s", "source", "Object payload file (use '-' to read from STDIN)", cxxopts::value(m_Filename), "<file name>"); + m_Options.add_option("", + "", + "show-type-info", + "Add type info annotation to compact binary objects", + cxxopts::value(m_ShowCbObjectTypeInfo), + "<showtypes>"); m_Options.parse_positional({"source"}); } @@ -62,6 +78,7 @@ PrintCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else { + MakeSafeAbsolutePathÍnPlace(m_Filename); Fc = ReadFile(m_Filename); } @@ -126,22 +143,59 @@ PrintCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ZEN_CONSOLE("---8<---"); - PrintCbObject(Object); - } - else if (CbValidateError Result = ValidateCompactBinary(Data, - CbValidateMode::Default | CbValidateMode::Names | CbValidateMode::Format | - CbValidateMode::Package | CbValidateMode::PackageHash); - Result == CbValidateError::None) - { - PrintCompactBinary(Data); + CbValidateError Result = ValidateCompactBinary(Object.GetView(), CbValidateMode::All); + + CbValidateError FilteredResult = MakeFilteredResult(Result); + + if (FilteredResult == CbValidateError::None && FilteredResult != Result) + { + ZEN_WARN( + "Object in package message file '{}' does not appear to be an optimal compact binary format (validation error {:#x}: '{}')", + m_Filename, + uint32_t(Result), + ToString(Result)); + } + + if (FilteredResult != CbValidateError::None) + { + ZEN_ERROR("Object in package message file '{}' does not appear to be compact binary (validation error {:#x}: '{}')", + m_Filename, + uint32_t(FilteredResult), + ToString(FilteredResult)); + return 1; + } + else + { + PrintCbObject(Object, m_ShowCbObjectTypeInfo); + } } else { - ZEN_ERROR("Data in file '{}' does not appear to be compact binary (validation error {:#x})", m_Filename, uint32_t(Result)); + CbValidateError Result = ValidateCompactBinary(Data, CbValidateMode::All); - return 1; - } + CbValidateError FilteredResult = MakeFilteredResult(Result); + + if (FilteredResult == CbValidateError::None && FilteredResult != Result) + { + ZEN_WARN("Data in file '{}' does not appear to be an optimal compact binary format (validation error {:#x}: '{}')", + m_Filename, + uint32_t(Result), + ToString(Result)); + } + if (FilteredResult != CbValidateError::None) + { + ZEN_ERROR("Data in file '{}' does not appear to be compact binary (validation error {:#x}: '{}')", + m_Filename, + uint32_t(FilteredResult), + ToString(FilteredResult)); + return 1; + } + else + { + PrintCompactBinary(Data, m_ShowCbObjectTypeInfo); + } + } return 0; } @@ -151,6 +205,12 @@ PrintPackageCommand::PrintPackageCommand() { m_Options.add_options()("h,help", "Print help"); m_Options.add_option("", "s", "source", "Package payload file", cxxopts::value(m_Filename), "<file name>"); + m_Options.add_option("", + "", + "show-type-info", + "Add type info annotation to compact binary objects", + cxxopts::value(m_ShowCbObjectTypeInfo), + "<showtypes>"); m_Options.parse_positional({"source"}); } @@ -173,6 +233,7 @@ PrintPackageCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** ar if (m_Filename.empty()) throw std::runtime_error("No file specified"); + MakeSafeAbsolutePathÍnPlace(m_Filename); FileContents Fc = ReadFile(m_Filename); IoBuffer Data = Fc.Flatten(); CbPackage Package; @@ -182,7 +243,7 @@ PrintPackageCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** ar if (Ok) { ExtendableStringBuilder<1024> ObjStr; - CompactBinaryToJson(Package.GetObject(), ObjStr); + CompactBinaryToJson(Package.GetObject(), ObjStr, m_ShowCbObjectTypeInfo); ZEN_CONSOLE("{}", ObjStr); } else diff --git a/src/zen/cmds/print_cmd.h b/src/zen/cmds/print_cmd.h index 4d6a492b7..80729901e 100644 --- a/src/zen/cmds/print_cmd.h +++ b/src/zen/cmds/print_cmd.h @@ -19,8 +19,9 @@ public: virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; } private: - cxxopts::Options m_Options{"print", "Print compact binary object"}; - std::string m_Filename; + cxxopts::Options m_Options{"print", "Print compact binary object"}; + std::filesystem::path m_Filename; + bool m_ShowCbObjectTypeInfo = false; }; /** Print Compact Binary Package @@ -36,8 +37,9 @@ public: virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; } private: - cxxopts::Options m_Options{"printpkg", "Print compact binary package"}; - std::string m_Filename; + cxxopts::Options m_Options{"printpkg", "Print compact binary package"}; + std::filesystem::path m_Filename; + bool m_ShowCbObjectTypeInfo = false; }; } // namespace zen diff --git a/src/zencore/compactbinaryfile.cpp b/src/zencore/compactbinaryfile.cpp index f2121a0bd..1526c21d5 100644 --- a/src/zencore/compactbinaryfile.cpp +++ b/src/zencore/compactbinaryfile.cpp @@ -19,7 +19,7 @@ LoadCompactBinaryObject(const std::filesystem::path& FilePath) IoBuffer ObjectBuffer = ObjectFile.Flatten(); - if (CbValidateError Result = ValidateCompactBinary(ObjectBuffer, CbValidateMode::All); Result == CbValidateError::None) + if (CbValidateError Result = ValidateCompactBinary(ObjectBuffer, CbValidateMode::Default); Result == CbValidateError::None) { CbObject Object = LoadCompactBinaryObject(ObjectBuffer); const IoHash WorkerId = IoHash::HashBuffer(ObjectBuffer); diff --git a/src/zencore/compactbinaryjson.cpp b/src/zencore/compactbinaryjson.cpp index 68ed09549..02f22ba4d 100644 --- a/src/zencore/compactbinaryjson.cpp +++ b/src/zencore/compactbinaryjson.cpp @@ -22,7 +22,10 @@ namespace zen { class CbJsonWriter { public: - explicit CbJsonWriter(StringBuilderBase& InBuilder) : Builder(InBuilder) { NewLineAndIndent << LINE_TERMINATOR_ANSI; } + explicit CbJsonWriter(StringBuilderBase& InBuilder, bool AddTypeComment) : Builder(InBuilder), m_AddTypeComment(AddTypeComment) + { + NewLineAndIndent << LINE_TERMINATOR_ANSI; + } void BeginObject() { @@ -74,11 +77,32 @@ public: switch (CbValue Accessor = Field.GetValue(); Accessor.GetType()) { case CbFieldType::Null: + if (m_AddTypeComment) + { + Builder << "[Null] "; + } Builder << "null"sv; break; case CbFieldType::Object: + { + if (m_AddTypeComment) + { + Builder << "[Object] "; + } + BeginObject(); + for (CbFieldView It : Field) + { + WriteField(It); + } + EndObject(); + } + break; case CbFieldType::UniformObject: { + if (m_AddTypeComment) + { + Builder << "[UniformObject] "; + } BeginObject(); for (CbFieldView It : Field) { @@ -88,8 +112,25 @@ public: } break; case CbFieldType::Array: + { + if (m_AddTypeComment) + { + Builder << "[Array] "; + } + BeginArray(); + for (CbFieldView It : Field) + { + WriteField(It); + } + EndArray(); + } + break; case CbFieldType::UniformArray: { + if (m_AddTypeComment) + { + Builder << "[UniformArray] "; + } BeginArray(); for (CbFieldView It : Field) { @@ -99,19 +140,39 @@ public: } break; case CbFieldType::Binary: + if (m_AddTypeComment) + { + Builder << "[Binary] "; + } AppendBase64String(Accessor.AsBinary()); break; case CbFieldType::String: + if (m_AddTypeComment) + { + Builder << "[String] "; + } AppendQuotedString(Accessor.AsU8String()); break; case CbFieldType::IntegerPositive: + if (m_AddTypeComment) + { + Builder << "[IntegerPositive] "; + } Builder << Accessor.AsIntegerPositive(); break; case CbFieldType::IntegerNegative: + if (m_AddTypeComment) + { + Builder << "[IntegerNegative] "; + } Builder << Accessor.AsIntegerNegative(); break; case CbFieldType::Float32: { + if (m_AddTypeComment) + { + Builder << "[Float32] "; + } const float Value = Accessor.AsFloat32(); if (std::isfinite(Value)) { @@ -125,6 +186,10 @@ public: break; case CbFieldType::Float64: { + if (m_AddTypeComment) + { + Builder << "[Float64] "; + } const double Value = Accessor.AsFloat64(); if (std::isfinite(Value)) { @@ -137,14 +202,36 @@ public: } break; case CbFieldType::BoolFalse: + if (m_AddTypeComment) + { + Builder << "[BoolFalse] "; + } Builder << "false"sv; break; case CbFieldType::BoolTrue: + if (m_AddTypeComment) + { + Builder << "[BoolTrue] "; + } Builder << "true"sv; break; case CbFieldType::ObjectAttachment: + { + if (m_AddTypeComment) + { + Builder << "[ObjectAttachment] "; + } + Builder << '"'; + Accessor.AsAttachment().ToHexString(Builder); + Builder << '"'; + } + break; case CbFieldType::BinaryAttachment: { + if (m_AddTypeComment) + { + Builder << "[BinaryAttachment] "; + } Builder << '"'; Accessor.AsAttachment().ToHexString(Builder); Builder << '"'; @@ -152,6 +239,10 @@ public: break; case CbFieldType::Hash: { + if (m_AddTypeComment) + { + Builder << "[Hash] "; + } Builder << '"'; Accessor.AsHash().ToHexString(Builder); Builder << '"'; @@ -159,16 +250,28 @@ public: break; case CbFieldType::Uuid: { + if (m_AddTypeComment) + { + Builder << "[Uuid] "; + } Builder << '"'; Accessor.AsUuid().ToString(Builder); Builder << '"'; } break; case CbFieldType::DateTime: + if (m_AddTypeComment) + { + Builder << "[DateTime] "; + } Builder << '"' << DateTime(Accessor.AsDateTimeTicks()).ToIso8601() << '"'; break; case CbFieldType::TimeSpan: { + if (m_AddTypeComment) + { + Builder << "[TimeSpan] "; + } const TimeSpan Span(Accessor.AsTimeSpanTicks()); if (Span.GetDays() == 0) { @@ -181,12 +284,20 @@ public: break; } case CbFieldType::ObjectId: + if (m_AddTypeComment) + { + Builder << "[ObjectId] "; + } Builder << '"'; Accessor.AsObjectId().ToString(Builder); Builder << '"'; break; case CbFieldType::CustomById: { + if (m_AddTypeComment) + { + Builder << "[CustomById] "; + } CbCustomById Custom = Accessor.AsCustomById(); Builder << "{ \"Id\": "; Builder << Custom.Id; @@ -197,6 +308,10 @@ public: } case CbFieldType::CustomByName: { + if (m_AddTypeComment) + { + Builder << "[CustomByName] "; + } CbCustomByName Custom = Accessor.AsCustomByName(); Builder << "{ \"Name\": "; AppendQuotedString(Custom.Name); @@ -299,29 +414,30 @@ private: private: StringBuilderBase& Builder; ExtendableStringBuilder<32> NewLineAndIndent; + const bool m_AddTypeComment; bool NeedsComma{false}; bool NeedsNewLine{false}; }; void -CompactBinaryToJson(const CbObjectView& Object, StringBuilderBase& Builder) +CompactBinaryToJson(const CbObjectView& Object, StringBuilderBase& Builder, bool AddTypeComment) { - CbJsonWriter Writer(Builder); + CbJsonWriter Writer(Builder, AddTypeComment); Writer.WriteField(Object.AsFieldView()); } void CompactBinaryToJson(const CbArrayView& Array, StringBuilderBase& Builder) { - CbJsonWriter Writer(Builder); + CbJsonWriter Writer(Builder, /*AddTypeComment*/ false); Writer.WriteField(Array.AsFieldView()); } void -CompactBinaryToJson(MemoryView Data, StringBuilderBase& InBuilder) +CompactBinaryToJson(MemoryView Data, StringBuilderBase& InBuilder, bool AddTypeComment) { std::vector<CbFieldView> Fields = ReadCompactBinaryStream(Data); - CbJsonWriter Writer(InBuilder); + CbJsonWriter Writer(InBuilder, AddTypeComment); if (!Fields.empty()) { if (Fields.size() == 1) diff --git a/src/zencore/compactbinaryvalidation.cpp b/src/zencore/compactbinaryvalidation.cpp index 833649b88..d7292f405 100644 --- a/src/zencore/compactbinaryvalidation.cpp +++ b/src/zencore/compactbinaryvalidation.cpp @@ -705,10 +705,11 @@ ToString(const CbValidateError Error) ExtendableStringBuilder<128> Out; - auto AppendFlag = [&, IsFirst = false](std::string_view FlagString) { + auto AppendFlag = [&, IsFirst = true](std::string_view FlagString) mutable { if (!IsFirst) Out.Append('|'); Out.Append(FlagString); + IsFirst = false; }; #define _ENUM_CASE(V) \ @@ -737,7 +738,11 @@ ToString(const CbValidateError Error) #undef _ENUM_CASE - return "Error"; + if (Out.Size() == 0) + { + return "Error"; + } + return Out.ToString(); } /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/zencore/include/zencore/compactbinary.h b/src/zencore/include/zencore/compactbinary.h index 0fdb56d67..82ca055ab 100644 --- a/src/zencore/include/zencore/compactbinary.h +++ b/src/zencore/include/zencore/compactbinary.h @@ -996,7 +996,7 @@ private: /** * Serialize a compact binary object to JSON. */ -ZENCORE_API void CompactBinaryToJson(const CbObjectView& Object, StringBuilderBase& Builder); +ZENCORE_API void CompactBinaryToJson(const CbObjectView& Object, StringBuilderBase& Builder, bool AddTypeComment = false); /** * Serialize a compact binary object to YAML. */ @@ -1520,7 +1520,7 @@ end(CbFieldView&) /** * Serialize serialized compact binary blob to JSON. It must be 0 to n fields with including type for each field */ -ZENCORE_API void CompactBinaryToJson(MemoryView Data, StringBuilderBase& InBuilder); +ZENCORE_API void CompactBinaryToJson(MemoryView Data, StringBuilderBase& InBuilder, bool AddTypeComment = false); /** * Serialize serialized compact binary blob to YAML. It must be 0 to n fields with including type for each field diff --git a/src/zenserver/buildstore/httpbuildstore.cpp b/src/zenserver/buildstore/httpbuildstore.cpp index 3e1bc4203..2a3ce41b7 100644 --- a/src/zenserver/buildstore/httpbuildstore.cpp +++ b/src/zenserver/buildstore/httpbuildstore.cpp @@ -534,7 +534,6 @@ HttpBuildStoreService::HandleStatsRequest(HttpServerRequest& Request) { Cbo << "count" << StorageStats.BlobCount; Cbo << "bytes" << StorageStats.BlobBytes; - Cbo.EndObject(); // small } Cbo.EndObject(); // blobs diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp index 07fd6e908..61c279afa 100644 --- a/src/zenserver/projectstore/projectstore.cpp +++ b/src/zenserver/projectstore/projectstore.cpp @@ -1492,7 +1492,7 @@ ProjectStore::Oplog::ReadStateFile(const std::filesystem::path& BasePath, std::f if (ValidationError != CbValidateError::None) { - ZEN_ERROR("validation error {} hit for oplog config at '{}'", int(ValidationError), StateFilePath); + ZEN_ERROR("validation error {} hit for oplog config at '{}'", ToString(ValidationError), StateFilePath); return CbObject(); } @@ -3168,7 +3168,7 @@ ProjectStore::Project::Read() } else { - ZEN_ERROR("validation error {} hit for '{}'", int(ValidationError), ProjectStateFilePath); + ZEN_ERROR("validation error {} hit for '{}'", ToString(ValidationError), ProjectStateFilePath); } ReadAccessTimes(); @@ -3260,7 +3260,7 @@ ProjectStore::Project::ReadAccessTimes() } else { - ZEN_WARN("project '{}': validation error {} hit for '{}'", Identifier, int(ValidationError), ProjectAccessTimesFilePath); + ZEN_WARN("project '{}': validation error {} hit for '{}'", Identifier, ToString(ValidationError), ProjectAccessTimesFilePath); } } diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index 5307d58d9..ba1413819 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -424,10 +424,10 @@ ZenServer::InitializeState(const ZenServerOptions& ServerOptions) if (CbValidateError ValidationResult = ValidateCompactBinary(Manifest, CbValidateMode::All); ValidationResult != CbValidateError::None) { - ZEN_WARN("Manifest validation failed: {}, state will be wiped", uint32_t(ValidationResult)); + ZEN_WARN("Manifest validation failed: {}, state will be wiped", zen::ToString(ValidationResult)); WipeState = true; - WipeReason = fmt::format("Validation of manifest at '{}' failed: {}", ManifestPath, uint32_t(ValidationResult)); + WipeReason = fmt::format("Validation of manifest at '{}' failed: {}", ManifestPath, zen::ToString(ValidationResult)); } else { diff --git a/src/zenstore/cas.cpp b/src/zenstore/cas.cpp index 460f0e10d..5879a6742 100644 --- a/src/zenstore/cas.cpp +++ b/src/zenstore/cas.cpp @@ -195,7 +195,7 @@ CasImpl::OpenOrCreateManifest() } else { - ZEN_WARN("Store manifest validation failed: {:#x}, will generate new manifest to recover", uint32_t(ValidationResult)); + ZEN_WARN("Store manifest validation failed: {}, will generate new manifest to recover", ToString(ValidationResult)); } if (ManifestIsOk) |