From 8878156f30e375b93ebe99f02d94c581bcbbf43c Mon Sep 17 00:00:00 2001 From: zousar <2936246+zousar@users.noreply.github.com> Date: Fri, 21 Mar 2025 22:53:59 -0600 Subject: Add CookPackageArtifacts attachment to web ui --- src/zenserver/frontend/html/pages/entry.js | 69 ++++++++++++++++++++++++++++- src/zenserver/projectstore/projectstore.cpp | 43 ++++++++++++++++-- 2 files changed, 106 insertions(+), 6 deletions(-) (limited to 'src/zenserver') diff --git a/src/zenserver/frontend/html/pages/entry.js b/src/zenserver/frontend/html/pages/entry.js index 65a3ef39b..3891239f0 100644 --- a/src/zenserver/frontend/html/pages/entry.js +++ b/src/zenserver/frontend/html/pages/entry.js @@ -59,6 +59,63 @@ export class Page extends ZenPage } } + _find_iohash_field(container, name) + { + const found_field = container.find(name); + if (found_field != undefined) + { + var found_value = found_field.as_value(); + if (found_value instanceof Uint8Array) + { + var ret = ""; + for (var x of found_value) + ret += x.toString(16).padStart(2, "0"); + return ret; + } + } + return null; + } + + async _build_meta(section, entry) + { + var tree = {} + const cookart = this._find_iohash_field(entry, "CookPackageArtifacts"); + if (cookart != null) + { + tree["cook"] = { CookPackageArtifacts: cookart}; + } + + if (Object.keys(tree).length == 0) + return; + + const sub_section = section.add_section("meta"); + + for (const cat_name in tree) + { + const cat_section = sub_section.add_section(cat_name); + const table = cat_section.add_widget( + Table, + ["name", "actions"], Table.Flag_PackRight + ); + Object.entries(tree[cat_name]).forEach(([key, value]) => + { + const row = table.add_row(key); + + const project = this.get_param("project"); + const oplog = this.get_param("oplog"); + const link = row.get_cell(0).link( + "/" + ["prj", project, "oplog", oplog, value+".json"].join("/") + ); + + const do_nothing = () => void(0); + const action_tb = new Toolbar(row.get_cell(-1), true); + action_tb.left().add("copy-hash").on_click(async (v) => { + await navigator.clipboard.writeText(v); + }, value); + }); + } + } + async _build_page() { var entry = await this._entry; @@ -78,8 +135,16 @@ export class Page extends ZenPage delete tree["$id"]; - const sub_section = section.add_section("deps"); - this._build_deps(sub_section, tree); + if (Object.keys(tree).length != 0) + { + const sub_section = section.add_section("deps"); + this._build_deps(sub_section, tree); + } + } + + // meta + { + this._build_meta(section, entry); } // data diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp index 86791e29a..1132e458e 100644 --- a/src/zenserver/projectstore/projectstore.cpp +++ b/src/zenserver/projectstore/projectstore.cpp @@ -4812,11 +4812,46 @@ ProjectStore::GetChunk(const std::string_view ProjectId, } } - if (AcceptType == ZenContentType::kUnknownContentType || AcceptType == ZenContentType::kBinary) + if (AcceptType == ZenContentType::kUnknownContentType || AcceptType == ZenContentType::kBinary || AcceptType == ZenContentType::kJSON || + AcceptType == ZenContentType::kYAML || AcceptType == ZenContentType::kCbObject) { - CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(OutChunk)); - OutChunk = Compressed.Decompress().AsIoBuffer(); - OutChunk.SetContentType(ZenContentType::kBinary); + CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(OutChunk)); + IoBuffer DecompressedBuffer = Compressed.Decompress().AsIoBuffer(); + + if (AcceptType == ZenContentType::kJSON || AcceptType == ZenContentType::kYAML || AcceptType == ZenContentType::kCbObject) + { + CbValidateError CbErr = ValidateCompactBinary(DecompressedBuffer.GetView(), CbValidateMode::Default); + if (!!CbErr) + { + return {HttpResponseCode::NotAcceptable, fmt::format("chunk - '{}' WRONGTYPE", Cid)}; + } + + if (AcceptType == HttpContentType::kJSON || AcceptType == HttpContentType::kYAML) + { + CbObject ContainerObject = LoadCompactBinaryObject(DecompressedBuffer); + ExtendableStringBuilder<1024> Sb; + if (AcceptType == HttpContentType::kJSON) + { + ContainerObject.ToJson(Sb); + } + else if (AcceptType == HttpContentType::kYAML) + { + ContainerObject.ToYaml(Sb); + } + IoBuffer SerializedBuffer(IoBuffer::Clone, Sb.Data(), Sb.Size()); + OutChunk = SerializedBuffer; + } + else + { + OutChunk = DecompressedBuffer; + } + OutChunk.SetContentType(AcceptType); + } + else + { + OutChunk = DecompressedBuffer; + OutChunk.SetContentType(ZenContentType::kBinary); + } } else { -- cgit v1.2.3 From cc4d4751201bbf83531ea9919f242bfac3add8d3 Mon Sep 17 00:00:00 2001 From: zousar <2936246+zousar@users.noreply.github.com> Date: Tue, 25 Mar 2025 15:14:47 -0600 Subject: Moved AcceptType Handling From ProjectStore To HttpProjectStore --- src/zenserver/projectstore/httpprojectstore.cpp | 34 +++++++++++++- src/zenserver/projectstore/projectstore.cpp | 61 ++----------------------- src/zenserver/projectstore/projectstore.h | 1 - 3 files changed, 35 insertions(+), 61 deletions(-) (limited to 'src/zenserver') diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp index 47748dd90..f9a13220a 100644 --- a/src/zenserver/projectstore/httpprojectstore.cpp +++ b/src/zenserver/projectstore/httpprojectstore.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -885,10 +886,39 @@ HttpProjectService::HandleChunkByCidRequest(HttpRouterRequest& Req) case HttpVerb::kGet: { IoBuffer Value; - std::pair Result = - m_ProjectStore->GetChunk(ProjectId, OplogId, Cid, AcceptType, Value, nullptr); + std::pair Result = m_ProjectStore->GetChunk(ProjectId, OplogId, Cid, Value, nullptr); if (Result.first == HttpResponseCode::OK) { + if (AcceptType == ZenContentType::kUnknownContentType || AcceptType == ZenContentType::kBinary || + AcceptType == ZenContentType::kJSON || AcceptType == ZenContentType::kYAML || + AcceptType == ZenContentType::kCbObject) + { + CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(Value)); + IoBuffer DecompressedBuffer = Compressed.Decompress().AsIoBuffer(); + + if (AcceptType == ZenContentType::kJSON || AcceptType == ZenContentType::kYAML || + AcceptType == ZenContentType::kCbObject) + { + CbValidateError CbErr = ValidateCompactBinary(DecompressedBuffer.GetView(), CbValidateMode::Default); + if (!!CbErr) + { + m_ProjectStats.BadRequestCount++; + ZEN_DEBUG("chunk - '{}/{}/{}' WRONGTYPE", ProjectId, OplogId, Cid); + return HttpReq.WriteResponse(HttpResponseCode::NotAcceptable, + HttpContentType::kText, + fmt::format("chunk - '{}' WRONGTYPE", Cid)); + } + + m_ProjectStats.ChunkHitCount++; + CbObject ContainerObject = LoadCompactBinaryObject(DecompressedBuffer); + return HttpReq.WriteResponse(HttpResponseCode::OK, ContainerObject); + } + else + { + Value = DecompressedBuffer; + Value.SetContentType(ZenContentType::kBinary); + } + } m_ProjectStats.ChunkHitCount++; return HttpReq.WriteResponse(HttpResponseCode::OK, Value.GetContentType(), Value); } diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp index 1132e458e..ea3f2aad9 100644 --- a/src/zenserver/projectstore/projectstore.cpp +++ b/src/zenserver/projectstore/projectstore.cpp @@ -4770,7 +4770,6 @@ std::pair ProjectStore::GetChunk(const std::string_view ProjectId, const std::string_view OplogId, const std::string_view Cid, - ZenContentType AcceptType, IoBuffer& OutChunk, uint64_t* OptionalInOutModificationTag) { @@ -4812,51 +4811,7 @@ ProjectStore::GetChunk(const std::string_view ProjectId, } } - if (AcceptType == ZenContentType::kUnknownContentType || AcceptType == ZenContentType::kBinary || AcceptType == ZenContentType::kJSON || - AcceptType == ZenContentType::kYAML || AcceptType == ZenContentType::kCbObject) - { - CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(OutChunk)); - IoBuffer DecompressedBuffer = Compressed.Decompress().AsIoBuffer(); - - if (AcceptType == ZenContentType::kJSON || AcceptType == ZenContentType::kYAML || AcceptType == ZenContentType::kCbObject) - { - CbValidateError CbErr = ValidateCompactBinary(DecompressedBuffer.GetView(), CbValidateMode::Default); - if (!!CbErr) - { - return {HttpResponseCode::NotAcceptable, fmt::format("chunk - '{}' WRONGTYPE", Cid)}; - } - - if (AcceptType == HttpContentType::kJSON || AcceptType == HttpContentType::kYAML) - { - CbObject ContainerObject = LoadCompactBinaryObject(DecompressedBuffer); - ExtendableStringBuilder<1024> Sb; - if (AcceptType == HttpContentType::kJSON) - { - ContainerObject.ToJson(Sb); - } - else if (AcceptType == HttpContentType::kYAML) - { - ContainerObject.ToYaml(Sb); - } - IoBuffer SerializedBuffer(IoBuffer::Clone, Sb.Data(), Sb.Size()); - OutChunk = SerializedBuffer; - } - else - { - OutChunk = DecompressedBuffer; - } - OutChunk.SetContentType(AcceptType); - } - else - { - OutChunk = DecompressedBuffer; - OutChunk.SetContentType(ZenContentType::kBinary); - } - } - else - { - OutChunk.SetContentType(ZenContentType::kCompressedBinary); - } + OutChunk.SetContentType(ZenContentType::kCompressedBinary); return {HttpResponseCode::OK, {}}; } @@ -8551,12 +8506,7 @@ TEST_CASE("project.store.partial.read") uint64_t ModificationTag = 0; IoBuffer Chunk; CHECK(ProjectStore - .GetChunk("proj1"sv, - "oplog1"sv, - Attachments[OpIds[1]][0].second.DecodeRawHash().ToHexString(), - HttpContentType::kCompressedBinary, - Chunk, - &ModificationTag) + .GetChunk("proj1"sv, "oplog1"sv, Attachments[OpIds[1]][0].second.DecodeRawHash().ToHexString(), Chunk, &ModificationTag) .first == HttpResponseCode::OK); IoHash RawHash; uint64_t RawSize; @@ -8565,12 +8515,7 @@ TEST_CASE("project.store.partial.read") CHECK(ModificationTag != 0); CHECK(ProjectStore - .GetChunk("proj1"sv, - "oplog1"sv, - Attachments[OpIds[1]][0].second.DecodeRawHash().ToHexString(), - HttpContentType::kCompressedBinary, - Chunk, - &ModificationTag) + .GetChunk("proj1"sv, "oplog1"sv, Attachments[OpIds[1]][0].second.DecodeRawHash().ToHexString(), Chunk, &ModificationTag) .first == HttpResponseCode::NotModified); } diff --git a/src/zenserver/projectstore/projectstore.h b/src/zenserver/projectstore/projectstore.h index 8f2d3ce0d..368da5ea4 100644 --- a/src/zenserver/projectstore/projectstore.h +++ b/src/zenserver/projectstore/projectstore.h @@ -449,7 +449,6 @@ public: std::pair GetChunk(const std::string_view ProjectId, const std::string_view OplogId, const std::string_view Cid, - ZenContentType AcceptType, IoBuffer& OutChunk, uint64_t* OptionalInOutModificationTag); -- cgit v1.2.3 From 44c61d6d310a605074df0baa2af27410c589daae Mon Sep 17 00:00:00 2001 From: zousar <2936246+zousar@users.noreply.github.com> Date: Tue, 25 Mar 2025 15:18:13 -0600 Subject: Removed do_nothing from entry.js --- src/zenserver/frontend/html/pages/entry.js | 2 -- 1 file changed, 2 deletions(-) (limited to 'src/zenserver') diff --git a/src/zenserver/frontend/html/pages/entry.js b/src/zenserver/frontend/html/pages/entry.js index 3891239f0..f127cb0a3 100644 --- a/src/zenserver/frontend/html/pages/entry.js +++ b/src/zenserver/frontend/html/pages/entry.js @@ -107,7 +107,6 @@ export class Page extends ZenPage "/" + ["prj", project, "oplog", oplog, value+".json"].join("/") ); - const do_nothing = () => void(0); const action_tb = new Toolbar(row.get_cell(-1), true); action_tb.left().add("copy-hash").on_click(async (v) => { await navigator.clipboard.writeText(v); @@ -193,7 +192,6 @@ export class Page extends ZenPage ); link.first_child().attr("download", `${io_hash}_${base_name}`); - const do_nothing = () => void(0); const action_tb = new Toolbar(row.get_cell(-1), true); action_tb.left().add("copy-hash").on_click(async (v) => { await navigator.clipboard.writeText(v); -- cgit v1.2.3 From df9bbea4fb1dad9e8c853f14600259a4af12d334 Mon Sep 17 00:00:00 2001 From: zousar <2936246+zousar@users.noreply.github.com> Date: Wed, 26 Mar 2025 16:57:48 -0600 Subject: Descriptive type conversion messages Handling decompression or validation errors with more descriptive messages --- src/zenserver/projectstore/httpprojectstore.cpp | 54 ++++++++++++++++++------- 1 file changed, 39 insertions(+), 15 deletions(-) (limited to 'src/zenserver') diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp index f9a13220a..6313fd69e 100644 --- a/src/zenserver/projectstore/httpprojectstore.cpp +++ b/src/zenserver/projectstore/httpprojectstore.cpp @@ -896,27 +896,51 @@ HttpProjectService::HandleChunkByCidRequest(HttpRouterRequest& Req) CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(Value)); IoBuffer DecompressedBuffer = Compressed.Decompress().AsIoBuffer(); - if (AcceptType == ZenContentType::kJSON || AcceptType == ZenContentType::kYAML || - AcceptType == ZenContentType::kCbObject) + if (DecompressedBuffer) { - CbValidateError CbErr = ValidateCompactBinary(DecompressedBuffer.GetView(), CbValidateMode::Default); - if (!!CbErr) + if (AcceptType == ZenContentType::kJSON || AcceptType == ZenContentType::kYAML || + AcceptType == ZenContentType::kCbObject) { - m_ProjectStats.BadRequestCount++; - ZEN_DEBUG("chunk - '{}/{}/{}' WRONGTYPE", ProjectId, OplogId, Cid); - return HttpReq.WriteResponse(HttpResponseCode::NotAcceptable, - HttpContentType::kText, - fmt::format("chunk - '{}' WRONGTYPE", Cid)); - } + CbValidateError CbErr = ValidateCompactBinary(DecompressedBuffer.GetView(), CbValidateMode::Default); + if (!!CbErr) + { + m_ProjectStats.BadRequestCount++; + ZEN_DEBUG( + "chunk - '{}/{}/{}' WRONGTYPE. Reason: `Requested {} format, but could not convert to object`", + ProjectId, + OplogId, + Cid, + ToString(AcceptType)); + return HttpReq.WriteResponse( + HttpResponseCode::NotAcceptable, + HttpContentType::kText, + fmt::format("Content format not supported, requested {} format, but could not convert to object", + ToString(AcceptType))); + } - m_ProjectStats.ChunkHitCount++; - CbObject ContainerObject = LoadCompactBinaryObject(DecompressedBuffer); - return HttpReq.WriteResponse(HttpResponseCode::OK, ContainerObject); + m_ProjectStats.ChunkHitCount++; + CbObject ContainerObject = LoadCompactBinaryObject(DecompressedBuffer); + return HttpReq.WriteResponse(HttpResponseCode::OK, ContainerObject); + } + else + { + Value = DecompressedBuffer; + Value.SetContentType(ZenContentType::kBinary); + } } else { - Value = DecompressedBuffer; - Value.SetContentType(ZenContentType::kBinary); + m_ProjectStats.BadRequestCount++; + ZEN_DEBUG("chunk - '{}/{}/{}' WRONGTYPE. Reason: `Requested {} format, but could not decompress stored data`", + ProjectId, + OplogId, + Cid, + ToString(AcceptType)); + return HttpReq.WriteResponse( + HttpResponseCode::NotAcceptable, + HttpContentType::kText, + fmt::format("Content format not supported, requested {} format, but could not decompress stored data", + ToString(AcceptType))); } } m_ProjectStats.ChunkHitCount++; -- cgit v1.2.3