From 0fee64c8c3f7fc350e2030c56cb6938369606013 Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Mon, 20 Sep 2021 10:28:28 +0200 Subject: Added support for skipping package attachments. --- zenserver-test/zenserver-test.cpp | 53 +++++++++++++++++++++++++++++++++++++ zenserver/cache/structuredcache.cpp | 46 +++++++++++++++++--------------- 2 files changed, 78 insertions(+), 21 deletions(-) diff --git a/zenserver-test/zenserver-test.cpp b/zenserver-test/zenserver-test.cpp index a556d896e..3ca5747f5 100644 --- a/zenserver-test/zenserver-test.cpp +++ b/zenserver-test/zenserver-test.cpp @@ -1682,6 +1682,59 @@ TEST_CASE("zcache.policy") CHECK(Result.status_code == 200); } } + + SUBCASE("skip - 'attachments' does not return attachments") + { + ZenConfig LocalCfg = ZenConfig::New(); + ZenServerInstance LocalInst(TestEnv); + const auto Bucket = "texture"sv; + + LocalCfg.Spawn(LocalInst); + + zen::IoHash Key; + zen::CbPackage Package = GeneratePackage(Key); + auto Buf = ToBuffer(Package); + + // Store package locally + { + CHECK(Package.GetAttachments().size() != 0); + cpr::Response Result = cpr::Put(cpr::Url{"{}/{}/{}"_format(LocalCfg.BaseUri, Bucket, Key)}, + cpr::Body{(const char*)Buf.GetData(), Buf.GetSize()}, + cpr::Header{{"Content-Type", "application/x-ue-cbpkg"}}); + CHECK(Result.status_code == 201); + } + + { + cpr::Response Result = cpr::Get(cpr::Url{"{}/{}/{}?skip=attachments"_format(LocalCfg.BaseUri, Bucket, Key)}, + cpr::Header{{"Accept", "application/x-ue-cbpkg"}}); + CHECK(Result.status_code == 200); + + zen::IoBuffer Body(zen::IoBuffer::Wrap, Result.text.data(), Result.text.size()); + zen::CbPackage Package; + const bool Ok = Package.TryLoad(Body); + + CbObject CacheRecord = Package.GetObject(); + std::vector AttachmentKeys; + + CacheRecord.IterateAttachments( + [&AttachmentKeys](CbFieldView AttachmentKey) { AttachmentKeys.push_back(AttachmentKey.AsHash()); }); + + CHECK(AttachmentKeys.size() != 0); + CHECK(Package.GetAttachments().size() == 0); + } + + { + cpr::Response Result = + cpr::Get(cpr::Url{"{}/{}/{}"_format(LocalCfg.BaseUri, Bucket, Key)}, cpr::Header{{"Accept", "application/x-ue-cbpkg"}}); + CHECK(Result.status_code == 200); + + zen::IoBuffer Body(zen::IoBuffer::Wrap, Result.text.data(), Result.text.size()); + zen::CbPackage Package; + const bool Ok = Package.TryLoad(Body); + + CHECK(Package.GetAttachments().size() != 0); + } + } } struct RemoteExecutionRequest diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index c66b1f98d..469e39bd4 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -376,32 +376,36 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req return Request.WriteResponse(zen::HttpResponseCode::NotFound, HttpContentType::kText, "Invalid cache record"sv); } - uint32_t AttachmentCount = 0; - uint32_t FoundCount = 0; - uint64_t AttachmentBytes = 0ull; + const bool SkipAttachments = zen::CachePolicy::SkipAttachments == (Policy & zen::CachePolicy::SkipAttachments); + uint32_t AttachmentCount = 0; + uint32_t FoundCount = 0; + uint64_t AttachmentBytes = 0ull; CbPackage Package; - CacheRecord.IterateAttachments( - [this, &Ref, &Package, &AttachmentCount, &FoundCount, &AttachmentBytes](CbFieldView AttachmentHash) { - if (IoBuffer Chunk = m_CidStore.FindChunkByCid(AttachmentHash.AsHash())) - { - Package.AddAttachment(CbAttachment(CompressedBuffer::FromCompressed(SharedBuffer(Chunk)))); - AttachmentBytes += Chunk.Size(); - FoundCount++; - } - AttachmentCount++; - }); - - if (FoundCount != AttachmentCount) + if (!SkipAttachments) { - ZEN_WARN("GET - cache record '{}/{}' FAILED, found '{}' of '{}' attachments", - Ref.BucketSegment, - Ref.HashKey, - FoundCount, - AttachmentCount); + CacheRecord.IterateAttachments( + [this, &Ref, &Package, &AttachmentCount, &FoundCount, &AttachmentBytes](CbFieldView AttachmentHash) { + if (IoBuffer Chunk = m_CidStore.FindChunkByCid(AttachmentHash.AsHash())) + { + Package.AddAttachment(CbAttachment(CompressedBuffer::FromCompressed(SharedBuffer(Chunk)))); + AttachmentBytes += Chunk.Size(); + FoundCount++; + } + AttachmentCount++; + }); + + if (FoundCount != AttachmentCount) + { + ZEN_WARN("GET - cache record '{}/{}' FAILED, found '{}' of '{}' attachments", + Ref.BucketSegment, + Ref.HashKey, + FoundCount, + AttachmentCount); - return Request.WriteResponse(zen::HttpResponseCode::NotFound, HttpContentType::kText, "Missing attachments"sv); + return Request.WriteResponse(zen::HttpResponseCode::NotFound, HttpContentType::kText, "Missing attachments"sv); + } } Package.SetObject(LoadCompactBinaryObject(Value.Value)); -- cgit v1.2.3 From f4b7639b0f26d16c9a482b2db5eb598e851d7bb3 Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Mon, 20 Sep 2021 10:58:23 +0200 Subject: Respect skip attachments when retrieved from upstream cache. --- zenserver-test/zenserver-test.cpp | 56 +++++++++++++++++++++++++++++++++++++ zenserver/cache/structuredcache.cpp | 12 ++++++++ 2 files changed, 68 insertions(+) diff --git a/zenserver-test/zenserver-test.cpp b/zenserver-test/zenserver-test.cpp index dca583399..7cd2bb1d1 100644 --- a/zenserver-test/zenserver-test.cpp +++ b/zenserver-test/zenserver-test.cpp @@ -1737,6 +1737,62 @@ TEST_CASE("zcache.policy") CHECK(Package.GetAttachments().size() != 0); } } + + SUBCASE("skip - 'attachments' does not return attachments when retrieved from upstream") + { + ZenConfig UpstreamCfg = ZenConfig::New(13338); + ZenServerInstance UpstreamInst(TestEnv); + ZenConfig LocalCfg = ZenConfig::NewWithUpstream(13338); + ZenServerInstance LocalInst(TestEnv); + const auto Bucket = "texture"sv; + + UpstreamCfg.Spawn(UpstreamInst); + LocalCfg.Spawn(LocalInst); + + zen::IoHash Key; + zen::CbPackage Package = GeneratePackage(Key); + auto Buf = ToBuffer(Package); + + // Store package upstream + { + CHECK(Package.GetAttachments().size() != 0); + cpr::Response Result = cpr::Put(cpr::Url{"{}/{}/{}"_format(UpstreamCfg.BaseUri, Bucket, Key)}, + cpr::Body{(const char*)Buf.GetData(), Buf.GetSize()}, + cpr::Header{{"Content-Type", "application/x-ue-cbpkg"}}); + CHECK(Result.status_code == 201); + } + + { + cpr::Response Result = cpr::Get(cpr::Url{"{}/{}/{}?skip=attachments"_format(LocalCfg.BaseUri, Bucket, Key)}, + cpr::Header{{"Accept", "application/x-ue-cbpkg"}}); + CHECK(Result.status_code == 200); + + zen::IoBuffer Body(zen::IoBuffer::Wrap, Result.text.data(), Result.text.size()); + zen::CbPackage Package; + const bool Ok = Package.TryLoad(Body); + + CbObject CacheRecord = Package.GetObject(); + std::vector AttachmentKeys; + + CacheRecord.IterateAttachments( + [&AttachmentKeys](CbFieldView AttachmentKey) { AttachmentKeys.push_back(AttachmentKey.AsHash()); }); + + CHECK(AttachmentKeys.size() != 0); + CHECK(Package.GetAttachments().size() == 0); + } + + { + cpr::Response Result = + cpr::Get(cpr::Url{"{}/{}/{}"_format(LocalCfg.BaseUri, Bucket, Key)}, cpr::Header{{"Accept", "application/x-ue-cbpkg"}}); + CHECK(Result.status_code == 200); + + zen::IoBuffer Body(zen::IoBuffer::Wrap, Result.text.data(), Result.text.size()); + zen::CbPackage Package; + const bool Ok = Package.TryLoad(Body); + + CHECK(Package.GetAttachments().size() != 0); + } + } } struct RemoteExecutionRequest diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 469e39bd4..2ab433b90 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -333,6 +333,18 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req if (FoundCount == AttachmentCount) { m_CacheStore.Put(Ref.BucketSegment, Ref.HashKey, {.Value = CacheRecord.GetBuffer().AsIoBuffer()}); + + if (zen::CachePolicy::SkipAttachments == (Policy & zen::CachePolicy::SkipAttachments)) + { + CbPackage PackageWithoutAttachments; + PackageWithoutAttachments.SetObject(CacheRecord); + + MemoryOutStream MemStream; + BinaryWriter Writer(MemStream); + PackageWithoutAttachments.Save(Writer); + + Value.Value = IoBuffer(IoBuffer::Clone, MemStream.Data(), MemStream.Size()); + } } else { -- cgit v1.2.3