From c7c4b3b05a0e9826a35758f6212140fd29e5b474 Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Thu, 16 Sep 2021 21:19:02 +0200 Subject: zcache - minor cleanup. --- zenserver/cache/structuredcache.cpp | 52 +++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 28 deletions(-) (limited to 'zenserver/cache/structuredcache.cpp') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index cf7deaa93..da2e8850e 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -437,26 +437,22 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req CbObject CacheRecord = Package.GetObject(); - int32_t AttachmentCount = 0; - int32_t NewAttachmentCount = 0; - uint64_t TotalAttachmentBytes = 0; - uint64_t TotalNewBytes = 0; - bool AttachmentsOk = true; - + struct AttachmentInsertResult + { + int32_t Count = 0; + int32_t NewCount = 0; + uint64_t Bytes = 0; + uint64_t NewBytes = 0; + bool Ok = false; + }; + + AttachmentInsertResult AttachmentResult{.Ok = true}; std::span Attachments = Package.GetAttachments(); + std::vector PayloadIds; - std::vector PayloadIds; PayloadIds.reserve(Attachments.size()); - CacheRecord.IterateAttachments([this, - &Ref, - &Package, - &AttachmentsOk, - &AttachmentCount, - &TotalAttachmentBytes, - &TotalNewBytes, - &NewAttachmentCount, - &PayloadIds](CbFieldView AttachmentHash) { + CacheRecord.IterateAttachments([this, &Ref, &Package, &AttachmentResult, &PayloadIds](CbFieldView AttachmentHash) { if (const CbAttachment* Attachment = Package.FindAttachment(AttachmentHash.AsHash())) { if (Attachment->IsCompressedBinary()) @@ -469,12 +465,12 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req if (InsertResult.New) { - TotalNewBytes += ChunkSize; - ++NewAttachmentCount; + AttachmentResult.NewBytes += ChunkSize; + AttachmentResult.NewCount++; } - TotalAttachmentBytes += ChunkSize; - AttachmentCount++; + AttachmentResult.Bytes += ChunkSize; + AttachmentResult.Count++; } else { @@ -482,7 +478,7 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req Ref.BucketSegment, Ref.HashKey, AttachmentHash.AsHash()); - AttachmentsOk = false; + AttachmentResult.Ok = false; } } else @@ -491,17 +487,17 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req Ref.BucketSegment, Ref.HashKey, AttachmentHash.AsHash()); - AttachmentsOk = false; + AttachmentResult.Ok = false; } }); - if (!AttachmentsOk) + if (!AttachmentResult.Ok) { return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid attachments"); } IoBuffer CacheRecordChunk = CacheRecord.GetBuffer().AsIoBuffer(); - const uint64_t TotalPackageBytes = TotalAttachmentBytes + CacheRecordChunk.Size(); + const uint64_t TotalPackageBytes = AttachmentResult.Bytes + CacheRecordChunk.Size(); ZenCacheValue CacheValue{.Value = CacheRecordChunk}; m_CacheStore.Put(Ref.BucketSegment, Ref.HashKey, CacheValue); @@ -517,10 +513,10 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req Ref.BucketSegment, Ref.HashKey, zen::NiceBytes(TotalPackageBytes), - NewAttachmentCount, - AttachmentCount, - zen::NiceBytes(TotalNewBytes), - zen::NiceBytes(TotalAttachmentBytes)); + AttachmentResult.NewCount, + AttachmentResult.Count, + zen::NiceBytes(AttachmentResult.NewBytes), + zen::NiceBytes(AttachmentResult.Bytes)); return Request.WriteResponse(zen::HttpResponseCode::Created); } -- cgit v1.2.3 From 0d787afc539113fb4570c958f14a66f996b0061c Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Fri, 17 Sep 2021 14:46:28 +0200 Subject: Initial support for cache policies. --- zenserver/cache/structuredcache.cpp | 151 ++++++++++++++++++++++++++++++++---- 1 file changed, 136 insertions(+), 15 deletions(-) (limited to 'zenserver/cache/structuredcache.cpp') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index da2e8850e..c66b1f98d 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -25,12 +25,125 @@ #include #include +#include + namespace zen { using namespace std::literals; ////////////////////////////////////////////////////////////////////////// +namespace detail { namespace cacheopt { + constexpr std::string_view Local = "local"sv; + constexpr std::string_view Remote = "remote"sv; + constexpr std::string_view Data = "data"sv; + constexpr std::string_view Meta = "meta"sv; + constexpr std::string_view Value = "value"sv; + constexpr std::string_view Attachments = "attachments"sv; +}} // namespace detail::cacheopt + +////////////////////////////////////////////////////////////////////////// + +enum class CachePolicy : uint8_t +{ + None = 0, + QueryLocal = 1 << 0, + QueryRemote = 1 << 1, + Query = QueryLocal | QueryRemote, + StoreLocal = 1 << 2, + StoreRemote = 1 << 3, + Store = StoreLocal | StoreRemote, + SkipMeta = 1 << 4, + SkipValue = 1 << 5, + SkipAttachments = 1 << 6, + SkipData = SkipMeta | SkipValue | SkipAttachments, + SkipLocalCopy = 1 << 7, + Local = QueryLocal | StoreLocal, + Remote = QueryRemote | StoreRemote, + Default = Query | Store, + Disable = None, +}; + +gsl_DEFINE_ENUM_BITMASK_OPERATORS(CachePolicy); + +CachePolicy +ParseCachePolicy(const zen::HttpServerRequest::QueryParams& QueryParams) +{ + CachePolicy QueryPolicy = CachePolicy::Query; + + { + std::string_view Opts = QueryParams.GetValue("query"sv); + if (!Opts.empty()) + { + QueryPolicy = CachePolicy::None; + zen::ForEachStrTok(Opts, ',', [&QueryPolicy](const std::string_view& Opt) { + if (Opt == detail::cacheopt::Local) + { + QueryPolicy |= CachePolicy::QueryLocal; + } + if (Opt == detail::cacheopt::Remote) + { + QueryPolicy |= CachePolicy::QueryRemote; + } + return true; + }); + } + } + + CachePolicy StorePolicy = CachePolicy::Store; + + { + std::string_view Opts = QueryParams.GetValue("store"sv); + if (!Opts.empty()) + { + StorePolicy = CachePolicy::None; + zen::ForEachStrTok(Opts, ',', [&StorePolicy](const std::string_view& Opt) { + if (Opt == detail::cacheopt::Local) + { + StorePolicy |= CachePolicy::StoreLocal; + } + if (Opt == detail::cacheopt::Remote) + { + StorePolicy |= CachePolicy::StoreRemote; + } + return true; + }); + } + } + + CachePolicy SkipPolicy = CachePolicy::None; + + { + std::string_view Opts = QueryParams.GetValue("skip"sv); + if (!Opts.empty()) + { + zen::ForEachStrTok(Opts, ',', [&SkipPolicy](const std::string_view& Opt) { + if (Opt == detail::cacheopt::Meta) + { + SkipPolicy |= CachePolicy::SkipMeta; + } + if (Opt == detail::cacheopt::Value) + { + SkipPolicy |= CachePolicy::SkipValue; + } + if (Opt == detail::cacheopt::Attachments) + { + SkipPolicy |= CachePolicy::SkipAttachments; + } + if (Opt == detail::cacheopt::Data) + { + SkipPolicy |= CachePolicy::SkipData; + } + return true; + }); + } + } + + return QueryPolicy | StorePolicy | SkipPolicy; +} + +////////////////////////////////////////////////////////////////////////// + HttpStructuredCacheService::HttpStructuredCacheService(::ZenCacheStore& InCacheStore, zen::CasStore& InStore, zen::CidStore& InCidStore, @@ -78,13 +191,16 @@ HttpStructuredCacheService::HandleRequest(zen::HttpServerRequest& Request) return Request.WriteResponse(zen::HttpResponseCode::BadRequest); // invalid URL } + const auto QueryParams = Request.GetQueryParams(); + CachePolicy Policy = ParseCachePolicy(QueryParams); + if (Ref.PayloadId == IoHash::Zero) { - return HandleCacheRecordRequest(Request, Ref); + return HandleCacheRecordRequest(Request, Ref, Policy); } else { - return HandleCachePayloadRequest(Request, Ref); + return HandleCachePayloadRequest(Request, Ref, Policy); } return; @@ -121,7 +237,7 @@ HttpStructuredCacheService::HandleCacheBucketRequest(zen::HttpServerRequest& Req } void -HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Request, CacheRef& Ref) +HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Request, CacheRef& Ref, CachePolicy Policy) { switch (auto Verb = Request.RequestVerb()) { @@ -136,7 +252,10 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req bool Success = m_CacheStore.Get(Ref.BucketSegment, Ref.HashKey, /* out */ Value); bool InUpstreamCache = false; - if (!Success && m_UpstreamCache) + const bool QueryUpstream = + !Success && m_UpstreamCache && (zen::CachePolicy::QueryRemote == (Policy & zen::CachePolicy::QueryRemote)); + + if (QueryUpstream) { const ZenContentType CacheRecordType = Ref.BucketSegment == "legacy"sv ? ZenContentType::kBinary : AcceptType == ZenContentType::kCbPackage ? ZenContentType::kCbPackage @@ -326,13 +445,15 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req const HttpContentType ContentType = Request.RequestContentType(); + const bool StoreUpstream = m_UpstreamCache && (zen::CachePolicy::StoreRemote == (Policy & zen::CachePolicy::StoreRemote)); + if (ContentType == HttpContentType::kBinary || ContentType == HttpContentType::kUnknownContentType) { // TODO: create a cache record and put value in CAS? m_CacheStore.Put(Ref.BucketSegment, Ref.HashKey, {.Value = Body}); ZEN_DEBUG("PUT - binary '{}/{}' {}", Ref.BucketSegment, Ref.HashKey, NiceBytes(Body.Size())); - if (m_UpstreamCache) + if (StoreUpstream) { auto Result = m_UpstreamCache->EnqueueUpstream( {.Type = ZenContentType::kBinary, .CacheKey = {Ref.BucketSegment, Ref.HashKey}}); @@ -397,15 +518,12 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req MissingRefs.size(), References.size()); - if (MissingRefs.empty()) + if (MissingRefs.empty() && StoreUpstream) { - // Only enqueue valid cache records, i.e. all referenced payloads exists - if (m_UpstreamCache) - { - auto Result = m_UpstreamCache->EnqueueUpstream({.Type = ZenContentType::kCbObject, - .CacheKey = {Ref.BucketSegment, Ref.HashKey}, - .PayloadIds = std::move(References)}); - } + ZEN_ASSERT(m_UpstreamCache); + auto Result = m_UpstreamCache->EnqueueUpstream({.Type = ZenContentType::kCbObject, + .CacheKey = {Ref.BucketSegment, Ref.HashKey}, + .PayloadIds = std::move(References)}); return Request.WriteResponse(zen::HttpResponseCode::Created); } @@ -502,8 +620,9 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req ZenCacheValue CacheValue{.Value = CacheRecordChunk}; m_CacheStore.Put(Ref.BucketSegment, Ref.HashKey, CacheValue); - if (m_UpstreamCache) + if (StoreUpstream) { + ZEN_ASSERT(m_UpstreamCache); auto Result = m_UpstreamCache->EnqueueUpstream({.Type = ZenContentType::kCbPackage, .CacheKey = {Ref.BucketSegment, Ref.HashKey}, .PayloadIds = std::move(PayloadIds)}); @@ -536,7 +655,7 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req } void -HttpStructuredCacheService::HandleCachePayloadRequest(zen::HttpServerRequest& Request, CacheRef& Ref) +HttpStructuredCacheService::HandleCachePayloadRequest(zen::HttpServerRequest& Request, CacheRef& Ref, CachePolicy Policy) { // Note: the URL references the uncompressed payload hash - so this maintains the mapping // from uncompressed CAS identity (aka CID/Content ID) to the stored payload hash @@ -544,6 +663,8 @@ HttpStructuredCacheService::HandleCachePayloadRequest(zen::HttpServerRequest& Re // this is a PITA but a consequence of the fact that the client side code is not able to // address data by compressed hash + ZEN_UNUSED(Policy); + switch (auto Verb = Request.RequestVerb()) { using enum zen::HttpVerb; -- cgit v1.2.3 From 0fee64c8c3f7fc350e2030c56cb6938369606013 Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Mon, 20 Sep 2021 10:28:28 +0200 Subject: Added support for skipping package attachments. --- zenserver/cache/structuredcache.cpp | 46 ++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 21 deletions(-) (limited to 'zenserver/cache/structuredcache.cpp') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index c66b1f98d..469e39bd4 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -376,32 +376,36 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req return Request.WriteResponse(zen::HttpResponseCode::NotFound, HttpContentType::kText, "Invalid cache record"sv); } - uint32_t AttachmentCount = 0; - uint32_t FoundCount = 0; - uint64_t AttachmentBytes = 0ull; + const bool SkipAttachments = zen::CachePolicy::SkipAttachments == (Policy & zen::CachePolicy::SkipAttachments); + uint32_t AttachmentCount = 0; + uint32_t FoundCount = 0; + uint64_t AttachmentBytes = 0ull; CbPackage Package; - CacheRecord.IterateAttachments( - [this, &Ref, &Package, &AttachmentCount, &FoundCount, &AttachmentBytes](CbFieldView AttachmentHash) { - if (IoBuffer Chunk = m_CidStore.FindChunkByCid(AttachmentHash.AsHash())) - { - Package.AddAttachment(CbAttachment(CompressedBuffer::FromCompressed(SharedBuffer(Chunk)))); - AttachmentBytes += Chunk.Size(); - FoundCount++; - } - AttachmentCount++; - }); - - if (FoundCount != AttachmentCount) + if (!SkipAttachments) { - ZEN_WARN("GET - cache record '{}/{}' FAILED, found '{}' of '{}' attachments", - Ref.BucketSegment, - Ref.HashKey, - FoundCount, - AttachmentCount); + CacheRecord.IterateAttachments( + [this, &Ref, &Package, &AttachmentCount, &FoundCount, &AttachmentBytes](CbFieldView AttachmentHash) { + if (IoBuffer Chunk = m_CidStore.FindChunkByCid(AttachmentHash.AsHash())) + { + Package.AddAttachment(CbAttachment(CompressedBuffer::FromCompressed(SharedBuffer(Chunk)))); + AttachmentBytes += Chunk.Size(); + FoundCount++; + } + AttachmentCount++; + }); + + if (FoundCount != AttachmentCount) + { + ZEN_WARN("GET - cache record '{}/{}' FAILED, found '{}' of '{}' attachments", + Ref.BucketSegment, + Ref.HashKey, + FoundCount, + AttachmentCount); - return Request.WriteResponse(zen::HttpResponseCode::NotFound, HttpContentType::kText, "Missing attachments"sv); + return Request.WriteResponse(zen::HttpResponseCode::NotFound, HttpContentType::kText, "Missing attachments"sv); + } } Package.SetObject(LoadCompactBinaryObject(Value.Value)); -- cgit v1.2.3 From f4b7639b0f26d16c9a482b2db5eb598e851d7bb3 Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Mon, 20 Sep 2021 10:58:23 +0200 Subject: Respect skip attachments when retrieved from upstream cache. --- zenserver/cache/structuredcache.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'zenserver/cache/structuredcache.cpp') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 469e39bd4..2ab433b90 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -333,6 +333,18 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req if (FoundCount == AttachmentCount) { m_CacheStore.Put(Ref.BucketSegment, Ref.HashKey, {.Value = CacheRecord.GetBuffer().AsIoBuffer()}); + + if (zen::CachePolicy::SkipAttachments == (Policy & zen::CachePolicy::SkipAttachments)) + { + CbPackage PackageWithoutAttachments; + PackageWithoutAttachments.SetObject(CacheRecord); + + MemoryOutStream MemStream; + BinaryWriter Writer(MemStream); + PackageWithoutAttachments.Save(Writer); + + Value.Value = IoBuffer(IoBuffer::Clone, MemStream.Data(), MemStream.Size()); + } } else { -- cgit v1.2.3 From 782351959f697fca6b0804c134467b7d9c29da1a Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 20 Sep 2021 12:08:44 +0200 Subject: Moved more code into zen namespace, for consistency Also removed snapshot_manifest (remnants of vfs prototype) --- zenserver/cache/structuredcache.cpp | 132 ++++++++++++++++++------------------ 1 file changed, 66 insertions(+), 66 deletions(-) (limited to 'zenserver/cache/structuredcache.cpp') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index c66b1f98d..b3867bbc3 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -67,7 +67,7 @@ enum class CachePolicy : uint8_t gsl_DEFINE_ENUM_BITMASK_OPERATORS(CachePolicy); CachePolicy -ParseCachePolicy(const zen::HttpServerRequest::QueryParams& QueryParams) +ParseCachePolicy(const HttpServerRequest::QueryParams& QueryParams) { CachePolicy QueryPolicy = CachePolicy::Query; @@ -76,7 +76,7 @@ ParseCachePolicy(const zen::HttpServerRequest::QueryParams& QueryParams) if (!Opts.empty()) { QueryPolicy = CachePolicy::None; - zen::ForEachStrTok(Opts, ',', [&QueryPolicy](const std::string_view& Opt) { + ForEachStrTok(Opts, ',', [&QueryPolicy](const std::string_view& Opt) { if (Opt == detail::cacheopt::Local) { QueryPolicy |= CachePolicy::QueryLocal; @@ -97,7 +97,7 @@ ParseCachePolicy(const zen::HttpServerRequest::QueryParams& QueryParams) if (!Opts.empty()) { StorePolicy = CachePolicy::None; - zen::ForEachStrTok(Opts, ',', [&StorePolicy](const std::string_view& Opt) { + ForEachStrTok(Opts, ',', [&StorePolicy](const std::string_view& Opt) { if (Opt == detail::cacheopt::Local) { StorePolicy |= CachePolicy::StoreLocal; @@ -117,7 +117,7 @@ ParseCachePolicy(const zen::HttpServerRequest::QueryParams& QueryParams) std::string_view Opts = QueryParams.GetValue("skip"sv); if (!Opts.empty()) { - zen::ForEachStrTok(Opts, ',', [&SkipPolicy](const std::string_view& Opt) { + ForEachStrTok(Opts, ',', [&SkipPolicy](const std::string_view& Opt) { if (Opt == detail::cacheopt::Meta) { SkipPolicy |= CachePolicy::SkipMeta; @@ -144,11 +144,11 @@ ParseCachePolicy(const zen::HttpServerRequest::QueryParams& QueryParams) ////////////////////////////////////////////////////////////////////////// -HttpStructuredCacheService::HttpStructuredCacheService(::ZenCacheStore& InCacheStore, - zen::CasStore& InStore, - zen::CidStore& InCidStore, +HttpStructuredCacheService::HttpStructuredCacheService(ZenCacheStore& InCacheStore, + CasStore& InStore, + CidStore& InCidStore, std::unique_ptr UpstreamCache) -: m_Log(zen::logging::Get("cache")) +: m_Log(logging::Get("cache")) , m_CacheStore(InCacheStore) , m_CasStore(InStore) , m_CidStore(InCidStore) @@ -173,7 +173,7 @@ HttpStructuredCacheService::Flush() } void -HttpStructuredCacheService::HandleRequest(zen::HttpServerRequest& Request) +HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request) { CacheRef Ref; @@ -188,7 +188,7 @@ HttpStructuredCacheService::HandleRequest(zen::HttpServerRequest& Request) return HandleCacheBucketRequest(Request, Key); } - return Request.WriteResponse(zen::HttpResponseCode::BadRequest); // invalid URL + return Request.WriteResponse(HttpResponseCode::BadRequest); // invalid URL } const auto QueryParams = Request.GetQueryParams(); @@ -207,12 +207,12 @@ HttpStructuredCacheService::HandleRequest(zen::HttpServerRequest& Request) } void -HttpStructuredCacheService::HandleCacheBucketRequest(zen::HttpServerRequest& Request, std::string_view Bucket) +HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, std::string_view Bucket) { ZEN_UNUSED(Request, Bucket); switch (auto Verb = Request.RequestVerb()) { - using enum zen::HttpVerb; + using enum HttpVerb; case kHead: case kGet: @@ -226,22 +226,22 @@ HttpStructuredCacheService::HandleCacheBucketRequest(zen::HttpServerRequest& Req if (m_CacheStore.DropBucket(Bucket)) { - return Request.WriteResponse(zen::HttpResponseCode::OK); + return Request.WriteResponse(HttpResponseCode::OK); } else { - return Request.WriteResponse(zen::HttpResponseCode::NotFound); + return Request.WriteResponse(HttpResponseCode::NotFound); } break; } } void -HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Request, CacheRef& Ref, CachePolicy Policy) +HttpStructuredCacheService::HandleCacheRecordRequest(HttpServerRequest& Request, CacheRef& Ref, CachePolicy Policy) { switch (auto Verb = Request.RequestVerb()) { - using enum zen::HttpVerb; + using enum HttpVerb; case kHead: case kGet: @@ -253,7 +253,7 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req bool InUpstreamCache = false; const bool QueryUpstream = - !Success && m_UpstreamCache && (zen::CachePolicy::QueryRemote == (Policy & zen::CachePolicy::QueryRemote)); + !Success && m_UpstreamCache && (CachePolicy::QueryRemote == (Policy & CachePolicy::QueryRemote)); if (QueryUpstream) { @@ -272,14 +272,14 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req { if (CacheRecordType == ZenContentType::kCbObject) { - const zen::CbValidateError ValidationResult = - zen::ValidateCompactBinary(UpstreamResult.Value, zen::CbValidateMode::All); + const CbValidateError ValidationResult = + ValidateCompactBinary(UpstreamResult.Value, CbValidateMode::All); if (ValidationResult == CbValidateError::None) { - zen::CbObjectView CacheRecord(UpstreamResult.Value.Data()); + CbObjectView CacheRecord(UpstreamResult.Value.Data()); - zen::CbObjectWriter IndexData; + CbObjectWriter IndexData; IndexData.BeginArray("references"); CacheRecord.IterateAttachments([&](CbFieldView Attachment) { IndexData.AddHash(Attachment.AsHash()); }); IndexData.EndArray(); @@ -355,7 +355,7 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req { ZEN_DEBUG("MISS - '{}/{}'", Ref.BucketSegment, Ref.HashKey); - return Request.WriteResponse(zen::HttpResponseCode::NotFound); + return Request.WriteResponse(HttpResponseCode::NotFound); } if (Verb == kHead) @@ -367,13 +367,13 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req { CbObjectView CacheRecord(Value.Value.Data()); - const zen::CbValidateError ValidationResult = zen::ValidateCompactBinary(Value.Value, zen::CbValidateMode::All); + const CbValidateError ValidationResult = ValidateCompactBinary(Value.Value, CbValidateMode::All); if (ValidationResult != CbValidateError::None) { ZEN_WARN("GET - cache record '{}/{}' FAILED, invalid compact binary object", Ref.BucketSegment, Ref.HashKey); - return Request.WriteResponse(zen::HttpResponseCode::NotFound, HttpContentType::kText, "Invalid cache record"sv); + return Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Invalid cache record"sv); } uint32_t AttachmentCount = 0; @@ -401,7 +401,7 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req FoundCount, AttachmentCount); - return Request.WriteResponse(zen::HttpResponseCode::NotFound, HttpContentType::kText, "Missing attachments"sv); + return Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Missing attachments"sv); } Package.SetObject(LoadCompactBinaryObject(Value.Value)); @@ -419,7 +419,7 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req IoBuffer Response(IoBuffer::Clone, MemStream.Data(), MemStream.Size()); - return Request.WriteResponse(zen::HttpResponseCode::OK, HttpContentType::kCbPackage, Response); + return Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, Response); } else { @@ -429,23 +429,23 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req NiceBytes(Value.Value.Size()), InUpstreamCache ? "UPSTREAM" : "LOCAL"); - return Request.WriteResponse(zen::HttpResponseCode::OK, Value.Value.GetContentType(), Value.Value); + return Request.WriteResponse(HttpResponseCode::OK, Value.Value.GetContentType(), Value.Value); } } break; case kPut: { - zen::IoBuffer Body = Request.ReadPayload(); + IoBuffer Body = Request.ReadPayload(); if (!Body || Body.Size() == 0) { - return Request.WriteResponse(zen::HttpResponseCode::BadRequest); + return Request.WriteResponse(HttpResponseCode::BadRequest); } const HttpContentType ContentType = Request.RequestContentType(); - const bool StoreUpstream = m_UpstreamCache && (zen::CachePolicy::StoreRemote == (Policy & zen::CachePolicy::StoreRemote)); + const bool StoreUpstream = m_UpstreamCache && (CachePolicy::StoreRemote == (Policy & CachePolicy::StoreRemote)); if (ContentType == HttpContentType::kBinary || ContentType == HttpContentType::kUnknownContentType) { @@ -459,13 +459,13 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req {.Type = ZenContentType::kBinary, .CacheKey = {Ref.BucketSegment, Ref.HashKey}}); } - return Request.WriteResponse(zen::HttpResponseCode::Created); + return Request.WriteResponse(HttpResponseCode::Created); } else if (ContentType == HttpContentType::kCbObject) { // Validate payload before accessing it - const zen::CbValidateError ValidationResult = - zen::ValidateCompactBinary(MemoryView(Body.Data(), Body.Size()), zen::CbValidateMode::All); + const CbValidateError ValidationResult = + ValidateCompactBinary(MemoryView(Body.Data(), Body.Size()), CbValidateMode::All); if (ValidationResult != CbValidateError::None) { @@ -481,7 +481,7 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req } // Extract referenced payload hashes - zen::CbObjectView Cbo(Body.Data()); + CbObjectView Cbo(Body.Data()); std::vector References; std::vector MissingRefs; @@ -492,7 +492,7 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req if (!References.empty()) { - zen::CbObjectWriter Idx; + CbObjectWriter Idx; Idx.BeginArray("references"); for (const IoHash& Hash : References) @@ -514,7 +514,7 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req ZEN_DEBUG("PUT - cache record '{}/{}' {}, {}/{} attachments missing", Ref.BucketSegment, Ref.HashKey, - zen::NiceBytes(CacheValue.Value.Size()), + NiceBytes(CacheValue.Value.Size()), MissingRefs.size(), References.size()); @@ -525,12 +525,12 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req .CacheKey = {Ref.BucketSegment, Ref.HashKey}, .PayloadIds = std::move(References)}); - return Request.WriteResponse(zen::HttpResponseCode::Created); + return Request.WriteResponse(HttpResponseCode::Created); } else { // TODO: Binary attachments? - zen::CbObjectWriter Response; + CbObjectWriter Response; Response.BeginArray("needs"); for (const IoHash& MissingRef : MissingRefs) { @@ -540,7 +540,7 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req Response.EndArray(); // Return Created | BadRequest? - return Request.WriteResponse(zen::HttpResponseCode::Created, Response.Save()); + return Request.WriteResponse(HttpResponseCode::Created, Response.Save()); } } else if (ContentType == HttpContentType::kCbPackage) @@ -631,17 +631,17 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req ZEN_DEBUG("PUT - cache record '{}/{}' {}, {}/{} ({}/{}) new attachments", Ref.BucketSegment, Ref.HashKey, - zen::NiceBytes(TotalPackageBytes), + NiceBytes(TotalPackageBytes), AttachmentResult.NewCount, AttachmentResult.Count, - zen::NiceBytes(AttachmentResult.NewBytes), - zen::NiceBytes(AttachmentResult.Bytes)); + NiceBytes(AttachmentResult.NewBytes), + NiceBytes(AttachmentResult.Bytes)); - return Request.WriteResponse(zen::HttpResponseCode::Created); + return Request.WriteResponse(HttpResponseCode::Created); } else { - return Request.WriteResponse(zen::HttpResponseCode::BadRequest); + return Request.WriteResponse(HttpResponseCode::BadRequest); } } break; @@ -655,7 +655,7 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req } void -HttpStructuredCacheService::HandleCachePayloadRequest(zen::HttpServerRequest& Request, CacheRef& Ref, CachePolicy Policy) +HttpStructuredCacheService::HandleCachePayloadRequest(HttpServerRequest& Request, CacheRef& Ref, CachePolicy Policy) { // Note: the URL references the uncompressed payload hash - so this maintains the mapping // from uncompressed CAS identity (aka CID/Content ID) to the stored payload hash @@ -667,12 +667,12 @@ HttpStructuredCacheService::HandleCachePayloadRequest(zen::HttpServerRequest& Re switch (auto Verb = Request.RequestVerb()) { - using enum zen::HttpVerb; + using enum HttpVerb; case kHead: case kGet: { - zen::IoBuffer Payload = m_CidStore.FindChunkByCid(Ref.PayloadId); + IoBuffer Payload = m_CidStore.FindChunkByCid(Ref.PayloadId); bool InUpstreamCache = false; if (!Payload && m_UpstreamCache) @@ -680,11 +680,11 @@ HttpStructuredCacheService::HandleCachePayloadRequest(zen::HttpServerRequest& Re if (auto UpstreamResult = m_UpstreamCache->GetCachePayload({{Ref.BucketSegment, Ref.HashKey}, Ref.PayloadId}); UpstreamResult.Success) { - if (zen::CompressedBuffer Compressed = zen::CompressedBuffer::FromCompressed(SharedBuffer(UpstreamResult.Value))) + if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(UpstreamResult.Value))) { Payload = UpstreamResult.Value; - zen::IoHash ChunkHash = zen::IoHash::HashBuffer(Payload); - zen::CasStore::InsertResult Result = m_CasStore.InsertChunk(Payload, ChunkHash); + IoHash ChunkHash = IoHash::HashBuffer(Payload); + CasStore::InsertResult Result = m_CasStore.InsertChunk(Payload, ChunkHash); InUpstreamCache = true; m_CidStore.AddCompressedCid(Ref.PayloadId, ChunkHash); @@ -699,7 +699,7 @@ HttpStructuredCacheService::HandleCachePayloadRequest(zen::HttpServerRequest& Re if (!Payload) { ZEN_DEBUG("MISS - '{}/{}/{}'", Ref.BucketSegment, Ref.HashKey, Ref.PayloadId); - return Request.WriteResponse(zen::HttpResponseCode::NotFound); + return Request.WriteResponse(HttpResponseCode::NotFound); } ZEN_DEBUG("HIT - '{}/{}/{}' {} (type: {}) ({})", @@ -715,29 +715,29 @@ HttpStructuredCacheService::HandleCachePayloadRequest(zen::HttpServerRequest& Re Request.SetSuppressResponseBody(); } - return Request.WriteResponse(zen::HttpResponseCode::OK, zen::HttpContentType::kBinary, Payload); + return Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, Payload); } break; case kPut: { - if (zen::IoBuffer Body = Request.ReadPayload()) + if (IoBuffer Body = Request.ReadPayload()) { if (Body.Size() == 0) { - return Request.WriteResponse(zen::HttpResponseCode::BadRequest, + return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Empty payload not permitted"); } - zen::IoHash ChunkHash = zen::IoHash::HashBuffer(Body); + IoHash ChunkHash = IoHash::HashBuffer(Body); - zen::CompressedBuffer Compressed = zen::CompressedBuffer::FromCompressed(SharedBuffer(Body)); + CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Body)); if (!Compressed) { // All attachment payloads need to be in compressed buffer format - return Request.WriteResponse(zen::HttpResponseCode::BadRequest, + return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Attachments must be compressed"); } @@ -749,7 +749,7 @@ HttpStructuredCacheService::HandleCachePayloadRequest(zen::HttpServerRequest& Re return Request.WriteResponse(HttpResponseCode::BadRequest); } - zen::CasStore::InsertResult Result = m_CasStore.InsertChunk(Body, ChunkHash); + CasStore::InsertResult Result = m_CasStore.InsertChunk(Body, ChunkHash); m_CidStore.AddCompressedCid(Ref.PayloadId, ChunkHash); @@ -763,11 +763,11 @@ HttpStructuredCacheService::HandleCachePayloadRequest(zen::HttpServerRequest& Re if (Result.New) { - return Request.WriteResponse(zen::HttpResponseCode::Created); + return Request.WriteResponse(HttpResponseCode::Created); } else { - return Request.WriteResponse(zen::HttpResponseCode::OK); + return Request.WriteResponse(HttpResponseCode::OK); } } } @@ -783,7 +783,7 @@ HttpStructuredCacheService::HandleCachePayloadRequest(zen::HttpServerRequest& Re } bool -HttpStructuredCacheService::ValidateKeyUri(zen::HttpServerRequest& Request, CacheRef& OutRef) +HttpStructuredCacheService::ValidateKeyUri(HttpServerRequest& Request, CacheRef& OutRef) { std::string_view Key = Request.RelativeUri(); std::string_view::size_type BucketSplitOffset = Key.find_first_of('/'); @@ -819,14 +819,14 @@ HttpStructuredCacheService::ValidateKeyUri(zen::HttpServerRequest& Request, Cach PayloadSegment = Key.substr(PayloadSplitOffset + 1); } - if (HashSegment.size() != zen::IoHash::StringLength) + if (HashSegment.size() != IoHash::StringLength) { return false; } - if (!PayloadSegment.empty() && PayloadSegment.size() == zen::IoHash::StringLength) + if (!PayloadSegment.empty() && PayloadSegment.size() == IoHash::StringLength) { - const bool IsOk = zen::ParseHexBytes(PayloadSegment.data(), PayloadSegment.size(), OutRef.PayloadId.Hash); + const bool IsOk = ParseHexBytes(PayloadSegment.data(), PayloadSegment.size(), OutRef.PayloadId.Hash); if (!IsOk) { @@ -835,10 +835,10 @@ HttpStructuredCacheService::ValidateKeyUri(zen::HttpServerRequest& Request, Cach } else { - OutRef.PayloadId = zen::IoHash::Zero; + OutRef.PayloadId = IoHash::Zero; } - const bool IsOk = zen::ParseHexBytes(HashSegment.data(), HashSegment.size(), OutRef.HashKey.Hash); + const bool IsOk = ParseHexBytes(HashSegment.data(), HashSegment.size(), OutRef.HashKey.Hash); if (!IsOk) { -- cgit v1.2.3 From c199ec351e98b876ba90651e209791fa2929c476 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 20 Sep 2021 14:03:41 +0200 Subject: clang-format --- zenserver/cache/structuredcache.cpp | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) (limited to 'zenserver/cache/structuredcache.cpp') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 84522748e..17f8e04ef 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -145,8 +145,8 @@ ParseCachePolicy(const HttpServerRequest::QueryParams& QueryParams) ////////////////////////////////////////////////////////////////////////// HttpStructuredCacheService::HttpStructuredCacheService(ZenCacheStore& InCacheStore, - CasStore& InStore, - CidStore& InCidStore, + CasStore& InStore, + CidStore& InCidStore, std::unique_ptr UpstreamCache) : m_Log(logging::Get("cache")) , m_CacheStore(InCacheStore) @@ -188,7 +188,7 @@ HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request) return HandleCacheBucketRequest(Request, Key); } - return Request.WriteResponse(HttpResponseCode::BadRequest); // invalid URL + return Request.WriteResponse(HttpResponseCode::BadRequest); // invalid URL } const auto QueryParams = Request.GetQueryParams(); @@ -252,8 +252,7 @@ HttpStructuredCacheService::HandleCacheRecordRequest(HttpServerRequest& Request, bool Success = m_CacheStore.Get(Ref.BucketSegment, Ref.HashKey, /* out */ Value); bool InUpstreamCache = false; - const bool QueryUpstream = - !Success && m_UpstreamCache && (CachePolicy::QueryRemote == (Policy & CachePolicy::QueryRemote)); + const bool QueryUpstream = !Success && m_UpstreamCache && (CachePolicy::QueryRemote == (Policy & CachePolicy::QueryRemote)); if (QueryUpstream) { @@ -272,8 +271,7 @@ HttpStructuredCacheService::HandleCacheRecordRequest(HttpServerRequest& Request, { if (CacheRecordType == ZenContentType::kCbObject) { - const CbValidateError ValidationResult = - ValidateCompactBinary(UpstreamResult.Value, CbValidateMode::All); + const CbValidateError ValidationResult = ValidateCompactBinary(UpstreamResult.Value, CbValidateMode::All); if (ValidationResult == CbValidateError::None) { @@ -688,8 +686,8 @@ HttpStructuredCacheService::HandleCachePayloadRequest(HttpServerRequest& Request case kHead: case kGet: { - IoBuffer Payload = m_CidStore.FindChunkByCid(Ref.PayloadId); - bool InUpstreamCache = false; + IoBuffer Payload = m_CidStore.FindChunkByCid(Ref.PayloadId); + bool InUpstreamCache = false; if (!Payload && m_UpstreamCache) { @@ -698,10 +696,10 @@ HttpStructuredCacheService::HandleCachePayloadRequest(HttpServerRequest& Request { if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(UpstreamResult.Value))) { - Payload = UpstreamResult.Value; - IoHash ChunkHash = IoHash::HashBuffer(Payload); - CasStore::InsertResult Result = m_CasStore.InsertChunk(Payload, ChunkHash); - InUpstreamCache = true; + Payload = UpstreamResult.Value; + IoHash ChunkHash = IoHash::HashBuffer(Payload); + CasStore::InsertResult Result = m_CasStore.InsertChunk(Payload, ChunkHash); + InUpstreamCache = true; m_CidStore.AddCompressedCid(Ref.PayloadId, ChunkHash); } @@ -741,9 +739,7 @@ HttpStructuredCacheService::HandleCachePayloadRequest(HttpServerRequest& Request { if (Body.Size() == 0) { - return Request.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - "Empty payload not permitted"); + return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Empty payload not permitted"); } IoHash ChunkHash = IoHash::HashBuffer(Body); -- cgit v1.2.3 From 961173f44df332cd17e1b9875e4ef5eb9cea1d3a Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 20 Sep 2021 21:01:01 +0200 Subject: Added more scrub stubs in higher level services --- zenserver/cache/structuredcache.cpp | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'zenserver/cache/structuredcache.cpp') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 17f8e04ef..7f1fe7b44 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include "structuredcache.h" #include "structuredcachestore.h" @@ -172,6 +173,12 @@ HttpStructuredCacheService::Flush() { } +void +HttpStructuredCacheService::Scrub(ScrubContext& Ctx) +{ + ZEN_UNUSED(Ctx); +} + void HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request) { -- cgit v1.2.3