From d361aa896e2e74ae4a790c4668c78c830f9b5d1c Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 23 Feb 2023 14:54:22 +0100 Subject: store cache rawhash and rawsize for unstructured cache values (#234) * refactored MemoryCacheBucket to allow for storing RawHash/RawSize. * remove redundant conversions in AccessTime * reduce max count for memory cache bucket to 32-bit value * refactored DiskCacheBucket to allow for storing RawHash/RawSize. * Use CompressedBuffer::ValidateCompressedHeader when applicable * Make sure we rewrite the snapshot if we read an legacy existing index/log * changelog --- zenserver/cache/structuredcache.cpp | 87 ++++++++++++++++++++++++------------- 1 file changed, 57 insertions(+), 30 deletions(-) (limited to 'zenserver/cache/structuredcache.cpp') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 1ef880396..154676d77 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -973,7 +973,22 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con if (ContentType == HttpContentType::kBinary || ContentType == HttpContentType::kCompressedBinary) { - m_CacheStore.Put(Ref.Namespace, Ref.BucketSegment, Ref.HashKey, {.Value = Body}); + IoHash RawHash = IoHash::Zero; + uint64_t RawSize = Body.GetSize(); + if (ContentType == HttpContentType::kCompressedBinary) + { + if (!CompressedBuffer::ValidateCompressedHeader(Body, RawHash, RawSize)) + { + return Request.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Payload is not a valid compressed binary"sv); + } + } + else + { + RawHash = IoHash::HashBuffer(SharedBuffer(Body)); + } + m_CacheStore.Put(Ref.Namespace, Ref.BucketSegment, Ref.HashKey, {.Value = Body, .RawSize = RawSize, .RawHash = RawHash}); if (EnumHasAllFlags(PolicyFromUrl, CachePolicy::StoreRemote)) { @@ -1965,6 +1980,7 @@ HttpStructuredCacheService::HandleRpcPutCacheValues(const CbPackage& BatchReques PolicyText = RequestObject["Policy"sv].AsString(); CachePolicy Policy = !PolicyText.empty() ? ParseCachePolicy(PolicyText) : DefaultPolicy; IoHash RawHash = RequestObject["RawHash"sv].AsBinaryAttachment(); + uint64_t RawSize = RequestObject["RawSize"sv].AsUInt64(); bool Succeeded = false; uint64_t TransferredSize = 0; @@ -1984,7 +2000,11 @@ HttpStructuredCacheService::HandleRpcPutCacheValues(const CbPackage& BatchReques { IoBuffer Value = Chunk.GetCompressed().Flatten().AsIoBuffer(); Value.SetContentType(ZenContentType::kCompressedBinary); - m_CacheStore.Put(*Namespace, Key.Bucket, Key.Hash, {.Value = Value}); + if (RawSize == 0) + { + RawSize = Chunk.DecodeRawSize(); + } + m_CacheStore.Put(*Namespace, Key.Bucket, Key.Hash, {.Value = Value, .RawSize = RawSize, .RawHash = RawHash}); TransferredSize = Chunk.GetCompressedSize(); } Succeeded = true; @@ -2089,7 +2109,9 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(CbObjectView RpcRequest) { if (m_CacheStore.Get(*Namespace, Key.Bucket, Key.Hash, CacheValue) && IsCompressedBinary(CacheValue.Value.GetContentType())) { - Request.Result = CompressedBuffer::FromCompressed(SharedBuffer(CacheValue.Value), Request.RawHash, Request.RawSize); + Request.RawHash = CacheValue.RawHash; + Request.RawSize = CacheValue.RawSize; + Request.Result = CompressedBuffer::FromCompressedNoValidate(std::move(CacheValue.Value)); } } if (Request.Result) @@ -2157,12 +2179,15 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(CbObjectView RpcRequest) { if (HasData && !SkipData) { - Request.Result = CompressedBuffer::FromCompressed(SharedBuffer(Params.Value), Request.RawHash, Request.RawSize); + Request.Result = CompressedBuffer::FromCompressedNoValidate(IoBuffer(Params.Value)); } if (HasData && StoreData) { - m_CacheStore.Put(*Namespace, Request.Key.Bucket, Request.Key.Hash, ZenCacheValue{Params.Value}); + m_CacheStore.Put(*Namespace, + Request.Key.Bucket, + Request.Key.Hash, + ZenCacheValue{.Value = Params.Value, .RawSize = Request.RawSize, .RawHash = Request.RawHash}); } ZEN_DEBUG("GETCACHEVALUES HIT - '{}/{}/{}' {} ({}) in {}", @@ -2531,16 +2556,23 @@ HttpStructuredCacheService::GetLocalCacheRecords(std::string_view Namespac } else if (IoBuffer Payload = m_CidStore.FindChunkByCid(Request->Key->ChunkId)) { - IoHash RawHash; - CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Payload), RawHash, Request->RawSize); - if (Compressed) + if (!EnumHasAllFlags(Request->DownstreamPolicy, CachePolicy::SkipData)) { - if (!EnumHasAllFlags(Request->DownstreamPolicy, CachePolicy::SkipData)) + Request->Value = CompressedBuffer::FromCompressedNoValidate(std::move(Payload)); + if (Request->Value) { - Request->Value = Compressed; + Request->Exists = true; + Request->RawSizeKnown = false; + } + } + else + { + IoHash _; + if (CompressedBuffer::ValidateCompressedHeader(Payload, _, Request->RawSize)) + { + Request->Exists = true; + Request->RawSizeKnown = true; } - Request->Exists = true; - Request->RawSizeKnown = true; } } } @@ -2571,19 +2603,13 @@ HttpStructuredCacheService::GetLocalCacheValues(std::string_view Namespa { if (IsCompressedBinary(CacheValue.Value.GetContentType())) { - IoHash RawHash; - uint64_t RawSize; - CompressedBuffer Result = CompressedBuffer::FromCompressed(SharedBuffer(CacheValue.Value), RawHash, RawSize); - if (Result) + Request->Key->ChunkId = CacheValue.RawHash; + Request->Exists = true; + Request->RawSize = CacheValue.RawSize; + Request->RawSizeKnown = true; + if (!EnumHasAllFlags(Request->DownstreamPolicy, CachePolicy::SkipData)) { - if (!EnumHasAllFlags(Request->DownstreamPolicy, CachePolicy::SkipData)) - { - Request->Value = Result; - } - Request->Key->ChunkId = RawHash; - Request->Exists = true; - Request->RawSize = RawSize; - Request->RawSizeKnown = true; + Request->Value = CompressedBuffer::FromCompressedNoValidate(std::move(CacheValue.Value)); } } } @@ -2625,10 +2651,8 @@ HttpStructuredCacheService::GetUpstreamCacheChunks(std::string_view Names if (EnumHasAllFlags(Request.DownstreamPolicy, CachePolicy::StoreLocal) || !EnumHasAllFlags(Request.DownstreamPolicy, CachePolicy::SkipData)) { - IoHash RawHash; - uint64_t RawSize; - CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Params.Value), RawHash, RawSize); - if (!Compressed || RawSize != Params.RawSize || RawHash != Params.RawHash) + CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(IoBuffer(Params.Value)); + if (!Compressed) { return; } @@ -2637,11 +2661,14 @@ HttpStructuredCacheService::GetUpstreamCacheChunks(std::string_view Names { if (Request.IsRecordRequest) { - m_CidStore.AddChunk(Params.Value, RawHash); + m_CidStore.AddChunk(Params.Value, Params.RawHash); } else { - m_CacheStore.Put(Namespace, Key.Key.Bucket, Key.Key.Hash, {.Value = Params.Value}); + m_CacheStore.Put(Namespace, + Key.Key.Bucket, + Key.Key.Hash, + {.Value = Params.Value, .RawSize = Params.RawSize, .RawHash = Params.RawHash}); } } if (!EnumHasAllFlags(Request.DownstreamPolicy, CachePolicy::SkipData)) -- cgit v1.2.3