diff options
| author | Stefan Boberg <[email protected]> | 2023-05-02 10:01:47 +0200 |
|---|---|---|
| committer | GitHub <[email protected]> | 2023-05-02 10:01:47 +0200 |
| commit | 075d17f8ada47e990fe94606c3d21df409223465 (patch) | |
| tree | e50549b766a2f3c354798a54ff73404217b4c9af /zenutil/cache/cacherequests.cpp | |
| parent | fix: bundle shouldn't append content zip to zen (diff) | |
| download | zen-075d17f8ada47e990fe94606c3d21df409223465.tar.xz zen-075d17f8ada47e990fe94606c3d21df409223465.zip | |
moved source directories into `/src` (#264)
* moved source directories into `/src`
* updated bundle.lua for new `src` path
* moved some docs, icon
* removed old test trees
Diffstat (limited to 'zenutil/cache/cacherequests.cpp')
| -rw-r--r-- | zenutil/cache/cacherequests.cpp | 1643 |
1 files changed, 0 insertions, 1643 deletions
diff --git a/zenutil/cache/cacherequests.cpp b/zenutil/cache/cacherequests.cpp deleted file mode 100644 index 4c865ec22..000000000 --- a/zenutil/cache/cacherequests.cpp +++ /dev/null @@ -1,1643 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include <zenutil/cache/cacherequests.h> - -#include <zencore/compactbinary.h> -#include <zencore/compactbinarybuilder.h> -#include <zencore/compactbinarypackage.h> -#include <zencore/fmtutils.h> -#include <zencore/logging.h> -#include <zencore/zencore.h> - -#include <string> -#include <string_view> - -#if ZEN_WITH_TESTS -# include <zencore/testing.h> -#endif - -namespace zen { - -namespace cacherequests { - - namespace { - constinit AsciiSet ValidNamespaceNameCharactersSet{"abcdefghijklmnopqrstuvwxyz0123456789-_.ABCDEFGHIJKLMNOPQRSTUVWXYZ"}; - constinit AsciiSet ValidBucketNameCharactersSet{"abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"}; - - std::optional<std::string> GetValidNamespaceName(std::string_view Name) - { - if (Name.empty()) - { - ZEN_WARN("Namespace is invalid, empty namespace is not allowed"); - return {}; - } - - if (Name.length() > 64) - { - ZEN_WARN("Namespace '{}' is invalid, length exceeds 64 characters", Name); - return {}; - } - - if (!AsciiSet::HasOnly(Name, ValidNamespaceNameCharactersSet)) - { - ZEN_WARN("Namespace '{}' is invalid, invalid characters detected", Name); - return {}; - } - - return ToLower(Name); - } - - std::optional<std::string> GetValidBucketName(std::string_view Name) - { - if (Name.empty()) - { - ZEN_WARN("Bucket name is invalid, empty bucket name is not allowed"); - return {}; - } - - if (!AsciiSet::HasOnly(Name, ValidBucketNameCharactersSet)) - { - ZEN_WARN("Bucket name '{}' is invalid, invalid characters detected", Name); - return {}; - } - - return ToLower(Name); - } - - std::optional<IoHash> GetValidIoHash(std::string_view Hash) - { - if (Hash.length() != IoHash::StringLength) - { - return {}; - } - - IoHash KeyHash; - if (!ParseHexBytes(Hash.data(), Hash.size(), KeyHash.Hash)) - { - return {}; - } - return KeyHash; - } - - std::optional<CacheRecordPolicy> Convert(const OptionalCacheRecordPolicy& Policy) - { - return Policy.IsValid() ? Policy.Get() : std::optional<CacheRecordPolicy>{}; - }; - } // namespace - - std::optional<std::string> GetRequestNamespace(const CbObjectView& Params) - { - CbFieldView NamespaceField = Params["Namespace"]; - if (!NamespaceField) - { - return std::string("!default!"); // ZenCacheStore::DefaultNamespace); - } - - if (NamespaceField.HasError()) - { - return {}; - } - if (!NamespaceField.IsString()) - { - return {}; - } - return GetValidNamespaceName(NamespaceField.AsString()); - } - - bool GetRequestCacheKey(const CbObjectView& KeyView, CacheKey& Key) - { - CbFieldView BucketField = KeyView["Bucket"]; - if (BucketField.HasError()) - { - return false; - } - if (!BucketField.IsString()) - { - return false; - } - std::optional<std::string> Bucket = GetValidBucketName(BucketField.AsString()); - if (!Bucket.has_value()) - { - return false; - } - CbFieldView HashField = KeyView["Hash"]; - if (HashField.HasError()) - { - return false; - } - if (!HashField.IsHash()) - { - return false; - } - Key.Bucket = *Bucket; - Key.Hash = HashField.AsHash(); - return true; - } - - void WriteCacheRequestKey(CbObjectWriter& Writer, const CacheKey& Value) - { - Writer.BeginObject("Key"); - { - Writer << "Bucket" << Value.Bucket; - Writer << "Hash" << Value.Hash; - } - Writer.EndObject(); - } - - void WriteOptionalCacheRequestPolicy(CbObjectWriter& Writer, std::string_view FieldName, const std::optional<CacheRecordPolicy>& Policy) - { - if (Policy) - { - Writer.SetName(FieldName); - Policy->Save(Writer); - } - } - - std::optional<CachePolicy> GetCachePolicy(CbObjectView ObjectView, std::string_view FieldName) - { - std::string_view DefaultPolicyText = ObjectView[FieldName].AsString(); - if (DefaultPolicyText.empty()) - { - return {}; - } - return ParseCachePolicy(DefaultPolicyText); - } - - void WriteCachePolicy(CbObjectWriter& Writer, std::string_view FieldName, const std::optional<CachePolicy>& Policy) - { - if (Policy) - { - Writer << FieldName << WriteToString<128>(*Policy); - } - } - - bool PutCacheRecordsRequest::Parse(const CbPackage& Package) - { - CbObjectView BatchObject = Package.GetObject(); - ZEN_ASSERT(BatchObject["Method"].AsString() == "PutCacheRecords"); - AcceptMagic = BatchObject["AcceptType"].AsUInt32(0); - - CbObjectView Params = BatchObject["Params"].AsObjectView(); - std::optional<std::string> RequestNamespace = GetRequestNamespace(Params); - if (!RequestNamespace) - { - return false; - } - Namespace = *RequestNamespace; - DefaultPolicy = GetCachePolicy(Params, "DefaultPolicy").value_or(CachePolicy::Default); - - CbArrayView RequestFieldArray = Params["Requests"].AsArrayView(); - Requests.resize(RequestFieldArray.Num()); - for (size_t RequestIndex = 0; CbFieldView RequestField : RequestFieldArray) - { - CbObjectView RequestObject = RequestField.AsObjectView(); - CbObjectView RecordObject = RequestObject["Record"].AsObjectView(); - CbObjectView KeyView = RecordObject["Key"].AsObjectView(); - - PutCacheRecordRequest& Request = Requests[RequestIndex++]; - - if (!GetRequestCacheKey(KeyView, Request.Key)) - { - return false; - } - - Request.Policy = Convert(CacheRecordPolicy::Load(RequestObject["Policy"].AsObjectView())); - - std::unordered_map<IoHash, size_t, IoHash::Hasher> RawHashToAttachmentIndex; - - CbArrayView ValuesArray = RecordObject["Values"].AsArrayView(); - Request.Values.resize(ValuesArray.Num()); - RawHashToAttachmentIndex.reserve(ValuesArray.Num()); - for (size_t Index = 0; CbFieldView Value : ValuesArray) - { - CbObjectView ObjectView = Value.AsObjectView(); - IoHash AttachmentHash = ObjectView["RawHash"].AsHash(); - RawHashToAttachmentIndex[AttachmentHash] = Index; - Request.Values[Index++] = {.Id = ObjectView["Id"].AsObjectId(), .RawHash = AttachmentHash}; - } - - RecordObject.IterateAttachments([&](CbFieldView HashView) { - const IoHash ValueHash = HashView.AsHash(); - if (const CbAttachment* Attachment = Package.FindAttachment(ValueHash)) - { - if (Attachment->IsCompressedBinary()) - { - auto It = RawHashToAttachmentIndex.find(ValueHash); - ZEN_ASSERT(It != RawHashToAttachmentIndex.end()); - PutCacheRecordRequestValue& Value = Request.Values[It->second]; - ZEN_ASSERT(Value.RawHash == ValueHash); - Value.Body = Attachment->AsCompressedBinary(); - ZEN_ASSERT_SLOW(Value.Body.DecodeRawHash() == Value.RawHash); - } - } - }); - } - - return true; - } - - bool PutCacheRecordsRequest::Format(CbPackage& OutPackage) const - { - CbObjectWriter Writer; - Writer << "Method" - << "PutCacheRecords"; - if (AcceptMagic != 0) - { - Writer << "Accept" << AcceptMagic; - } - - Writer.BeginObject("Params"); - { - Writer << "DefaultPolicy" << WriteToString<128>(DefaultPolicy); - Writer << "Namespace" << Namespace; - - Writer.BeginArray("Requests"); - for (const PutCacheRecordRequest& RecordRequest : Requests) - { - Writer.BeginObject(); - { - Writer.BeginObject("Record"); - { - WriteCacheRequestKey(Writer, RecordRequest.Key); - Writer.BeginArray("Values"); - for (const PutCacheRecordRequestValue& Value : RecordRequest.Values) - { - Writer.BeginObject(); - { - Writer.AddObjectId("Id", Value.Id); - const CompressedBuffer& Buffer = Value.Body; - if (Buffer) - { - IoHash AttachmentHash = Buffer.DecodeRawHash(); // TODO: Slow! - Writer.AddBinaryAttachment("RawHash", AttachmentHash); - OutPackage.AddAttachment(CbAttachment(Buffer, AttachmentHash)); - Writer.AddInteger("RawSize", Buffer.DecodeRawSize()); // TODO: Slow! - } - else - { - if (Value.RawHash == IoHash::Zero) - { - return false; - } - Writer.AddBinaryAttachment("RawHash", Value.RawHash); - } - } - Writer.EndObject(); - } - Writer.EndArray(); - } - Writer.EndObject(); - WriteOptionalCacheRequestPolicy(Writer, "Policy", RecordRequest.Policy); - } - Writer.EndObject(); - } - Writer.EndArray(); - } - Writer.EndObject(); - OutPackage.SetObject(Writer.Save()); - - return true; - } - - bool PutCacheRecordsResult::Parse(const CbPackage& Package) - { - CbArrayView ResultsArray = Package.GetObject()["Result"].AsArrayView(); - if (!ResultsArray) - { - return false; - } - CbFieldViewIterator It = ResultsArray.CreateViewIterator(); - while (It.HasValue()) - { - Success.push_back(It.AsBool()); - It++; - } - return true; - } - - bool PutCacheRecordsResult::Format(CbPackage& OutPackage) const - { - CbObjectWriter ResponseObject; - ResponseObject.BeginArray("Result"); - for (bool Value : Success) - { - ResponseObject.AddBool(Value); - } - ResponseObject.EndArray(); - - OutPackage.SetObject(ResponseObject.Save()); - return true; - } - - bool GetCacheRecordsRequest::Parse(const CbObjectView& RpcRequest) - { - ZEN_ASSERT(RpcRequest["Method"].AsString() == "GetCacheRecords"); - AcceptMagic = RpcRequest["AcceptType"].AsUInt32(0); - AcceptOptions = RpcRequest["AcceptFlags"].AsUInt16(0); - ProcessPid = RpcRequest["Pid"].AsInt32(0); - - CbObjectView Params = RpcRequest["Params"].AsObjectView(); - std::optional<std::string> RequestNamespace = GetRequestNamespace(Params); - if (!RequestNamespace) - { - return false; - } - - Namespace = *RequestNamespace; - DefaultPolicy = GetCachePolicy(Params, "DefaultPolicy").value_or(CachePolicy::Default); - - CbArrayView RequestsArray = Params["Requests"].AsArrayView(); - Requests.reserve(RequestsArray.Num()); - for (CbFieldView RequestField : RequestsArray) - { - CbObjectView RequestObject = RequestField.AsObjectView(); - CbObjectView KeyObject = RequestObject["Key"].AsObjectView(); - - GetCacheRecordRequest& Request = Requests.emplace_back(); - - if (!GetRequestCacheKey(KeyObject, Request.Key)) - { - return false; - } - - Request.Policy = Convert(CacheRecordPolicy::Load(RequestObject["Policy"].AsObjectView())); - } - return true; - } - - bool GetCacheRecordsRequest::Parse(const CbPackage& RpcRequest) { return Parse(RpcRequest.GetObject()); } - - bool GetCacheRecordsRequest::Format(CbObjectWriter& Writer, const std::span<const size_t> OptionalRecordFilter) const - { - Writer << "Method" - << "GetCacheRecords"; - if (AcceptMagic != 0) - { - Writer << "Accept" << AcceptMagic; - } - if (AcceptOptions != 0) - { - Writer << "AcceptFlags" << AcceptOptions; - } - if (ProcessPid != 0) - { - Writer << "Pid" << ProcessPid; - } - - Writer.BeginObject("Params"); - { - Writer << "DefaultPolicy" << WriteToString<128>(DefaultPolicy); - Writer << "Namespace" << Namespace; - Writer.BeginArray("Requests"); - if (OptionalRecordFilter.empty()) - { - for (const GetCacheRecordRequest& RecordRequest : Requests) - { - Writer.BeginObject(); - { - WriteCacheRequestKey(Writer, RecordRequest.Key); - WriteOptionalCacheRequestPolicy(Writer, "Policy", RecordRequest.Policy); - } - Writer.EndObject(); - } - } - else - { - for (size_t Index : OptionalRecordFilter) - { - const GetCacheRecordRequest& RecordRequest = Requests[Index]; - Writer.BeginObject(); - { - WriteCacheRequestKey(Writer, RecordRequest.Key); - WriteOptionalCacheRequestPolicy(Writer, "Policy", RecordRequest.Policy); - } - Writer.EndObject(); - } - } - Writer.EndArray(); - } - Writer.EndObject(); - - return true; - } - - bool GetCacheRecordsRequest::Format(CbPackage& OutPackage, const std::span<const size_t> OptionalRecordFilter) const - { - CbObjectWriter Writer; - if (!Format(Writer, OptionalRecordFilter)) - { - return false; - } - OutPackage.SetObject(Writer.Save()); - return true; - } - - bool GetCacheRecordsResult::Parse(const CbPackage& Package, const std::span<const size_t> OptionalRecordResultIndexes) - { - CbObject ResponseObject = Package.GetObject(); - CbArrayView ResultsArray = ResponseObject["Result"].AsArrayView(); - if (!ResultsArray) - { - return false; - } - - Results.reserve(ResultsArray.Num()); - if (!OptionalRecordResultIndexes.empty() && ResultsArray.Num() != OptionalRecordResultIndexes.size()) - { - return false; - } - for (size_t Index = 0; CbFieldView RecordView : ResultsArray) - { - size_t ResultIndex = OptionalRecordResultIndexes.empty() ? Index : OptionalRecordResultIndexes[Index]; - Index++; - - if (Results.size() <= ResultIndex) - { - Results.resize(ResultIndex + 1); - } - if (RecordView.IsNull()) - { - continue; - } - Results[ResultIndex] = GetCacheRecordResult{}; - GetCacheRecordResult& Request = Results[ResultIndex].value(); - CbObjectView RecordObject = RecordView.AsObjectView(); - CbObjectView KeyObject = RecordObject["Key"].AsObjectView(); - if (!GetRequestCacheKey(KeyObject, Request.Key)) - { - return false; - } - - CbArrayView ValuesArray = RecordObject["Values"].AsArrayView(); - Request.Values.reserve(ValuesArray.Num()); - for (CbFieldView Value : ValuesArray) - { - CbObjectView ValueObject = Value.AsObjectView(); - IoHash RawHash = ValueObject["RawHash"].AsHash(); - uint64_t RawSize = ValueObject["RawSize"].AsUInt64(); - Oid Id = ValueObject["Id"].AsObjectId(); - const CbAttachment* Attachment = Package.FindAttachment(RawHash); - if (!Attachment) - { - Request.Values.push_back({.Id = Id, .RawHash = RawHash, .RawSize = RawSize, .Body = {}}); - continue; - } - if (!Attachment->IsCompressedBinary()) - { - return false; - } - Request.Values.push_back({.Id = Id, .RawHash = RawHash, .RawSize = RawSize, .Body = Attachment->AsCompressedBinary()}); - } - } - return true; - } - - bool GetCacheRecordsResult::Format(CbPackage& OutPackage) const - { - CbObjectWriter Writer; - - Writer.BeginArray("Result"); - for (const std::optional<GetCacheRecordResult>& RecordResult : Results) - { - if (!RecordResult.has_value()) - { - Writer.AddNull(); - continue; - } - Writer.BeginObject(); - WriteCacheRequestKey(Writer, RecordResult->Key); - - Writer.BeginArray("Values"); - for (const GetCacheRecordResultValue& Value : RecordResult->Values) - { - IoHash AttachmentHash = Value.Body ? Value.Body.DecodeRawHash() : Value.RawHash; - Writer.BeginObject(); - { - Writer.AddObjectId("Id", Value.Id); - Writer.AddHash("RawHash", AttachmentHash); - Writer.AddInteger("RawSize", Value.Body ? Value.Body.DecodeRawSize() : Value.RawSize); - } - Writer.EndObject(); - if (Value.Body) - { - OutPackage.AddAttachment(CbAttachment(Value.Body, AttachmentHash)); - } - } - - Writer.EndArray(); - Writer.EndObject(); - } - Writer.EndArray(); - - OutPackage.SetObject(Writer.Save()); - return true; - } - - bool PutCacheValuesRequest::Parse(const CbPackage& Package) - { - CbObjectView BatchObject = Package.GetObject(); - ZEN_ASSERT(BatchObject["Method"].AsString() == "PutCacheValues"); - AcceptMagic = BatchObject["AcceptType"].AsUInt32(0); - - CbObjectView Params = BatchObject["Params"].AsObjectView(); - std::optional<std::string> RequestNamespace = cacherequests::GetRequestNamespace(Params); - if (!RequestNamespace) - { - return false; - } - - Namespace = *RequestNamespace; - DefaultPolicy = GetCachePolicy(Params, "DefaultPolicy").value_or(CachePolicy::Default); - - CbArrayView RequestsArray = Params["Requests"].AsArrayView(); - Requests.reserve(RequestsArray.Num()); - for (CbFieldView RequestField : RequestsArray) - { - CbObjectView RequestObject = RequestField.AsObjectView(); - CbObjectView KeyObject = RequestObject["Key"].AsObjectView(); - - PutCacheValueRequest& Request = Requests.emplace_back(); - - if (!GetRequestCacheKey(KeyObject, Request.Key)) - { - return false; - } - - Request.RawHash = RequestObject["RawHash"].AsBinaryAttachment(); - Request.Policy = GetCachePolicy(RequestObject, "Policy"); - - if (const CbAttachment* Attachment = Package.FindAttachment(Request.RawHash)) - { - if (!Attachment->IsCompressedBinary()) - { - return false; - } - Request.Body = Attachment->AsCompressedBinary(); - } - } - return true; - } - - bool PutCacheValuesRequest::Format(CbPackage& OutPackage) const - { - CbObjectWriter Writer; - Writer << "Method" - << "PutCacheValues"; - if (AcceptMagic != 0) - { - Writer << "Accept" << AcceptMagic; - } - - Writer.BeginObject("Params"); - { - Writer << "DefaultPolicy" << WriteToString<128>(DefaultPolicy); - Writer << "Namespace" << Namespace; - - Writer.BeginArray("Requests"); - for (const PutCacheValueRequest& ValueRequest : Requests) - { - Writer.BeginObject(); - { - WriteCacheRequestKey(Writer, ValueRequest.Key); - if (ValueRequest.Body) - { - IoHash AttachmentHash = ValueRequest.Body.DecodeRawHash(); - if (ValueRequest.RawHash != IoHash::Zero && AttachmentHash != ValueRequest.RawHash) - { - return false; - } - Writer.AddBinaryAttachment("RawHash", AttachmentHash); - OutPackage.AddAttachment(CbAttachment(ValueRequest.Body, AttachmentHash)); - } - else if (ValueRequest.RawHash != IoHash::Zero) - { - Writer.AddBinaryAttachment("RawHash", ValueRequest.RawHash); - } - else - { - return false; - } - WriteCachePolicy(Writer, "Policy", ValueRequest.Policy); - } - Writer.EndObject(); - } - Writer.EndArray(); - } - Writer.EndObject(); - - OutPackage.SetObject(Writer.Save()); - return true; - } - - bool PutCacheValuesResult::Parse(const CbPackage& Package) - { - CbArrayView ResultsArray = Package.GetObject()["Result"].AsArrayView(); - if (!ResultsArray) - { - return false; - } - CbFieldViewIterator It = ResultsArray.CreateViewIterator(); - while (It.HasValue()) - { - Success.push_back(It.AsBool()); - It++; - } - return true; - } - - bool PutCacheValuesResult::Format(CbPackage& OutPackage) const - { - if (Success.empty()) - { - return false; - } - CbObjectWriter ResponseObject; - ResponseObject.BeginArray("Result"); - for (bool Value : Success) - { - ResponseObject.AddBool(Value); - } - ResponseObject.EndArray(); - - OutPackage.SetObject(ResponseObject.Save()); - return true; - } - - bool GetCacheValuesRequest::Parse(const CbObjectView& BatchObject) - { - ZEN_ASSERT(BatchObject["Method"].AsString() == "GetCacheValues"); - AcceptMagic = BatchObject["AcceptType"].AsUInt32(0); - AcceptOptions = BatchObject["AcceptFlags"].AsUInt16(0); - ProcessPid = BatchObject["Pid"].AsInt32(0); - - CbObjectView Params = BatchObject["Params"].AsObjectView(); - std::optional<std::string> RequestNamespace = cacherequests::GetRequestNamespace(Params); - if (!RequestNamespace) - { - return false; - } - - Namespace = *RequestNamespace; - DefaultPolicy = GetCachePolicy(Params, "DefaultPolicy").value_or(CachePolicy::Default); - - CbArrayView RequestsArray = Params["Requests"].AsArrayView(); - Requests.reserve(RequestsArray.Num()); - for (CbFieldView RequestField : RequestsArray) - { - CbObjectView RequestObject = RequestField.AsObjectView(); - CbObjectView KeyObject = RequestObject["Key"].AsObjectView(); - - GetCacheValueRequest& Request = Requests.emplace_back(); - - if (!GetRequestCacheKey(KeyObject, Request.Key)) - { - return false; - } - - Request.Policy = GetCachePolicy(RequestObject, "Policy"); - } - return true; - } - - bool GetCacheValuesRequest::Format(CbPackage& OutPackage, const std::span<const size_t> OptionalValueFilter) const - { - CbObjectWriter Writer; - Writer << "Method" - << "GetCacheValues"; - if (AcceptMagic != 0) - { - Writer << "Accept" << AcceptMagic; - } - if (AcceptOptions != 0) - { - Writer << "AcceptFlags" << AcceptOptions; - } - if (ProcessPid != 0) - { - Writer << "Pid" << ProcessPid; - } - - Writer.BeginObject("Params"); - { - Writer << "DefaultPolicy" << WriteToString<128>(DefaultPolicy); - Writer << "Namespace" << Namespace; - - Writer.BeginArray("Requests"); - if (OptionalValueFilter.empty()) - { - for (const GetCacheValueRequest& ValueRequest : Requests) - { - Writer.BeginObject(); - { - WriteCacheRequestKey(Writer, ValueRequest.Key); - WriteCachePolicy(Writer, "Policy", ValueRequest.Policy); - } - Writer.EndObject(); - } - } - else - { - for (size_t Index : OptionalValueFilter) - { - const GetCacheValueRequest& ValueRequest = Requests[Index]; - Writer.BeginObject(); - { - WriteCacheRequestKey(Writer, ValueRequest.Key); - WriteCachePolicy(Writer, "Policy", ValueRequest.Policy); - } - Writer.EndObject(); - } - } - Writer.EndArray(); - } - Writer.EndObject(); - - OutPackage.SetObject(Writer.Save()); - return true; - } - - bool CacheValuesResult::Parse(const CbPackage& Package, const std::span<const size_t> OptionalValueResultIndexes) - { - CbObject ResponseObject = Package.GetObject(); - CbArrayView ResultsArray = ResponseObject["Result"].AsArrayView(); - if (!ResultsArray) - { - return false; - } - Results.reserve(ResultsArray.Num()); - if (!OptionalValueResultIndexes.empty() && ResultsArray.Num() != OptionalValueResultIndexes.size()) - { - return false; - } - for (size_t Index = 0; CbFieldView RecordView : ResultsArray) - { - size_t ResultIndex = OptionalValueResultIndexes.empty() ? Index : OptionalValueResultIndexes[Index]; - Index++; - - if (Results.size() <= ResultIndex) - { - Results.resize(ResultIndex + 1); - } - if (RecordView.IsNull()) - { - continue; - } - - CacheValueResult& ValueResult = Results[ResultIndex]; - CbObjectView RecordObject = RecordView.AsObjectView(); - - CbFieldView RawHashField = RecordObject["RawHash"]; - ValueResult.RawHash = RawHashField.AsHash(); - bool Succeeded = !RawHashField.HasError(); - if (Succeeded) - { - const CbAttachment* Attachment = Package.FindAttachment(ValueResult.RawHash); - ValueResult.Body = Attachment ? Attachment->AsCompressedBinary() : CompressedBuffer(); - if (ValueResult.Body) - { - ValueResult.RawSize = ValueResult.Body.DecodeRawSize(); - } - else - { - ValueResult.RawSize = RecordObject["RawSize"].AsUInt64(UINT64_MAX); - } - } - } - return true; - } - - bool CacheValuesResult::Format(CbPackage& OutPackage) const - { - CbObjectWriter ResponseObject; - - ResponseObject.BeginArray("Result"); - for (const CacheValueResult& ValueResult : Results) - { - ResponseObject.BeginObject(); - if (ValueResult.RawHash != IoHash::Zero) - { - ResponseObject.AddHash("RawHash", ValueResult.RawHash); - if (ValueResult.Body) - { - OutPackage.AddAttachment(CbAttachment(ValueResult.Body, ValueResult.RawHash)); - } - else - { - ResponseObject.AddInteger("RawSize", ValueResult.RawSize); - } - } - ResponseObject.EndObject(); - } - ResponseObject.EndArray(); - - OutPackage.SetObject(ResponseObject.Save()); - return true; - } - - bool GetCacheChunksRequest::Parse(const CbObjectView& BatchObject) - { - ZEN_ASSERT(BatchObject["Method"].AsString() == "GetCacheChunks"); - AcceptMagic = BatchObject["AcceptType"].AsUInt32(0); - AcceptOptions = BatchObject["AcceptFlags"].AsUInt16(0); - ProcessPid = BatchObject["Pid"].AsInt32(0); - - CbObjectView Params = BatchObject["Params"].AsObjectView(); - std::optional<std::string> RequestNamespace = cacherequests::GetRequestNamespace(Params); - if (!RequestNamespace) - { - return false; - } - - Namespace = *RequestNamespace; - DefaultPolicy = GetCachePolicy(Params, "DefaultPolicy").value_or(CachePolicy::Default); - - CbArrayView RequestsArray = Params["ChunkRequests"].AsArrayView(); - Requests.reserve(RequestsArray.Num()); - for (CbFieldView RequestField : RequestsArray) - { - CbObjectView RequestObject = RequestField.AsObjectView(); - CbObjectView KeyObject = RequestObject["Key"].AsObjectView(); - - GetCacheChunkRequest& Request = Requests.emplace_back(); - - if (!GetRequestCacheKey(KeyObject, Request.Key)) - { - return false; - } - - Request.ValueId = RequestObject["ValueId"].AsObjectId(); - Request.ChunkId = RequestObject["ChunkId"].AsHash(); - Request.RawOffset = RequestObject["RawOffset"].AsUInt64(); - Request.RawSize = RequestObject["RawSize"].AsUInt64(UINT64_MAX); - - Request.Policy = GetCachePolicy(RequestObject, "Policy"); - } - return true; - } - - bool GetCacheChunksRequest::Format(CbPackage& OutPackage) const - { - CbObjectWriter Writer; - Writer << "Method" - << "GetCacheChunks"; - if (AcceptMagic != 0) - { - Writer << "Accept" << AcceptMagic; - } - if (AcceptOptions != 0) - { - Writer << "AcceptFlags" << AcceptOptions; - } - if (ProcessPid != 0) - { - Writer << "Pid" << ProcessPid; - } - - Writer.BeginObject("Params"); - { - Writer << "DefaultPolicy" << WriteToString<128>(DefaultPolicy); - Writer << "Namespace" << Namespace; - - Writer.BeginArray("ChunkRequests"); - for (const GetCacheChunkRequest& ValueRequest : Requests) - { - Writer.BeginObject(); - { - WriteCacheRequestKey(Writer, ValueRequest.Key); - - Writer.AddObjectId("ValueId", ValueRequest.ValueId); - Writer.AddHash("ChunkId", ValueRequest.ChunkId); - Writer.AddInteger("RawOffset", ValueRequest.RawOffset); - Writer.AddInteger("RawSize", ValueRequest.RawSize); - - WriteCachePolicy(Writer, "Policy", ValueRequest.Policy); - } - Writer.EndObject(); - } - Writer.EndArray(); - } - Writer.EndObject(); - - OutPackage.SetObject(Writer.Save()); - return true; - } - - bool HttpRequestParseRelativeUri(std::string_view Key, HttpRequestData& Data) - { - std::vector<std::string_view> Tokens; - uint32_t TokenCount = zen::ForEachStrTok(Key, '/', [&](const std::string_view& Token) { - Tokens.push_back(Token); - return true; - }); - - switch (TokenCount) - { - case 1: - Data.Namespace = GetValidNamespaceName(Tokens[0]); - return Data.Namespace.has_value(); - case 2: - { - std::optional<IoHash> PossibleHashKey = GetValidIoHash(Tokens[1]); - if (PossibleHashKey.has_value()) - { - // Legacy bucket/key request - Data.Bucket = GetValidBucketName(Tokens[0]); - if (!Data.Bucket.has_value()) - { - return false; - } - Data.HashKey = PossibleHashKey; - return true; - } - Data.Namespace = GetValidNamespaceName(Tokens[0]); - if (!Data.Namespace.has_value()) - { - return false; - } - Data.Bucket = GetValidBucketName(Tokens[1]); - if (!Data.Bucket.has_value()) - { - return false; - } - return true; - } - case 3: - { - std::optional<IoHash> PossibleHashKey = GetValidIoHash(Tokens[1]); - if (PossibleHashKey.has_value()) - { - // Legacy bucket/key/valueid request - Data.Bucket = GetValidBucketName(Tokens[0]); - if (!Data.Bucket.has_value()) - { - return false; - } - Data.HashKey = PossibleHashKey; - Data.ValueContentId = GetValidIoHash(Tokens[2]); - if (!Data.ValueContentId.has_value()) - { - return false; - } - return true; - } - Data.Namespace = GetValidNamespaceName(Tokens[0]); - if (!Data.Namespace.has_value()) - { - return false; - } - Data.Bucket = GetValidBucketName(Tokens[1]); - if (!Data.Bucket.has_value()) - { - return false; - } - Data.HashKey = GetValidIoHash(Tokens[2]); - if (!Data.HashKey) - { - return false; - } - return true; - } - case 4: - { - Data.Namespace = GetValidNamespaceName(Tokens[0]); - if (!Data.Namespace.has_value()) - { - return false; - } - - Data.Bucket = GetValidBucketName(Tokens[1]); - if (!Data.Bucket.has_value()) - { - return false; - } - - Data.HashKey = GetValidIoHash(Tokens[2]); - if (!Data.HashKey.has_value()) - { - return false; - } - - Data.ValueContentId = GetValidIoHash(Tokens[3]); - if (!Data.ValueContentId.has_value()) - { - return false; - } - return true; - } - default: - return false; - } - } - - // bool CacheRecord::Parse(CbObjectView& Reader) - // { - // CbObjectView KeyView = Reader["Key"].AsObjectView(); - // - // if (!GetRequestCacheKey(KeyView, Key)) - // { - // return false; - // } - // CbArrayView ValuesArray = Reader["Values"].AsArrayView(); - // Values.reserve(ValuesArray.Num()); - // for (CbFieldView Value : ValuesArray) - // { - // CbObjectView ObjectView = Value.AsObjectView(); - // Values.push_back({.Id = ObjectView["Id"].AsObjectId(), - // .RawHash = ObjectView["RawHash"].AsHash(), - // .RawSize = ObjectView["RawSize"].AsUInt64()}); - // } - // return true; - // } - // - // bool CacheRecord::Format(CbObjectWriter& Writer) const - // { - // WriteCacheRequestKey(Writer, Key); - // Writer.BeginArray("Values"); - // for (const CacheRecordValue& Value : Values) - // { - // Writer.BeginObject(); - // { - // Writer.AddObjectId("Id", Value.Id); - // Writer.AddHash("RawHash", Value.RawHash); - // Writer.AddInteger("RawSize", Value.RawSize); - // } - // Writer.EndObject(); - // } - // Writer.EndArray(); - // return true; - // } - -#if ZEN_WITH_TESTS - - static bool operator==(const PutCacheRecordRequestValue& Lhs, const PutCacheRecordRequestValue& Rhs) - { - const IoHash LhsRawHash = Lhs.RawHash != IoHash::Zero ? Lhs.RawHash : Lhs.Body.DecodeRawHash(); - const IoHash RhsRawHash = Rhs.RawHash != IoHash::Zero ? Rhs.RawHash : Rhs.Body.DecodeRawHash(); - return Lhs.Id == Rhs.Id && LhsRawHash == RhsRawHash && - Lhs.Body.GetCompressed().Flatten().GetView().EqualBytes(Rhs.Body.GetCompressed().Flatten().GetView()); - } - - static bool operator==(const zen::CacheValuePolicy& Lhs, const zen::CacheValuePolicy& Rhs) - { - return (Lhs.Id == Rhs.Id) && (Lhs.Policy == Rhs.Policy); - } - - static bool operator==(const std::span<const zen::CacheValuePolicy>& Lhs, const std::span<const zen::CacheValuePolicy>& Rhs) - { - if (Lhs.size() != Lhs.size()) - { - return false; - } - for (size_t Idx = 0; Idx < Lhs.size(); ++Idx) - { - if (Lhs[Idx] != Rhs[Idx]) - { - return false; - } - } - return true; - } - - static bool operator==(const zen::CacheRecordPolicy& Lhs, const zen::CacheRecordPolicy& Rhs) - { - return (Lhs.GetRecordPolicy() == Rhs.GetRecordPolicy()) && (Lhs.GetBasePolicy() == Rhs.GetBasePolicy()) && - (Lhs.GetValuePolicies() == Rhs.GetValuePolicies()); - } - - static bool operator==(const std::optional<CacheRecordPolicy>& Lhs, const std::optional<CacheRecordPolicy>& Rhs) - { - return (Lhs.has_value() == Rhs.has_value()) && (!Lhs || (*Lhs == *Rhs)); - } - - static bool operator==(const PutCacheRecordRequest& Lhs, const PutCacheRecordRequest& Rhs) - { - return (Lhs.Key == Rhs.Key) && (Lhs.Values == Rhs.Values) && (Lhs.Policy == Rhs.Policy); - } - - static bool operator==(const PutCacheRecordsRequest& Lhs, const PutCacheRecordsRequest& Rhs) - { - return (Lhs.DefaultPolicy == Rhs.DefaultPolicy) && (Lhs.Namespace == Rhs.Namespace) && (Lhs.Requests == Rhs.Requests); - } - - static bool operator==(const PutCacheRecordsResult& Lhs, const PutCacheRecordsResult& Rhs) { return (Lhs.Success == Rhs.Success); } - - static bool operator==(const GetCacheRecordRequest& Lhs, const GetCacheRecordRequest& Rhs) - { - return (Lhs.Key == Rhs.Key) && (Lhs.Policy == Rhs.Policy); - } - - static bool operator==(const GetCacheRecordsRequest& Lhs, const GetCacheRecordsRequest& Rhs) - { - return (Lhs.DefaultPolicy == Rhs.DefaultPolicy) && (Lhs.Namespace == Rhs.Namespace) && (Lhs.Requests == Rhs.Requests); - } - - static bool operator==(const GetCacheRecordResultValue& Lhs, const GetCacheRecordResultValue& Rhs) - { - if ((Lhs.Id != Rhs.Id) || (Lhs.RawHash != Rhs.RawHash) || (Lhs.RawSize != Rhs.RawSize)) - { - return false; - } - if (bool(Lhs.Body) != bool(Rhs.Body)) - { - return false; - } - if (bool(Lhs.Body) && Lhs.Body.DecodeRawHash() != Rhs.Body.DecodeRawHash()) - { - return false; - } - return true; - } - - static bool operator==(const GetCacheRecordResult& Lhs, const GetCacheRecordResult& Rhs) - { - return Lhs.Key == Rhs.Key && Lhs.Values == Rhs.Values; - } - - static bool operator==(const std::optional<GetCacheRecordResult>& Lhs, const std::optional<GetCacheRecordResult>& Rhs) - { - if (Lhs.has_value() != Rhs.has_value()) - { - return false; - } - return *Lhs == Rhs; - } - - static bool operator==(const GetCacheRecordsResult& Lhs, const GetCacheRecordsResult& Rhs) { return Lhs.Results == Rhs.Results; } - - static bool operator==(const PutCacheValueRequest& Lhs, const PutCacheValueRequest& Rhs) - { - if ((Lhs.Key != Rhs.Key) || (Lhs.RawHash != Rhs.RawHash)) - { - return false; - } - - if (bool(Lhs.Body) != bool(Rhs.Body)) - { - return false; - } - if (!Lhs.Body) - { - return true; - } - return Lhs.Body.GetCompressed().Flatten().GetView().EqualBytes(Rhs.Body.GetCompressed().Flatten().GetView()); - } - - static bool operator==(const PutCacheValuesRequest& Lhs, const PutCacheValuesRequest& Rhs) - { - return (Lhs.DefaultPolicy == Rhs.DefaultPolicy) && (Lhs.Namespace == Rhs.Namespace) && (Lhs.Requests == Rhs.Requests); - } - - static bool operator==(const PutCacheValuesResult& Lhs, const PutCacheValuesResult& Rhs) { return (Lhs.Success == Rhs.Success); } - - static bool operator==(const GetCacheValueRequest& Lhs, const GetCacheValueRequest& Rhs) - { - return Lhs.Key == Rhs.Key && Lhs.Policy == Rhs.Policy; - } - - static bool operator==(const GetCacheValuesRequest& Lhs, const GetCacheValuesRequest& Rhs) - { - return Lhs.DefaultPolicy == Rhs.DefaultPolicy && Lhs.Namespace == Rhs.Namespace && Lhs.Requests == Rhs.Requests; - } - - static bool operator==(const CacheValueResult& Lhs, const CacheValueResult& Rhs) - { - if (Lhs.RawHash != Rhs.RawHash) - { - return false; - }; - if (Lhs.Body) - { - if (!Rhs.Body) - { - return false; - } - return Lhs.Body.GetCompressed().Flatten().GetView().EqualBytes(Rhs.Body.GetCompressed().Flatten().GetView()); - } - return Lhs.RawSize == Rhs.RawSize; - } - - static bool operator==(const CacheValuesResult& Lhs, const CacheValuesResult& Rhs) { return Lhs.Results == Rhs.Results; } - - static bool operator==(const GetCacheChunkRequest& Lhs, const GetCacheChunkRequest& Rhs) - { - return Lhs.Key == Rhs.Key && Lhs.ValueId == Rhs.ValueId && Lhs.ChunkId == Rhs.ChunkId && Lhs.RawOffset == Rhs.RawOffset && - Lhs.RawSize == Rhs.RawSize && Lhs.Policy == Rhs.Policy; - } - - static bool operator==(const GetCacheChunksRequest& Lhs, const GetCacheChunksRequest& Rhs) - { - return Lhs.DefaultPolicy == Rhs.DefaultPolicy && Lhs.Namespace == Rhs.Namespace && Lhs.Requests == Rhs.Requests; - } - - static CompressedBuffer MakeCompressedBuffer(size_t Size) { return CompressedBuffer::Compress(SharedBuffer(IoBuffer(Size))); }; - - TEST_CASE("cacherequests.put.cache.records") - { - PutCacheRecordsRequest EmptyRequest; - CbPackage EmptyRequestPackage; - CHECK(EmptyRequest.Format(EmptyRequestPackage)); - PutCacheRecordsRequest EmptyRequestCopy; - CHECK(!EmptyRequestCopy.Parse(EmptyRequestPackage)); // Namespace is required - - PutCacheRecordsRequest FullRequest = { - .DefaultPolicy = CachePolicy::Remote, - .Namespace = "the_namespace", - .Requests = {{.Key = {.Bucket = "thebucket", .Hash = IoHash::FromHexString("177030568fdd461bf4fe5ddbf4d463e514e8178e")}, - .Values = {{.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(2134)}, - {.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(213)}, - {.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(7)}}, - .Policy = CachePolicy::StoreLocal}, - {.Key = {.Bucket = "thebucket", .Hash = IoHash::FromHexString("d1df59fcab06793a5f2c372d795bb907a15cab15")}, - .Values = {{.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(1234)}, - {.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(99)}, - {.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(124)}}, - .Policy = CachePolicy::Store}, - {.Key = {.Bucket = "theotherbucket", .Hash = IoHash::FromHexString("e1ce9e1ac8a6f5953dc14c1fa9512b804ed689df")}, - .Values = {{.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(19)}, - {.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(1248)}, - {.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(823)}}}}}; - - CbPackage FullRequestPackage; - CHECK(FullRequest.Format(FullRequestPackage)); - PutCacheRecordsRequest FullRequestCopy; - CHECK(FullRequestCopy.Parse(FullRequestPackage)); - CHECK(FullRequest == FullRequestCopy); - - PutCacheRecordsResult EmptyResult; - CbPackage EmptyResponsePackage; - CHECK(EmptyResult.Format(EmptyResponsePackage)); - PutCacheRecordsResult EmptyResultCopy; - CHECK(!EmptyResultCopy.Parse(EmptyResponsePackage)); - CHECK(EmptyResult == EmptyResultCopy); - - PutCacheRecordsResult FullResult = {.Success = {true, false, true, true, false}}; - CbPackage FullResponsePackage; - CHECK(FullResult.Format(FullResponsePackage)); - PutCacheRecordsResult FullResultCopy; - CHECK(FullResultCopy.Parse(FullResponsePackage)); - CHECK(FullResult == FullResultCopy); - } - - TEST_CASE("cacherequests.get.cache.records") - { - GetCacheRecordsRequest EmptyRequest; - CbPackage EmptyRequestPackage; - CHECK(EmptyRequest.Format(EmptyRequestPackage)); - GetCacheRecordsRequest EmptyRequestCopy; - CHECK(!EmptyRequestCopy.Parse(EmptyRequestPackage)); // Namespace is required - - GetCacheRecordsRequest FullRequest = { - .DefaultPolicy = CachePolicy::StoreLocal, - .Namespace = "other_namespace", - .Requests = {{.Key = {.Bucket = "finebucket", .Hash = IoHash::FromHexString("d1df59fcab06793a5f2c372d795bb907a15cab15")}, - .Policy = CachePolicy::Local}, - {.Key = {.Bucket = "badbucket", .Hash = IoHash::FromHexString("177030568fdd461bf4fe5ddbf4d463e514e8178e")}, - .Policy = CachePolicy::Remote}, - {.Key = {.Bucket = "badbucket", .Hash = IoHash::FromHexString("e1ce9e1ac8a6f5953dc14c1fa9512b804ed689df")}}}}; - - CbPackage FullRequestPackage; - CHECK(FullRequest.Format(FullRequestPackage)); - GetCacheRecordsRequest FullRequestCopy; - CHECK(FullRequestCopy.Parse(FullRequestPackage)); - CHECK(FullRequest == FullRequestCopy); - - CbPackage PartialRequestPackage; - CHECK(FullRequest.Format(PartialRequestPackage, std::initializer_list<size_t>{0, 2})); - GetCacheRecordsRequest PartialRequest = FullRequest; - PartialRequest.Requests.erase(PartialRequest.Requests.begin() + 1); - GetCacheRecordsRequest PartialRequestCopy; - CHECK(PartialRequestCopy.Parse(PartialRequestPackage)); - CHECK(PartialRequest == PartialRequestCopy); - - GetCacheRecordsResult EmptyResult; - CbPackage EmptyResponsePackage; - CHECK(EmptyResult.Format(EmptyResponsePackage)); - GetCacheRecordsResult EmptyResultCopy; - CHECK(!EmptyResultCopy.Parse(EmptyResponsePackage)); - CHECK(EmptyResult == EmptyResultCopy); - - PutCacheRecordsRequest FullPutRequest = { - .DefaultPolicy = CachePolicy::Remote, - .Namespace = "the_namespace", - .Requests = {{.Key = {.Bucket = "thebucket", .Hash = IoHash::FromHexString("177030568fdd461bf4fe5ddbf4d463e514e8178e")}, - .Values = {{.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(2134)}, - {.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(213)}, - {.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(7)}}, - .Policy = CachePolicy::StoreLocal}, - {.Key = {.Bucket = "thebucket", .Hash = IoHash::FromHexString("d1df59fcab06793a5f2c372d795bb907a15cab15")}, - .Values = {{.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(1234)}, - {.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(99)}, - {.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(124)}}, - .Policy = CachePolicy::Store}, - {.Key = {.Bucket = "theotherbucket", .Hash = IoHash::FromHexString("e1ce9e1ac8a6f5953dc14c1fa9512b804ed689df")}, - .Values = {{.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(19)}, - {.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(1248)}, - {.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(823)}}}}}; - - CbPackage FullPutRequestPackage; - CHECK(FullPutRequest.Format(FullPutRequestPackage)); - PutCacheRecordsRequest FullPutRequestCopy; - CHECK(FullPutRequestCopy.Parse(FullPutRequestPackage)); - - GetCacheRecordsResult FullResult = { - {GetCacheRecordResult{.Key = FullPutRequestCopy.Requests[0].Key, - .Values = {{.Id = FullPutRequestCopy.Requests[0].Values[0].Id, - .RawHash = FullPutRequestCopy.Requests[0].Values[0].Body.DecodeRawHash(), - .RawSize = FullPutRequestCopy.Requests[0].Values[0].Body.DecodeRawSize(), - .Body = FullPutRequestCopy.Requests[0].Values[0].Body}, - {.Id = FullPutRequestCopy.Requests[0].Values[1].Id, - - .RawHash = FullPutRequestCopy.Requests[0].Values[1].Body.DecodeRawHash(), - .RawSize = FullPutRequestCopy.Requests[0].Values[1].Body.DecodeRawSize(), - .Body = FullPutRequestCopy.Requests[0].Values[1].Body}, - {.Id = FullPutRequestCopy.Requests[0].Values[2].Id, - .RawHash = FullPutRequestCopy.Requests[0].Values[2].Body.DecodeRawHash(), - .RawSize = FullPutRequestCopy.Requests[0].Values[2].Body.DecodeRawSize(), - .Body = FullPutRequestCopy.Requests[0].Values[2].Body}}}, - {}, // Simulate not have! - GetCacheRecordResult{.Key = FullPutRequestCopy.Requests[2].Key, - .Values = {{.Id = FullPutRequestCopy.Requests[2].Values[0].Id, - .RawHash = FullPutRequestCopy.Requests[2].Values[0].Body.DecodeRawHash(), - .RawSize = FullPutRequestCopy.Requests[2].Values[0].Body.DecodeRawSize(), - .Body = FullPutRequestCopy.Requests[2].Values[0].Body}, - {.Id = FullPutRequestCopy.Requests[2].Values[1].Id, - .RawHash = FullPutRequestCopy.Requests[2].Values[1].Body.DecodeRawHash(), - .RawSize = FullPutRequestCopy.Requests[2].Values[1].Body.DecodeRawSize(), - .Body = {}}, // Simulate not have - {.Id = FullPutRequestCopy.Requests[2].Values[2].Id, - .RawHash = FullPutRequestCopy.Requests[2].Values[2].Body.DecodeRawHash(), - .RawSize = FullPutRequestCopy.Requests[2].Values[2].Body.DecodeRawSize(), - .Body = FullPutRequestCopy.Requests[2].Values[2].Body}}}}}; - CbPackage FullResponsePackage; - CHECK(FullResult.Format(FullResponsePackage)); - GetCacheRecordsResult FullResultCopy; - CHECK(FullResultCopy.Parse(FullResponsePackage)); - CHECK(FullResult.Results[0] == FullResultCopy.Results[0]); - CHECK(!FullResultCopy.Results[1]); - CHECK(FullResult.Results[2] == FullResultCopy.Results[2]); - - GetCacheRecordsResult PartialResultCopy; - CHECK(PartialResultCopy.Parse(FullResponsePackage, std::initializer_list<size_t>{0, 3, 4})); - CHECK(FullResult.Results[0] == PartialResultCopy.Results[0]); - CHECK(!PartialResultCopy.Results[1]); - CHECK(!PartialResultCopy.Results[2]); - CHECK(!PartialResultCopy.Results[3]); - CHECK(FullResult.Results[2] == PartialResultCopy.Results[4]); - } - - TEST_CASE("cacherequests.put.cache.values") - { - PutCacheValuesRequest EmptyRequest; - CbPackage EmptyRequestPackage; - CHECK(EmptyRequest.Format(EmptyRequestPackage)); - PutCacheValuesRequest EmptyRequestCopy; - CHECK(!EmptyRequestCopy.Parse(EmptyRequestPackage)); // Namespace is required - - CompressedBuffer Buffers[3] = {MakeCompressedBuffer(969), MakeCompressedBuffer(3469), MakeCompressedBuffer(9)}; - PutCacheValuesRequest FullRequest = { - .DefaultPolicy = CachePolicy::StoreLocal, - .Namespace = "other_namespace", - .Requests = {{.Key = {.Bucket = "finebucket", .Hash = IoHash::FromHexString("d1df59fcab06793a5f2c372d795bb907a15cab15")}, - .RawHash = Buffers[0].DecodeRawHash(), - .Body = Buffers[0], - .Policy = CachePolicy::Local}, - {.Key = {.Bucket = "badbucket", .Hash = IoHash::FromHexString("177030568fdd461bf4fe5ddbf4d463e514e8178e")}, - .RawHash = Buffers[1].DecodeRawHash(), - .Body = Buffers[1], - .Policy = CachePolicy::Remote}, - {.Key = {.Bucket = "badbucket", .Hash = IoHash::FromHexString("e1ce9e1ac8a6f5953dc14c1fa9512b804ed689df")}, - .RawHash = Buffers[2].DecodeRawHash()}}}; - - CbPackage FullRequestPackage; - CHECK(FullRequest.Format(FullRequestPackage)); - PutCacheValuesRequest FullRequestCopy; - CHECK(FullRequestCopy.Parse(FullRequestPackage)); - CHECK(FullRequest == FullRequestCopy); - - PutCacheValuesResult EmptyResult; - CbPackage EmptyResponsePackage; - CHECK(!EmptyResult.Format(EmptyResponsePackage)); - - PutCacheValuesResult FullResult = {.Success = {true, false, true}}; - - CbPackage FullResponsePackage; - CHECK(FullResult.Format(FullResponsePackage)); - PutCacheValuesResult FullResultCopy; - CHECK(FullResultCopy.Parse(FullResponsePackage)); - CHECK(FullResult == FullResultCopy); - } - - TEST_CASE("cacherequests.get.cache.values") - { - GetCacheValuesRequest EmptyRequest; - CbPackage EmptyRequestPackage; - CHECK(EmptyRequest.Format(EmptyRequestPackage)); - GetCacheValuesRequest EmptyRequestCopy; - CHECK(!EmptyRequestCopy.Parse(EmptyRequestPackage.GetObject())); // Namespace is required - - GetCacheValuesRequest FullRequest = { - .DefaultPolicy = CachePolicy::StoreLocal, - .Namespace = "other_namespace", - .Requests = {{.Key = {.Bucket = "finebucket", .Hash = IoHash::FromHexString("d1df59fcab06793a5f2c372d795bb907a15cab15")}, - .Policy = CachePolicy::Local}, - {.Key = {.Bucket = "badbucket", .Hash = IoHash::FromHexString("177030568fdd461bf4fe5ddbf4d463e514e8178e")}, - .Policy = CachePolicy::Remote}, - {.Key = {.Bucket = "badbucket", .Hash = IoHash::FromHexString("e1ce9e1ac8a6f5953dc14c1fa9512b804ed689df")}}}}; - - CbPackage FullRequestPackage; - CHECK(FullRequest.Format(FullRequestPackage)); - GetCacheValuesRequest FullRequestCopy; - CHECK(FullRequestCopy.Parse(FullRequestPackage.GetObject())); - CHECK(FullRequest == FullRequestCopy); - - CbPackage PartialRequestPackage; - CHECK(FullRequest.Format(PartialRequestPackage, std::initializer_list<size_t>{0, 2})); - GetCacheValuesRequest PartialRequest = FullRequest; - PartialRequest.Requests.erase(PartialRequest.Requests.begin() + 1); - GetCacheValuesRequest PartialRequestCopy; - CHECK(PartialRequestCopy.Parse(PartialRequestPackage.GetObject())); - CHECK(PartialRequest == PartialRequestCopy); - - CacheValuesResult EmptyResult; - CbPackage EmptyResponsePackage; - CHECK(EmptyResult.Format(EmptyResponsePackage)); - CacheValuesResult EmptyResultCopy; - CHECK(!EmptyResultCopy.Parse(EmptyResponsePackage)); - CHECK(EmptyResult == EmptyResultCopy); - - CompressedBuffer Buffers[3][3] = {{MakeCompressedBuffer(123), MakeCompressedBuffer(321), MakeCompressedBuffer(333)}, - {MakeCompressedBuffer(6123), MakeCompressedBuffer(8321), MakeCompressedBuffer(7333)}, - {MakeCompressedBuffer(5123), MakeCompressedBuffer(2321), MakeCompressedBuffer(2333)}}; - CacheValuesResult FullResult = { - .Results = {CacheValueResult{.RawSize = 0, .RawHash = Buffers[0][0].DecodeRawHash(), .Body = Buffers[0][0]}, - CacheValueResult{.RawSize = 0, .RawHash = Buffers[0][1].DecodeRawHash(), .Body = Buffers[0][1]}, - CacheValueResult{.RawSize = 0, .RawHash = Buffers[0][2].DecodeRawHash(), .Body = Buffers[0][2]}, - CacheValueResult{.RawSize = 0, .RawHash = Buffers[2][0].DecodeRawHash(), .Body = Buffers[2][0]}, - CacheValueResult{.RawSize = 0, .RawHash = Buffers[2][1].DecodeRawHash(), .Body = Buffers[2][1]}, - CacheValueResult{.RawSize = Buffers[2][2].DecodeRawSize(), .RawHash = Buffers[2][2].DecodeRawHash()}}}; - CbPackage FullResponsePackage; - CHECK(FullResult.Format(FullResponsePackage)); - CacheValuesResult FullResultCopy; - CHECK(FullResultCopy.Parse(FullResponsePackage)); - CHECK(FullResult == FullResultCopy); - - CacheValuesResult PartialResultCopy; - CHECK(PartialResultCopy.Parse(FullResponsePackage, std::initializer_list<size_t>{0, 3, 4, 5, 6, 9})); - CHECK(PartialResultCopy.Results[0] == FullResult.Results[0]); - CHECK(PartialResultCopy.Results[1].RawHash == IoHash::Zero); - CHECK(PartialResultCopy.Results[2].RawHash == IoHash::Zero); - CHECK(PartialResultCopy.Results[3] == FullResult.Results[1]); - CHECK(PartialResultCopy.Results[4] == FullResult.Results[2]); - CHECK(PartialResultCopy.Results[5] == FullResult.Results[3]); - CHECK(PartialResultCopy.Results[6] == FullResult.Results[4]); - CHECK(PartialResultCopy.Results[7].RawHash == IoHash::Zero); - CHECK(PartialResultCopy.Results[8].RawHash == IoHash::Zero); - CHECK(PartialResultCopy.Results[9] == FullResult.Results[5]); - } - - TEST_CASE("cacherequests.get.cache.chunks") - { - GetCacheChunksRequest EmptyRequest; - CbPackage EmptyRequestPackage; - CHECK(EmptyRequest.Format(EmptyRequestPackage)); - GetCacheChunksRequest EmptyRequestCopy; - CHECK(!EmptyRequestCopy.Parse(EmptyRequestPackage.GetObject())); // Namespace is required - - GetCacheChunksRequest FullRequest = { - .DefaultPolicy = CachePolicy::StoreLocal, - .Namespace = "other_namespace", - .Requests = {{.Key = {.Bucket = "finebucket", .Hash = IoHash::FromHexString("d1df59fcab06793a5f2c372d795bb907a15cab15")}, - .ValueId = Oid::NewOid(), - .ChunkId = IoHash::FromHexString("ab3917854bfef7e7af2c372d795bb907a15cab15"), - .RawOffset = 77, - .RawSize = 33, - .Policy = CachePolicy::Local}, - {.Key = {.Bucket = "badbucket", .Hash = IoHash::FromHexString("177030568fdd461bf4fe5ddbf4d463e514e8178e")}, - .ValueId = Oid::NewOid(), - .ChunkId = IoHash::FromHexString("372d795bb907a15cab15ab3917854bfef7e7af2c"), - .Policy = CachePolicy::Remote}, - { - .Key = {.Bucket = "badbucket", .Hash = IoHash::FromHexString("e1ce9e1ac8a6f5953dc14c1fa9512b804ed689df")}, - .ChunkId = IoHash::FromHexString("372d795bb907a15cab15ab3917854bfef7e7af2c"), - }}}; - - CbPackage FullRequestPackage; - CHECK(FullRequest.Format(FullRequestPackage)); - GetCacheChunksRequest FullRequestCopy; - CHECK(FullRequestCopy.Parse(FullRequestPackage.GetObject())); - CHECK(FullRequest == FullRequestCopy); - - GetCacheChunksResult EmptyResult; - CbPackage EmptyResponsePackage; - CHECK(EmptyResult.Format(EmptyResponsePackage)); - GetCacheChunksResult EmptyResultCopy; - CHECK(!EmptyResultCopy.Parse(EmptyResponsePackage)); - CHECK(EmptyResult == EmptyResultCopy); - - CompressedBuffer Buffers[3][3] = {{MakeCompressedBuffer(123), MakeCompressedBuffer(321), MakeCompressedBuffer(333)}, - {MakeCompressedBuffer(6123), MakeCompressedBuffer(8321), MakeCompressedBuffer(7333)}, - {MakeCompressedBuffer(5123), MakeCompressedBuffer(2321), MakeCompressedBuffer(2333)}}; - GetCacheChunksResult FullResult = { - .Results = {CacheValueResult{.RawSize = 0, .RawHash = Buffers[0][0].DecodeRawHash(), .Body = Buffers[0][0]}, - CacheValueResult{.RawSize = 0, .RawHash = Buffers[0][1].DecodeRawHash(), .Body = Buffers[0][1]}, - CacheValueResult{.RawSize = 0, .RawHash = Buffers[0][2].DecodeRawHash(), .Body = Buffers[0][2]}, - CacheValueResult{.RawSize = 0, .RawHash = Buffers[2][0].DecodeRawHash(), .Body = Buffers[2][0]}, - CacheValueResult{.RawSize = 0, .RawHash = Buffers[2][1].DecodeRawHash(), .Body = Buffers[2][1]}, - CacheValueResult{.RawSize = Buffers[2][2].DecodeRawSize(), .RawHash = Buffers[2][2].DecodeRawHash()}}}; - CbPackage FullResponsePackage; - CHECK(FullResult.Format(FullResponsePackage)); - GetCacheChunksResult FullResultCopy; - CHECK(FullResultCopy.Parse(FullResponsePackage)); - CHECK(FullResult == FullResultCopy); - } - - TEST_CASE("z$service.parse.relative.Uri") - { - HttpRequestData LegacyBucketRequestBecomesNamespaceRequest; - CHECK(HttpRequestParseRelativeUri("test", LegacyBucketRequestBecomesNamespaceRequest)); - CHECK(LegacyBucketRequestBecomesNamespaceRequest.Namespace == "test"); - CHECK(!LegacyBucketRequestBecomesNamespaceRequest.Bucket.has_value()); - CHECK(!LegacyBucketRequestBecomesNamespaceRequest.HashKey.has_value()); - CHECK(!LegacyBucketRequestBecomesNamespaceRequest.ValueContentId.has_value()); - - HttpRequestData LegacyHashKeyRequest; - CHECK(HttpRequestParseRelativeUri("test/0123456789abcdef12340123456789abcdef1234", LegacyHashKeyRequest)); - CHECK(!LegacyHashKeyRequest.Namespace); - CHECK(LegacyHashKeyRequest.Bucket == "test"); - CHECK(LegacyHashKeyRequest.HashKey == IoHash::FromHexString("0123456789abcdef12340123456789abcdef1234")); - CHECK(!LegacyHashKeyRequest.ValueContentId.has_value()); - - HttpRequestData LegacyValueContentIdRequest; - CHECK(HttpRequestParseRelativeUri("test/0123456789abcdef12340123456789abcdef1234/56789abcdef12345678956789abcdef123456789", - LegacyValueContentIdRequest)); - CHECK(!LegacyValueContentIdRequest.Namespace); - CHECK(LegacyValueContentIdRequest.Bucket == "test"); - CHECK(LegacyValueContentIdRequest.HashKey == IoHash::FromHexString("0123456789abcdef12340123456789abcdef1234")); - CHECK(LegacyValueContentIdRequest.ValueContentId == IoHash::FromHexString("56789abcdef12345678956789abcdef123456789")); - - HttpRequestData V2DefaultNamespaceRequest; - CHECK(HttpRequestParseRelativeUri("ue4.ddc", V2DefaultNamespaceRequest)); - CHECK(V2DefaultNamespaceRequest.Namespace == "ue4.ddc"); - CHECK(!V2DefaultNamespaceRequest.Bucket.has_value()); - CHECK(!V2DefaultNamespaceRequest.HashKey.has_value()); - CHECK(!V2DefaultNamespaceRequest.ValueContentId.has_value()); - - HttpRequestData V2NamespaceRequest; - CHECK(HttpRequestParseRelativeUri("nicenamespace", V2NamespaceRequest)); - CHECK(V2NamespaceRequest.Namespace == "nicenamespace"); - CHECK(!V2NamespaceRequest.Bucket.has_value()); - CHECK(!V2NamespaceRequest.HashKey.has_value()); - CHECK(!V2NamespaceRequest.ValueContentId.has_value()); - - HttpRequestData V2BucketRequestWithDefaultNamespace; - CHECK(HttpRequestParseRelativeUri("ue4.ddc/test", V2BucketRequestWithDefaultNamespace)); - CHECK(V2BucketRequestWithDefaultNamespace.Namespace == "ue4.ddc"); - CHECK(V2BucketRequestWithDefaultNamespace.Bucket == "test"); - CHECK(!V2BucketRequestWithDefaultNamespace.HashKey.has_value()); - CHECK(!V2BucketRequestWithDefaultNamespace.ValueContentId.has_value()); - - HttpRequestData V2BucketRequestWithNamespace; - CHECK(HttpRequestParseRelativeUri("nicenamespace/test", V2BucketRequestWithNamespace)); - CHECK(V2BucketRequestWithNamespace.Namespace == "nicenamespace"); - CHECK(V2BucketRequestWithNamespace.Bucket == "test"); - CHECK(!V2BucketRequestWithNamespace.HashKey.has_value()); - CHECK(!V2BucketRequestWithNamespace.ValueContentId.has_value()); - - HttpRequestData V2HashKeyRequest; - CHECK(HttpRequestParseRelativeUri("test/0123456789abcdef12340123456789abcdef1234", V2HashKeyRequest)); - CHECK(!V2HashKeyRequest.Namespace); - CHECK(V2HashKeyRequest.Bucket == "test"); - CHECK(V2HashKeyRequest.HashKey == IoHash::FromHexString("0123456789abcdef12340123456789abcdef1234")); - CHECK(!V2HashKeyRequest.ValueContentId.has_value()); - - HttpRequestData V2ValueContentIdRequest; - CHECK(HttpRequestParseRelativeUri( - "nicenamespace/test/0123456789abcdef12340123456789abcdef1234/56789abcdef12345678956789abcdef123456789", - V2ValueContentIdRequest)); - CHECK(V2ValueContentIdRequest.Namespace == "nicenamespace"); - CHECK(V2ValueContentIdRequest.Bucket == "test"); - CHECK(V2ValueContentIdRequest.HashKey == IoHash::FromHexString("0123456789abcdef12340123456789abcdef1234")); - CHECK(V2ValueContentIdRequest.ValueContentId == IoHash::FromHexString("56789abcdef12345678956789abcdef123456789")); - - HttpRequestData Invalid; - CHECK(!HttpRequestParseRelativeUri("", Invalid)); - CHECK(!HttpRequestParseRelativeUri("/", Invalid)); - CHECK(!HttpRequestParseRelativeUri("bad\2_namespace", Invalid)); - CHECK(!HttpRequestParseRelativeUri("nice/\2\1bucket", Invalid)); - CHECK(!HttpRequestParseRelativeUri("namespace/bucket/0123456789a", Invalid)); - CHECK(!HttpRequestParseRelativeUri("namespace/bucket/0123456789abcdef12340123456789abcdef1234/56789abcdef1234", Invalid)); - CHECK(!HttpRequestParseRelativeUri("namespace/bucket/pppppppp89abcdef12340123456789abcdef1234", Invalid)); - CHECK(!HttpRequestParseRelativeUri("namespace/bucket/0123456789abcdef12340123456789abcdef1234/56789abcd", Invalid)); - CHECK(!HttpRequestParseRelativeUri( - "namespace/bucket/0123456789abcdef12340123456789abcdef1234/ppppppppdef12345678956789abcdef123456789", - Invalid)); - } -#endif -} // namespace cacherequests - -void -cacherequests_forcelink() -{ -} - -} // namespace zen |