diff options
| author | Dan Engelbrecht <[email protected]> | 2022-06-27 06:40:30 -0700 |
|---|---|---|
| committer | GitHub <[email protected]> | 2022-06-27 06:40:30 -0700 |
| commit | dd7c5d5f6eec62a76b01ab4dd80e6d710a7a1167 (patch) | |
| tree | e0bf24635cbe3170151570804b068521cc8e860f /zenutil/cache/cacherequests.cpp | |
| parent | add macos builds (#133) (diff) | |
| download | zen-dd7c5d5f6eec62a76b01ab4dd80e6d710a7a1167.tar.xz zen-dd7c5d5f6eec62a76b01ab4dd80e6d710a7a1167.zip | |
cache requests API (#134)
Diffstat (limited to 'zenutil/cache/cacherequests.cpp')
| -rw-r--r-- | zenutil/cache/cacherequests.cpp | 1495 |
1 files changed, 1495 insertions, 0 deletions
diff --git a/zenutil/cache/cacherequests.cpp b/zenutil/cache/cacherequests.cpp new file mode 100644 index 000000000..fe3b63dc1 --- /dev/null +++ b/zenutil/cache/cacherequests.cpp @@ -0,0 +1,1495 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <zenutil/cache/cacherequests.h> + +#include <zencore/compactbinary.h> +#include <zencore/compactbinarybuilder.h> +#include <zencore/compactbinarypackage.h> +#include <zencore/fmtutils.h> +#include <zencore/logging.h> +#include <zencore/zencore.h> + +#include <string> +#include <string_view> + +#if ZEN_WITH_TESTS +# include <zencore/testing.h> +#endif + +namespace zen { + +namespace cacherequests { + + namespace { + constinit AsciiSet ValidNamespaceNameCharactersSet{"abcdefghijklmnopqrstuvwxyz0123456789-_.ABCDEFGHIJKLMNOPQRSTUVWXYZ"}; + constinit AsciiSet ValidBucketNameCharactersSet{"abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"}; + + std::optional<std::string> GetValidNamespaceName(std::string_view Name) + { + if (Name.empty()) + { + ZEN_WARN("Namespace is invalid, empty namespace is not allowed"); + return {}; + } + + if (Name.length() > 64) + { + ZEN_WARN("Namespace '{}' is invalid, length exceeds 64 characters", Name); + return {}; + } + + if (!AsciiSet::HasOnly(Name, ValidNamespaceNameCharactersSet)) + { + ZEN_WARN("Namespace '{}' is invalid, invalid characters detected", Name); + return {}; + } + + return ToLower(Name); + } + + std::optional<std::string> GetValidBucketName(std::string_view Name) + { + if (Name.empty()) + { + ZEN_WARN("Bucket name is invalid, empty bucket name is not allowed"); + return {}; + } + + if (!AsciiSet::HasOnly(Name, ValidBucketNameCharactersSet)) + { + ZEN_WARN("Bucket name '{}' is invalid, invalid characters detected", Name); + return {}; + } + + return ToLower(Name); + } + + std::optional<IoHash> GetValidIoHash(std::string_view Hash) + { + if (Hash.length() != IoHash::StringLength) + { + return {}; + } + + IoHash KeyHash; + if (!ParseHexBytes(Hash.data(), Hash.size(), KeyHash.Hash)) + { + return {}; + } + return KeyHash; + } + + std::optional<CacheRecordPolicy> Convert(const OptionalCacheRecordPolicy& Policy) + { + return Policy.IsValid() ? Policy.Get() : std::optional<CacheRecordPolicy>{}; + }; + } // namespace + + std::optional<std::string> GetRequestNamespace(const CbObjectView& Params) + { + CbFieldView NamespaceField = Params["Namespace"]; + if (!NamespaceField) + { + return std::string("!default!"); // ZenCacheStore::DefaultNamespace); + } + + if (NamespaceField.HasError()) + { + return {}; + } + if (!NamespaceField.IsString()) + { + return {}; + } + return GetValidNamespaceName(NamespaceField.AsString()); + } + + bool GetRequestCacheKey(const CbObjectView& KeyView, CacheKey& Key) + { + CbFieldView BucketField = KeyView["Bucket"]; + if (BucketField.HasError()) + { + return false; + } + if (!BucketField.IsString()) + { + return false; + } + std::optional<std::string> Bucket = GetValidBucketName(BucketField.AsString()); + if (!Bucket.has_value()) + { + return false; + } + CbFieldView HashField = KeyView["Hash"]; + if (HashField.HasError()) + { + return false; + } + if (!HashField.IsHash()) + { + return false; + } + Key.Bucket = *Bucket; + Key.Hash = HashField.AsHash(); + return true; + } + + void WriteCacheRequestKey(CbObjectWriter& Writer, const CacheKey& Value) + { + Writer.BeginObject("Key"); + { + Writer << "Bucket" << Value.Bucket; + Writer << "Hash" << Value.Hash; + } + Writer.EndObject(); + } + + void WriteOptionalCacheRequestPolicy(CbObjectWriter& Writer, std::string_view FieldName, const std::optional<CacheRecordPolicy>& Policy) + { + if (Policy) + { + Writer.SetName(FieldName); + Policy->Save(Writer); + } + } + + std::optional<CachePolicy> GetCachePolicy(CbObjectView ObjectView, std::string_view FieldName) + { + std::string_view DefaultPolicyText = ObjectView[FieldName].AsString(); + if (DefaultPolicyText.empty()) + { + return {}; + } + return ParseCachePolicy(DefaultPolicyText); + } + + void WriteCachePolicy(CbObjectWriter& Writer, std::string_view FieldName, const std::optional<CachePolicy>& Policy) + { + if (Policy) + { + Writer << FieldName << WriteToString<128>(*Policy); + } + } + + bool PutCacheRecordsRequest::Parse(const CbPackage& Package) + { + CbObjectView BatchObject = Package.GetObject(); + ZEN_ASSERT(BatchObject["Method"].AsString() == "PutCacheRecords"); + + CbObjectView Params = BatchObject["Params"].AsObjectView(); + std::optional<std::string> RequestNamespace = GetRequestNamespace(Params); + if (!RequestNamespace) + { + return false; + } + Namespace = *RequestNamespace; + DefaultPolicy = GetCachePolicy(Params, "DefaultPolicy").value_or(CachePolicy::Default); + + CbArrayView RequestFieldArray = Params["Requests"].AsArrayView(); + Requests.resize(RequestFieldArray.Num()); + for (size_t RequestIndex = 0; CbFieldView RequestField : RequestFieldArray) + { + CbObjectView RequestObject = RequestField.AsObjectView(); + CbObjectView RecordObject = RequestObject["Record"].AsObjectView(); + CbObjectView KeyView = RecordObject["Key"].AsObjectView(); + + PutCacheRecordRequest& Request = Requests[RequestIndex++]; + + if (!GetRequestCacheKey(KeyView, Request.Key)) + { + return false; + } + + Request.Policy = Convert(CacheRecordPolicy::Load(RequestObject["Policy"].AsObjectView())); + + std::unordered_map<IoHash, size_t, IoHash::Hasher> RawHashToAttachmentIndex; + + CbArrayView ValuesArray = RecordObject["Values"].AsArrayView(); + Request.Values.resize(ValuesArray.Num()); + RawHashToAttachmentIndex.reserve(ValuesArray.Num()); + for (size_t Index = 0; CbFieldView Value : ValuesArray) + { + CbObjectView ObjectView = Value.AsObjectView(); + IoHash AttachmentHash = ObjectView["RawHash"].AsHash(); + RawHashToAttachmentIndex[AttachmentHash] = Index; + Request.Values[Index++] = {.Id = ObjectView["Id"].AsObjectId(), .RawHash = AttachmentHash}; + } + + RecordObject.IterateAttachments([&](CbFieldView HashView) { + const IoHash ValueHash = HashView.AsHash(); + if (const CbAttachment* Attachment = Package.FindAttachment(ValueHash)) + { + if (Attachment->IsCompressedBinary()) + { + auto It = RawHashToAttachmentIndex.find(ValueHash); + ZEN_ASSERT(It != RawHashToAttachmentIndex.end()); + PutCacheRecordRequestValue& Value = Request.Values[It->second]; + ZEN_ASSERT(Value.RawHash == ValueHash); + Value.Body = Attachment->AsCompressedBinary(); + ZEN_ASSERT(IoHash::FromBLAKE3(Value.Body.GetRawHash()) == Value.RawHash); + } + } + }); + Request.RecordBody = [RecordObject]() { + IoBuffer RecordBody = IoBuffer(RecordObject.GetSize()); + RecordObject.CopyTo(RecordBody.GetMutableView()); + RecordBody.SetContentType(ZenContentType::kCbObject); + return RecordBody; + }; + } + + return true; + } + + bool PutCacheRecordsRequest::Format(CbPackage& OutPackage) const + { + CbObjectWriter Writer; + Writer << "Method" + << "PutCacheRecords"; + Writer.BeginObject("Params"); + { + Writer << "DefaultPolicy" << WriteToString<128>(DefaultPolicy); + Writer << "Namespace" << Namespace; + + Writer.BeginArray("Requests"); + for (const PutCacheRecordRequest& RecordRequest : Requests) + { + Writer.BeginObject(); + { + Writer.BeginObject("Record"); + { + WriteCacheRequestKey(Writer, RecordRequest.Key); + Writer.BeginArray("Values"); + for (const PutCacheRecordRequestValue& Value : RecordRequest.Values) + { + Writer.BeginObject(); + { + Writer.AddObjectId("Id", Value.Id); + const CompressedBuffer& Buffer = Value.Body; + if (Buffer) + { + Writer.AddBinaryAttachment("RawHash", IoHash::FromBLAKE3(Buffer.GetRawHash())); + OutPackage.AddAttachment(CbAttachment(Buffer)); + Writer.AddInteger("RawSize", Buffer.GetRawSize()); + } + else + { + if (Value.RawHash == IoHash::Zero) + { + return false; + } + Writer.AddBinaryAttachment("RawHash", Value.RawHash); + } + } + Writer.EndObject(); + } + Writer.EndArray(); + } + Writer.EndObject(); + WriteOptionalCacheRequestPolicy(Writer, "Policy", RecordRequest.Policy); + } + Writer.EndObject(); + } + Writer.EndArray(); + } + Writer.EndObject(); + OutPackage.SetObject(Writer.Save()); + + return true; + } + + bool PutCacheRecordsResult::Parse(const CbPackage& Package) + { + CbArrayView ResultsArray = Package.GetObject()["Result"].AsArrayView(); + if (!ResultsArray) + { + return false; + } + CbFieldViewIterator It = ResultsArray.CreateViewIterator(); + while (It.HasValue()) + { + Success.push_back(It.AsBool()); + It++; + } + return true; + } + + bool PutCacheRecordsResult::Format(CbPackage& OutPackage) const + { + CbObjectWriter ResponseObject; + ResponseObject.BeginArray("Result"); + for (bool Value : Success) + { + ResponseObject.AddBool(Value); + } + ResponseObject.EndArray(); + + OutPackage.SetObject(ResponseObject.Save()); + return true; + } + + bool GetCacheRecordsRequest::Parse(const CbObjectView& RpcRequest) + { + ZEN_ASSERT(RpcRequest["Method"].AsString() == "GetCacheRecords"); + + CbObjectView Params = RpcRequest["Params"].AsObjectView(); + std::optional<std::string> RequestNamespace = GetRequestNamespace(Params); + if (!RequestNamespace) + { + return false; + } + + Namespace = *RequestNamespace; + DefaultPolicy = GetCachePolicy(Params, "DefaultPolicy").value_or(CachePolicy::Default); + + CbArrayView RequestsArray = Params["Requests"].AsArrayView(); + Requests.reserve(RequestsArray.Num()); + for (CbFieldView RequestField : RequestsArray) + { + CbObjectView RequestObject = RequestField.AsObjectView(); + CbObjectView KeyObject = RequestObject["Key"].AsObjectView(); + + GetCacheRecordRequest& Request = Requests.emplace_back(); + + if (!GetRequestCacheKey(KeyObject, Request.Key)) + { + return false; + } + + Request.Policy = Convert(CacheRecordPolicy::Load(RequestObject["Policy"].AsObjectView())); + } + return true; + } + + bool GetCacheRecordsRequest::Parse(const CbPackage& RpcRequest) { return Parse(RpcRequest.GetObject()); } + + bool GetCacheRecordsRequest::Format(CbObjectWriter& Writer) const + { + Writer << "Method" + << "GetCacheRecords"; + Writer.BeginObject("Params"); + { + Writer << "DefaultPolicy" << WriteToString<128>(DefaultPolicy); + Writer << "Namespace" << Namespace; + Writer.BeginArray("Requests"); + for (const GetCacheRecordRequest& RecordRequest : Requests) + { + Writer.BeginObject(); + { + WriteCacheRequestKey(Writer, RecordRequest.Key); + WriteOptionalCacheRequestPolicy(Writer, "Policy", RecordRequest.Policy); + } + Writer.EndObject(); + } + Writer.EndArray(); + } + Writer.EndObject(); + + return true; + } + + bool GetCacheRecordsRequest::Format(CbPackage& OutPackage) const + { + CbObjectWriter Writer; + if (!Format(Writer)) + { + return false; + } + OutPackage.SetObject(Writer.Save()); + return true; + } + + bool GetCacheRecordsResult::Parse(const CbPackage& Package) + { + CbObject ResponseObject = Package.GetObject(); + CbArrayView ResultsArray = ResponseObject["Result"].AsArrayView(); + if (!ResultsArray) + { + return false; + } + + Results.reserve(ResultsArray.Num()); + for (CbFieldView RecordView : ResultsArray) + { + if (RecordView.IsNull()) + { + Results.push_back({}); + continue; + } + Results.push_back(GetCacheRecordResult{}); + GetCacheRecordResult& Request = Results.back().value(); + CbObjectView RecordObject = RecordView.AsObjectView(); + CbObjectView KeyObject = RecordObject["Key"].AsObjectView(); + if (!GetRequestCacheKey(KeyObject, Request.Key)) + { + return false; + } + + Request.RecordBody = [RecordObject]() { + IoBuffer RecordBody = IoBuffer(RecordObject.GetSize()); + RecordObject.CopyTo(RecordBody.GetMutableView()); + RecordBody.SetContentType(ZenContentType::kCbObject); + return RecordBody; + }; + + CbArrayView ValuesArray = RecordObject["Values"].AsArrayView(); + Request.Values.reserve(ValuesArray.Num()); + for (CbFieldView Value : ValuesArray) + { + CbObjectView ValueObject = Value.AsObjectView(); + IoHash RawHash = ValueObject["RawHash"].AsHash(); + uint64_t RawSize = ValueObject["RawSize"].AsUInt64(); + Oid Id = ValueObject["Id"].AsObjectId(); + const CbAttachment* Attachment = Package.FindAttachment(RawHash); + if (!Attachment) + { + Request.Values.push_back({.Id = Id, .RawHash = RawHash, .RawSize = RawSize, .Body = {}}); + continue; + } + if (!Attachment->IsCompressedBinary()) + { + return false; + } + Request.Values.push_back({.Id = Id, .RawHash = RawHash, .RawSize = RawSize, .Body = Attachment->AsCompressedBinary()}); + } + } + return true; + } + + bool GetCacheRecordsResult::Format(CbPackage& OutPackage) const + { + CbObjectWriter ResponseObject; + + ResponseObject.BeginArray("Result"); + for (const std::optional<GetCacheRecordResult>& RecordResult : Results) + { + if (!RecordResult.has_value()) + { + ResponseObject.AddNull(); + continue; + } + IoBuffer RecordBody = RecordResult->RecordBody(); + if (!RecordBody) + { + return false; + } + ResponseObject << CbObjectView(RecordBody.Data()); + for (const GetCacheRecordResultValue& Value : RecordResult->Values) + { + if (Value.Body) + { + OutPackage.AddAttachment(CbAttachment(Value.Body)); + } + } + } + ResponseObject.EndArray(); + + OutPackage.SetObject(ResponseObject.Save()); + return true; + } + + bool PutCacheValuesRequest::Parse(const CbPackage& Package) + { + CbObjectView BatchObject = Package.GetObject(); + ZEN_ASSERT(BatchObject["Method"].AsString() == "PutCacheValues"); + + CbObjectView Params = BatchObject["Params"].AsObjectView(); + std::optional<std::string> RequestNamespace = cacherequests::GetRequestNamespace(Params); + if (!RequestNamespace) + { + return false; + } + + Namespace = *RequestNamespace; + DefaultPolicy = GetCachePolicy(Params, "DefaultPolicy").value_or(CachePolicy::Default); + + CbArrayView RequestsArray = Params["Requests"].AsArrayView(); + Requests.reserve(RequestsArray.Num()); + for (CbFieldView RequestField : RequestsArray) + { + CbObjectView RequestObject = RequestField.AsObjectView(); + CbObjectView KeyObject = RequestObject["Key"].AsObjectView(); + + PutCacheValueRequest& Request = Requests.emplace_back(); + + if (!GetRequestCacheKey(KeyObject, Request.Key)) + { + return false; + } + + Request.RawHash = RequestObject["RawHash"].AsBinaryAttachment(); + Request.Policy = GetCachePolicy(RequestObject, "Policy"); + + if (const CbAttachment* Attachment = Package.FindAttachment(Request.RawHash)) + { + if (!Attachment->IsCompressedBinary()) + { + return false; + } + Request.Body = Attachment->AsCompressedBinary(); + } + } + return true; + } + + bool PutCacheValuesRequest::Format(CbPackage& OutPackage) const + { + CbObjectWriter Writer; + Writer << "Method" + << "PutCacheValues"; + + Writer.BeginObject("Params"); + { + Writer << "DefaultPolicy" << WriteToString<128>(DefaultPolicy); + Writer << "Namespace" << Namespace; + + Writer.BeginArray("Requests"); + for (const PutCacheValueRequest& ValueRequest : Requests) + { + Writer.BeginObject(); + { + WriteCacheRequestKey(Writer, ValueRequest.Key); + if (ValueRequest.Body) + { + if (ValueRequest.RawHash != IoHash::Zero && + IoHash::FromBLAKE3(ValueRequest.Body.GetRawHash()) != ValueRequest.RawHash) + { + return false; + } + Writer.AddBinaryAttachment("RawHash", IoHash::FromBLAKE3(ValueRequest.Body.GetRawHash())); + OutPackage.AddAttachment(CbAttachment(ValueRequest.Body)); + } + else if (ValueRequest.RawHash != IoHash::Zero) + { + Writer.AddBinaryAttachment("RawHash", ValueRequest.RawHash); + } + else + { + return false; + } + WriteCachePolicy(Writer, "Policy", ValueRequest.Policy); + } + Writer.EndObject(); + } + Writer.EndArray(); + } + Writer.EndObject(); + + OutPackage.SetObject(Writer.Save()); + return true; + } + + bool PutCacheValuesResult::Parse(const CbPackage& Package) + { + CbArrayView ResultsArray = Package.GetObject()["Result"].AsArrayView(); + if (!ResultsArray) + { + return false; + } + CbFieldViewIterator It = ResultsArray.CreateViewIterator(); + while (It.HasValue()) + { + Success.push_back(It.AsBool()); + It++; + } + return true; + } + + bool PutCacheValuesResult::Format(CbPackage& OutPackage) const + { + if (Success.empty()) + { + return false; + } + CbObjectWriter ResponseObject; + ResponseObject.BeginArray("Result"); + for (bool Value : Success) + { + ResponseObject.AddBool(Value); + } + ResponseObject.EndArray(); + + OutPackage.SetObject(ResponseObject.Save()); + return true; + } + + bool GetCacheValuesRequest::Parse(const CbObjectView& BatchObject) + { + ZEN_ASSERT(BatchObject["Method"].AsString() == "GetCacheValues"); + + CbObjectView Params = BatchObject["Params"].AsObjectView(); + std::optional<std::string> RequestNamespace = cacherequests::GetRequestNamespace(Params); + if (!RequestNamespace) + { + return false; + } + + Namespace = *RequestNamespace; + DefaultPolicy = GetCachePolicy(Params, "DefaultPolicy").value_or(CachePolicy::Default); + + CbArrayView RequestsArray = Params["Requests"].AsArrayView(); + Requests.reserve(RequestsArray.Num()); + for (CbFieldView RequestField : RequestsArray) + { + CbObjectView RequestObject = RequestField.AsObjectView(); + CbObjectView KeyObject = RequestObject["Key"].AsObjectView(); + + GetCacheValueRequest& Request = Requests.emplace_back(); + + if (!GetRequestCacheKey(KeyObject, Request.Key)) + { + return false; + } + + Request.Policy = GetCachePolicy(RequestObject, "Policy"); + } + return true; + } + + bool GetCacheValuesRequest::Format(CbPackage& OutPackage) const + { + CbObjectWriter Writer; + Writer << "Method" + << "GetCacheValues"; + + Writer.BeginObject("Params"); + { + Writer << "DefaultPolicy" << WriteToString<128>(DefaultPolicy); + Writer << "Namespace" << Namespace; + + Writer.BeginArray("Requests"); + for (const GetCacheValueRequest& ValueRequest : Requests) + { + Writer.BeginObject(); + { + WriteCacheRequestKey(Writer, ValueRequest.Key); + WriteCachePolicy(Writer, "Policy", ValueRequest.Policy); + } + Writer.EndObject(); + } + Writer.EndArray(); + } + Writer.EndObject(); + + OutPackage.SetObject(Writer.Save()); + return true; + } + + bool CacheValuesResult::Parse(const CbPackage& Package) + { + CbObject ResponseObject = Package.GetObject(); + CbArrayView ResultsArray = ResponseObject["Result"].AsArrayView(); + if (!ResultsArray) + { + return false; + } + Results.reserve(ResultsArray.Num()); + for (CbFieldView RecordView : ResultsArray) + { + if (RecordView.IsNull()) + { + Results.push_back({}); + continue; + } + CacheValueResult ValueResult; + CbObjectView RecordObject = RecordView.AsObjectView(); + + CbFieldView RawHashField = RecordObject["RawHash"]; + ValueResult.RawHash = RawHashField.AsHash(); + bool Succeeded = !RawHashField.HasError(); + if (Succeeded) + { + const CbAttachment* Attachment = Package.FindAttachment(ValueResult.RawHash); + ValueResult.Body = Attachment ? Attachment->AsCompressedBinary() : CompressedBuffer(); + if (ValueResult.Body) + { + ValueResult.RawSize = ValueResult.Body.GetRawSize(); + } + else + { + ValueResult.RawSize = RecordObject["RawSize"].AsUInt64(UINT64_MAX); + } + } + Results.emplace_back(ValueResult); + } + return true; + } + + bool CacheValuesResult::Format(CbPackage& OutPackage) const + { + CbObjectWriter ResponseObject; + + ResponseObject.BeginArray("Result"); + for (const CacheValueResult& ValueResult : Results) + { + ResponseObject.BeginObject(); + if (ValueResult.RawHash != IoHash::Zero) + { + ResponseObject.AddHash("RawHash", ValueResult.RawHash); + if (ValueResult.Body) + { + OutPackage.AddAttachment(CbAttachment(ValueResult.Body)); + } + else + { + ResponseObject.AddInteger("RawSize", ValueResult.RawSize); + } + } + ResponseObject.EndObject(); + } + ResponseObject.EndArray(); + + OutPackage.SetObject(ResponseObject.Save()); + return true; + } + + bool GetCacheChunksRequest::Parse(const CbObjectView& BatchObject) + { + ZEN_ASSERT(BatchObject["Method"].AsString() == "GetCacheChunks"); + + CbObjectView Params = BatchObject["Params"].AsObjectView(); + std::optional<std::string> RequestNamespace = cacherequests::GetRequestNamespace(Params); + if (!RequestNamespace) + { + return false; + } + + Namespace = *RequestNamespace; + DefaultPolicy = GetCachePolicy(Params, "DefaultPolicy").value_or(CachePolicy::Default); + + CbArrayView RequestsArray = Params["Requests"].AsArrayView(); + Requests.reserve(RequestsArray.Num()); + for (CbFieldView RequestField : RequestsArray) + { + CbObjectView RequestObject = RequestField.AsObjectView(); + CbObjectView KeyObject = RequestObject["Key"].AsObjectView(); + + GetCacheChunkRequest& Request = Requests.emplace_back(); + + if (!GetRequestCacheKey(KeyObject, Request.Key)) + { + return false; + } + + Request.ValueId = RequestObject["ValueId"].AsObjectId(); + Request.ChunkId = RequestObject["ChunkId"].AsHash(); + Request.RawOffset = RequestObject["RawOffset"].AsUInt64(); + Request.RawSize = RequestObject["RawSize"].AsUInt64(UINT64_MAX); + + Request.Policy = GetCachePolicy(RequestObject, "Policy"); + } + return true; + } + + bool GetCacheChunksRequest::Format(CbPackage& OutPackage) const + { + CbObjectWriter Writer; + Writer << "Method" + << "GetCacheChunks"; + + Writer.BeginObject("Params"); + { + Writer << "DefaultPolicy" << WriteToString<128>(DefaultPolicy); + Writer << "Namespace" << Namespace; + + Writer.BeginArray("Requests"); + for (const GetCacheChunkRequest& ValueRequest : Requests) + { + Writer.BeginObject(); + { + WriteCacheRequestKey(Writer, ValueRequest.Key); + + Writer.AddObjectId("ValueId", ValueRequest.ValueId); + Writer.AddHash("ChunkId", ValueRequest.ChunkId); + Writer.AddInteger("RawOffset", ValueRequest.RawOffset); + Writer.AddInteger("RawSize", ValueRequest.RawSize); + + WriteCachePolicy(Writer, "Policy", ValueRequest.Policy); + } + Writer.EndObject(); + } + Writer.EndArray(); + } + Writer.EndObject(); + + OutPackage.SetObject(Writer.Save()); + return true; + } + + bool HttpRequestParseRelativeUri(std::string_view Key, HttpRequestData& Data) + { + std::vector<std::string_view> Tokens; + uint32_t TokenCount = zen::ForEachStrTok(Key, '/', [&](const std::string_view& Token) { + Tokens.push_back(Token); + return true; + }); + + switch (TokenCount) + { + case 1: + Data.Namespace = GetValidNamespaceName(Tokens[0]); + return Data.Namespace.has_value(); + case 2: + { + std::optional<IoHash> PossibleHashKey = GetValidIoHash(Tokens[1]); + if (PossibleHashKey.has_value()) + { + // Legacy bucket/key request + Data.Bucket = GetValidBucketName(Tokens[0]); + if (!Data.Bucket.has_value()) + { + return false; + } + Data.HashKey = PossibleHashKey; + return true; + } + Data.Namespace = GetValidNamespaceName(Tokens[0]); + if (!Data.Namespace.has_value()) + { + return false; + } + Data.Bucket = GetValidBucketName(Tokens[1]); + if (!Data.Bucket.has_value()) + { + return false; + } + return true; + } + case 3: + { + std::optional<IoHash> PossibleHashKey = GetValidIoHash(Tokens[1]); + if (PossibleHashKey.has_value()) + { + // Legacy bucket/key/valueid request + Data.Bucket = GetValidBucketName(Tokens[0]); + if (!Data.Bucket.has_value()) + { + return false; + } + Data.HashKey = PossibleHashKey; + Data.ValueContentId = GetValidIoHash(Tokens[2]); + if (!Data.ValueContentId.has_value()) + { + return false; + } + return true; + } + Data.Namespace = GetValidNamespaceName(Tokens[0]); + if (!Data.Namespace.has_value()) + { + return false; + } + Data.Bucket = GetValidBucketName(Tokens[1]); + if (!Data.Bucket.has_value()) + { + return false; + } + Data.HashKey = GetValidIoHash(Tokens[2]); + if (!Data.HashKey) + { + return false; + } + return true; + } + case 4: + { + Data.Namespace = GetValidNamespaceName(Tokens[0]); + if (!Data.Namespace.has_value()) + { + return false; + } + + Data.Bucket = GetValidBucketName(Tokens[1]); + if (!Data.Bucket.has_value()) + { + return false; + } + + Data.HashKey = GetValidIoHash(Tokens[2]); + if (!Data.HashKey.has_value()) + { + return false; + } + + Data.ValueContentId = GetValidIoHash(Tokens[3]); + if (!Data.ValueContentId.has_value()) + { + return false; + } + return true; + } + default: + return false; + } + } + + // bool CacheRecord::Parse(CbObjectView& Reader) + // { + // CbObjectView KeyView = Reader["Key"].AsObjectView(); + // + // if (!GetRequestCacheKey(KeyView, Key)) + // { + // return false; + // } + // CbArrayView ValuesArray = Reader["Values"].AsArrayView(); + // Values.reserve(ValuesArray.Num()); + // for (CbFieldView Value : ValuesArray) + // { + // CbObjectView ObjectView = Value.AsObjectView(); + // Values.push_back({.Id = ObjectView["Id"].AsObjectId(), + // .RawHash = ObjectView["RawHash"].AsHash(), + // .RawSize = ObjectView["RawSize"].AsUInt64()}); + // } + // return true; + // } + // + // bool CacheRecord::Format(CbObjectWriter& Writer) const + // { + // WriteCacheRequestKey(Writer, Key); + // Writer.BeginArray("Values"); + // for (const CacheRecordValue& Value : Values) + // { + // Writer.BeginObject(); + // { + // Writer.AddObjectId("Id", Value.Id); + // Writer.AddHash("RawHash", Value.RawHash); + // Writer.AddInteger("RawSize", Value.RawSize); + // } + // Writer.EndObject(); + // } + // Writer.EndArray(); + // return true; + // } + +#if ZEN_WITH_TESTS + + static bool operator==(const PutCacheRecordRequestValue& Lhs, const PutCacheRecordRequestValue& Rhs) + { + const IoHash LhsRawHash = Lhs.RawHash != IoHash::Zero ? Lhs.RawHash : IoHash::FromBLAKE3(Lhs.Body.GetRawHash()); + const IoHash RhsRawHash = Rhs.RawHash != IoHash::Zero ? Rhs.RawHash : IoHash::FromBLAKE3(Rhs.Body.GetRawHash()); + return Lhs.Id == Rhs.Id && LhsRawHash == RhsRawHash && + Lhs.Body.GetCompressed().Flatten().GetView().EqualBytes(Rhs.Body.GetCompressed().Flatten().GetView()); + } + + static bool operator==(const zen::CacheValuePolicy& Lhs, const zen::CacheValuePolicy& Rhs) + { + return (Lhs.Id == Rhs.Id) && (Lhs.Policy == Rhs.Policy); + } + + static bool operator==(const std::span<const zen::CacheValuePolicy>& Lhs, const std::span<const zen::CacheValuePolicy>& Rhs) + { + if (Lhs.size() != Lhs.size()) + { + return false; + } + for (size_t Idx = 0; Idx < Lhs.size(); ++Idx) + { + if (Lhs[Idx] != Rhs[Idx]) + { + return false; + } + } + return true; + } + + static bool operator==(const zen::CacheRecordPolicy& Lhs, const zen::CacheRecordPolicy& Rhs) + { + return (Lhs.GetRecordPolicy() == Rhs.GetRecordPolicy()) && (Lhs.GetBasePolicy() == Rhs.GetBasePolicy()) && + (Lhs.GetValuePolicies() == Rhs.GetValuePolicies()); + } + + static bool operator==(const std::optional<CacheRecordPolicy>& Lhs, const std::optional<CacheRecordPolicy>& Rhs) + { + return (Lhs.has_value() == Rhs.has_value()) && (!Lhs || (*Lhs == *Rhs)); + } + + static bool operator==(const PutCacheRecordRequest& Lhs, const PutCacheRecordRequest& Rhs) + { + return (Lhs.Key == Rhs.Key) && (Lhs.Values == Rhs.Values) && (Lhs.Policy == Rhs.Policy); + } + + static bool operator==(const PutCacheRecordsRequest& Lhs, const PutCacheRecordsRequest& Rhs) + { + return (Lhs.DefaultPolicy == Rhs.DefaultPolicy) && (Lhs.Namespace == Rhs.Namespace) && (Lhs.Requests == Rhs.Requests); + } + + static bool operator==(const PutCacheRecordsResult& Lhs, const PutCacheRecordsResult& Rhs) { return (Lhs.Success == Rhs.Success); } + + static bool operator==(const GetCacheRecordRequest& Lhs, const GetCacheRecordRequest& Rhs) + { + return (Lhs.Key == Rhs.Key) && (Lhs.Policy == Rhs.Policy); + } + + static bool operator==(const GetCacheRecordsRequest& Lhs, const GetCacheRecordsRequest& Rhs) + { + return (Lhs.DefaultPolicy == Rhs.DefaultPolicy) && (Lhs.Namespace == Rhs.Namespace) && (Lhs.Requests == Rhs.Requests); + } + + static bool operator==(const GetCacheRecordResultValue& Lhs, const GetCacheRecordResultValue& Rhs) + { + if ((Lhs.Id != Rhs.Id) || (Lhs.RawHash != Rhs.RawHash) || (Lhs.RawSize != Rhs.RawSize)) + { + return false; + } + if (bool(Lhs.Body) != bool(Rhs.Body)) + { + return false; + } + if (bool(Lhs.Body) && Lhs.Body.GetRawHash() != Rhs.Body.GetRawHash()) + { + return false; + } + return true; + } + + static bool operator==(const GetCacheRecordResult& Lhs, const GetCacheRecordResult& Rhs) + { + return Lhs.Key == Rhs.Key && Lhs.Values == Rhs.Values; + } + + static bool operator==(const GetCacheRecordsResult& Lhs, const GetCacheRecordsResult& Rhs) { return Lhs.Results == Rhs.Results; } + + static bool operator==(const PutCacheValueRequest& Lhs, const PutCacheValueRequest& Rhs) + { + if ((Lhs.Key != Rhs.Key) || (Lhs.RawHash != Rhs.RawHash)) + { + return false; + } + + if (bool(Lhs.Body) != bool(Rhs.Body)) + { + return false; + } + if (!Lhs.Body) + { + return true; + } + return Lhs.Body.GetCompressed().Flatten().GetView().EqualBytes(Rhs.Body.GetCompressed().Flatten().GetView()); + } + + static bool operator==(const PutCacheValuesRequest& Lhs, const PutCacheValuesRequest& Rhs) + { + return (Lhs.DefaultPolicy == Rhs.DefaultPolicy) && (Lhs.Namespace == Rhs.Namespace) && (Lhs.Requests == Rhs.Requests); + } + + static bool operator==(const PutCacheValuesResult& Lhs, const PutCacheValuesResult& Rhs) { return (Lhs.Success == Rhs.Success); } + + static bool operator==(const GetCacheValueRequest& Lhs, const GetCacheValueRequest& Rhs) + { + return Lhs.Key == Rhs.Key && Lhs.Policy == Rhs.Policy; + } + + static bool operator==(const GetCacheValuesRequest& Lhs, const GetCacheValuesRequest& Rhs) + { + return Lhs.DefaultPolicy == Rhs.DefaultPolicy && Lhs.Namespace == Rhs.Namespace && Lhs.Requests == Rhs.Requests; + } + + static bool operator==(const CacheValueResult& Lhs, const CacheValueResult& Rhs) + { + if (Lhs.RawHash != Rhs.RawHash) + { + return false; + }; + if (Lhs.Body) + { + if (!Rhs.Body) + { + return false; + } + return Lhs.Body.GetCompressed().Flatten().GetView().EqualBytes(Rhs.Body.GetCompressed().Flatten().GetView()); + } + return Lhs.RawSize == Rhs.RawSize; + } + + static bool operator==(const CacheValuesResult& Lhs, const CacheValuesResult& Rhs) { return Lhs.Results == Rhs.Results; } + + static bool operator==(const GetCacheChunkRequest& Lhs, const GetCacheChunkRequest& Rhs) + { + return Lhs.Key == Rhs.Key && Lhs.ValueId == Rhs.ValueId && Lhs.ChunkId == Rhs.ChunkId && Lhs.RawOffset == Rhs.RawOffset && + Lhs.RawSize == Rhs.RawSize && Lhs.Policy == Rhs.Policy; + } + + static bool operator==(const GetCacheChunksRequest& Lhs, const GetCacheChunksRequest& Rhs) + { + return Lhs.DefaultPolicy == Rhs.DefaultPolicy && Lhs.Namespace == Rhs.Namespace && Lhs.Requests == Rhs.Requests; + } + + static CompressedBuffer MakeCompressedBuffer(size_t Size) { return CompressedBuffer::Compress(SharedBuffer(IoBuffer(Size))); }; + + TEST_CASE("cacherequests.put.cache.records") + { + PutCacheRecordsRequest EmptyRequest; + CbPackage EmptyRequestPackage; + CHECK(EmptyRequest.Format(EmptyRequestPackage)); + PutCacheRecordsRequest EmptyRequestCopy; + CHECK(!EmptyRequestCopy.Parse(EmptyRequestPackage)); // Namespace is required + + PutCacheRecordsRequest FullRequest = { + .DefaultPolicy = CachePolicy::Remote, + .Namespace = "the_namespace", + .Requests = {{.Key = {.Bucket = "thebucket", .Hash = IoHash::FromHexString("177030568fdd461bf4fe5ddbf4d463e514e8178e")}, + .Values = {{.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(2134)}, + {.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(213)}, + {.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(7)}}, + .Policy = CachePolicy::StoreLocal}, + {.Key = {.Bucket = "thebucket", .Hash = IoHash::FromHexString("d1df59fcab06793a5f2c372d795bb907a15cab15")}, + .Values = {{.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(1234)}, + {.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(99)}, + {.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(124)}}, + .Policy = CachePolicy::Store}, + {.Key = {.Bucket = "theotherbucket", .Hash = IoHash::FromHexString("e1ce9e1ac8a6f5953dc14c1fa9512b804ed689df")}, + .Values = {{.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(19)}, + {.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(1248)}, + {.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(823)}}}}}; + + CbPackage FullRequestPackage; + CHECK(FullRequest.Format(FullRequestPackage)); + PutCacheRecordsRequest FullRequestCopy; + CHECK(FullRequestCopy.Parse(FullRequestPackage)); + CHECK(FullRequest == FullRequestCopy); + + PutCacheRecordsResult EmptyResult; + CbPackage EmptyResponsePackage; + CHECK(EmptyResult.Format(EmptyResponsePackage)); + PutCacheRecordsResult EmptyResultCopy; + CHECK(!EmptyResultCopy.Parse(EmptyResponsePackage)); + CHECK(EmptyResult == EmptyResultCopy); + + PutCacheRecordsResult FullResult = {.Success = {true, false, true, true, false}}; + CbPackage FullResponsePackage; + CHECK(FullResult.Format(FullResponsePackage)); + PutCacheRecordsResult FullResultCopy; + CHECK(FullResultCopy.Parse(FullResponsePackage)); + CHECK(FullResult == FullResultCopy); + } + + TEST_CASE("cacherequests.get.cache.records") + { + GetCacheRecordsRequest EmptyRequest; + CbPackage EmptyRequestPackage; + CHECK(EmptyRequest.Format(EmptyRequestPackage)); + GetCacheRecordsRequest EmptyRequestCopy; + CHECK(!EmptyRequestCopy.Parse(EmptyRequestPackage)); // Namespace is required + + GetCacheRecordsRequest FullRequest = { + .DefaultPolicy = CachePolicy::StoreLocal, + .Namespace = "other_namespace", + .Requests = {{.Key = {.Bucket = "finebucket", .Hash = IoHash::FromHexString("d1df59fcab06793a5f2c372d795bb907a15cab15")}, + .Policy = CachePolicy::Local}, + {.Key = {.Bucket = "badbucket", .Hash = IoHash::FromHexString("177030568fdd461bf4fe5ddbf4d463e514e8178e")}, + .Policy = CachePolicy::Remote}, + {.Key = {.Bucket = "badbucket", .Hash = IoHash::FromHexString("e1ce9e1ac8a6f5953dc14c1fa9512b804ed689df")}}}}; + + CbPackage FullRequestPackage; + CHECK(FullRequest.Format(FullRequestPackage)); + GetCacheRecordsRequest FullRequestCopy; + CHECK(FullRequestCopy.Parse(FullRequestPackage)); + CHECK(FullRequest == FullRequestCopy); + + GetCacheRecordsResult EmptyResult; + CbPackage EmptyResponsePackage; + CHECK(EmptyResult.Format(EmptyResponsePackage)); + GetCacheRecordsResult EmptyResultCopy; + CHECK(!EmptyResultCopy.Parse(EmptyResponsePackage)); + CHECK(EmptyResult == EmptyResultCopy); + + PutCacheRecordsRequest FullPutRequest = { + .DefaultPolicy = CachePolicy::Remote, + .Namespace = "the_namespace", + .Requests = {{.Key = {.Bucket = "thebucket", .Hash = IoHash::FromHexString("177030568fdd461bf4fe5ddbf4d463e514e8178e")}, + .Values = {{.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(2134)}, + {.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(213)}, + {.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(7)}}, + .Policy = CachePolicy::StoreLocal}, + {.Key = {.Bucket = "thebucket", .Hash = IoHash::FromHexString("d1df59fcab06793a5f2c372d795bb907a15cab15")}, + .Values = {{.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(1234)}, + {.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(99)}, + {.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(124)}}, + .Policy = CachePolicy::Store}, + {.Key = {.Bucket = "theotherbucket", .Hash = IoHash::FromHexString("e1ce9e1ac8a6f5953dc14c1fa9512b804ed689df")}, + .Values = {{.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(19)}, + {.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(1248)}, + {.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(823)}}}}}; + + CbPackage FullPutRequestPackage; + CHECK(FullPutRequest.Format(FullPutRequestPackage)); + PutCacheRecordsRequest FullPutRequestCopy; + CHECK(FullPutRequestCopy.Parse(FullPutRequestPackage)); + + GetCacheRecordsResult FullResult = { + {GetCacheRecordResult{.Key = FullPutRequestCopy.Requests[0].Key, + .RecordBody = FullPutRequestCopy.Requests[0].RecordBody, + .Values = {{.Id = FullPutRequestCopy.Requests[0].Values[0].Id, + .RawHash = IoHash::FromBLAKE3(FullPutRequestCopy.Requests[0].Values[0].Body.GetRawHash()), + .RawSize = FullPutRequestCopy.Requests[0].Values[0].Body.GetRawSize(), + .Body = FullPutRequestCopy.Requests[0].Values[0].Body}, + {.Id = FullPutRequestCopy.Requests[0].Values[1].Id, + + .RawHash = IoHash::FromBLAKE3(FullPutRequestCopy.Requests[0].Values[1].Body.GetRawHash()), + .RawSize = FullPutRequestCopy.Requests[0].Values[1].Body.GetRawSize(), + .Body = FullPutRequestCopy.Requests[0].Values[1].Body}, + {.Id = FullPutRequestCopy.Requests[0].Values[2].Id, + .RawHash = IoHash::FromBLAKE3(FullPutRequestCopy.Requests[0].Values[2].Body.GetRawHash()), + .RawSize = FullPutRequestCopy.Requests[0].Values[2].Body.GetRawSize(), + .Body = FullPutRequestCopy.Requests[0].Values[2].Body}}}, + {}, // Simulate not have! + GetCacheRecordResult{.Key = FullPutRequestCopy.Requests[2].Key, + .RecordBody = FullPutRequestCopy.Requests[2].RecordBody, + .Values = {{.Id = FullPutRequestCopy.Requests[2].Values[0].Id, + .RawHash = IoHash::FromBLAKE3(FullPutRequestCopy.Requests[2].Values[0].Body.GetRawHash()), + .RawSize = FullPutRequestCopy.Requests[2].Values[0].Body.GetRawSize(), + .Body = FullPutRequestCopy.Requests[2].Values[0].Body}, + {.Id = FullPutRequestCopy.Requests[2].Values[1].Id, + .RawHash = IoHash::FromBLAKE3(FullPutRequestCopy.Requests[2].Values[1].Body.GetRawHash()), + .RawSize = FullPutRequestCopy.Requests[2].Values[1].Body.GetRawSize(), + .Body = {}}, // Simulate not have + {.Id = FullPutRequestCopy.Requests[2].Values[2].Id, + .RawHash = IoHash::FromBLAKE3(FullPutRequestCopy.Requests[2].Values[2].Body.GetRawHash()), + .RawSize = FullPutRequestCopy.Requests[2].Values[2].Body.GetRawSize(), + .Body = FullPutRequestCopy.Requests[2].Values[2].Body}}}}}; + CbPackage FullResponsePackage; + CHECK(FullResult.Format(FullResponsePackage)); + GetCacheRecordsResult FullResultCopy; + CHECK(FullResultCopy.Parse(FullResponsePackage)); + CHECK(FullResult.Results[0] == FullResultCopy.Results[0]); + CHECK(!FullResult.Results[1]); + CHECK(FullResult.Results[2] == FullResultCopy.Results[2]); + } + + TEST_CASE("cacherequests.put.cache.values") + { + PutCacheValuesRequest EmptyRequest; + CbPackage EmptyRequestPackage; + CHECK(EmptyRequest.Format(EmptyRequestPackage)); + PutCacheValuesRequest EmptyRequestCopy; + CHECK(!EmptyRequestCopy.Parse(EmptyRequestPackage)); // Namespace is required + + CompressedBuffer Buffers[3] = {MakeCompressedBuffer(969), MakeCompressedBuffer(3469), MakeCompressedBuffer(9)}; + PutCacheValuesRequest FullRequest = { + .DefaultPolicy = CachePolicy::StoreLocal, + .Namespace = "other_namespace", + .Requests = {{.Key = {.Bucket = "finebucket", .Hash = IoHash::FromHexString("d1df59fcab06793a5f2c372d795bb907a15cab15")}, + .RawHash = IoHash::FromBLAKE3(Buffers[0].GetRawHash()), + .Body = Buffers[0], + .Policy = CachePolicy::Local}, + {.Key = {.Bucket = "badbucket", .Hash = IoHash::FromHexString("177030568fdd461bf4fe5ddbf4d463e514e8178e")}, + .RawHash = IoHash::FromBLAKE3(Buffers[1].GetRawHash()), + .Body = Buffers[1], + .Policy = CachePolicy::Remote}, + {.Key = {.Bucket = "badbucket", .Hash = IoHash::FromHexString("e1ce9e1ac8a6f5953dc14c1fa9512b804ed689df")}, + .RawHash = IoHash::FromBLAKE3(Buffers[2].GetRawHash())}}}; + + CbPackage FullRequestPackage; + CHECK(FullRequest.Format(FullRequestPackage)); + PutCacheValuesRequest FullRequestCopy; + CHECK(FullRequestCopy.Parse(FullRequestPackage)); + CHECK(FullRequest == FullRequestCopy); + + PutCacheValuesResult EmptyResult; + CbPackage EmptyResponsePackage; + CHECK(!EmptyResult.Format(EmptyResponsePackage)); + + PutCacheValuesResult FullResult = {.Success = {true, false, true}}; + + CbPackage FullResponsePackage; + CHECK(FullResult.Format(FullResponsePackage)); + PutCacheValuesResult FullResultCopy; + CHECK(FullResultCopy.Parse(FullResponsePackage)); + CHECK(FullResult == FullResultCopy); + } + + TEST_CASE("cacherequests.get.cache.values") + { + GetCacheValuesRequest EmptyRequest; + CbPackage EmptyRequestPackage; + CHECK(EmptyRequest.Format(EmptyRequestPackage)); + GetCacheValuesRequest EmptyRequestCopy; + CHECK(!EmptyRequestCopy.Parse(EmptyRequestPackage.GetObject())); // Namespace is required + + GetCacheValuesRequest FullRequest = { + .DefaultPolicy = CachePolicy::StoreLocal, + .Namespace = "other_namespace", + .Requests = {{.Key = {.Bucket = "finebucket", .Hash = IoHash::FromHexString("d1df59fcab06793a5f2c372d795bb907a15cab15")}, + .Policy = CachePolicy::Local}, + {.Key = {.Bucket = "badbucket", .Hash = IoHash::FromHexString("177030568fdd461bf4fe5ddbf4d463e514e8178e")}, + .Policy = CachePolicy::Remote}, + {.Key = {.Bucket = "badbucket", .Hash = IoHash::FromHexString("e1ce9e1ac8a6f5953dc14c1fa9512b804ed689df")}}}}; + + CbPackage FullRequestPackage; + CHECK(FullRequest.Format(FullRequestPackage)); + GetCacheValuesRequest FullRequestCopy; + CHECK(FullRequestCopy.Parse(FullRequestPackage.GetObject())); + CHECK(FullRequest == FullRequestCopy); + + CacheValuesResult EmptyResult; + CbPackage EmptyResponsePackage; + CHECK(EmptyResult.Format(EmptyResponsePackage)); + CacheValuesResult EmptyResultCopy; + CHECK(!EmptyResultCopy.Parse(EmptyResponsePackage)); + CHECK(EmptyResult == EmptyResultCopy); + + CompressedBuffer Buffers[3][3] = {{MakeCompressedBuffer(123), MakeCompressedBuffer(321), MakeCompressedBuffer(333)}, + {MakeCompressedBuffer(6123), MakeCompressedBuffer(8321), MakeCompressedBuffer(7333)}, + {MakeCompressedBuffer(5123), MakeCompressedBuffer(2321), MakeCompressedBuffer(2333)}}; + CacheValuesResult FullResult = { + .Results = { + CacheValueResult{.RawSize = 0, .RawHash = IoHash::FromBLAKE3(Buffers[0][0].GetRawHash()), .Body = Buffers[0][0]}, + CacheValueResult{.RawSize = 0, .RawHash = IoHash::FromBLAKE3(Buffers[0][1].GetRawHash()), .Body = Buffers[0][1]}, + CacheValueResult{.RawSize = 0, .RawHash = IoHash::FromBLAKE3(Buffers[0][2].GetRawHash()), .Body = Buffers[0][2]}, + CacheValueResult{.RawSize = 0, .RawHash = IoHash::FromBLAKE3(Buffers[2][0].GetRawHash()), .Body = Buffers[2][0]}, + CacheValueResult{.RawSize = 0, .RawHash = IoHash::FromBLAKE3(Buffers[2][1].GetRawHash()), .Body = Buffers[2][1]}, + CacheValueResult{.RawSize = Buffers[2][2].GetRawSize(), .RawHash = IoHash::FromBLAKE3(Buffers[2][2].GetRawHash())}}}; + CbPackage FullResponsePackage; + CHECK(FullResult.Format(FullResponsePackage)); + CacheValuesResult FullResultCopy; + CHECK(FullResultCopy.Parse(FullResponsePackage)); + CHECK(FullResult == FullResultCopy); + } + + TEST_CASE("cacherequests.get.cache.chunks") + { + GetCacheChunksRequest EmptyRequest; + CbPackage EmptyRequestPackage; + CHECK(EmptyRequest.Format(EmptyRequestPackage)); + GetCacheChunksRequest EmptyRequestCopy; + CHECK(!EmptyRequestCopy.Parse(EmptyRequestPackage.GetObject())); // Namespace is required + + GetCacheChunksRequest FullRequest = { + .DefaultPolicy = CachePolicy::StoreLocal, + .Namespace = "other_namespace", + .Requests = {{.Key = {.Bucket = "finebucket", .Hash = IoHash::FromHexString("d1df59fcab06793a5f2c372d795bb907a15cab15")}, + .ValueId = Oid::NewOid(), + .ChunkId = IoHash::FromHexString("ab3917854bfef7e7af2c372d795bb907a15cab15"), + .RawOffset = 77, + .RawSize = 33, + .Policy = CachePolicy::Local}, + {.Key = {.Bucket = "badbucket", .Hash = IoHash::FromHexString("177030568fdd461bf4fe5ddbf4d463e514e8178e")}, + .ValueId = Oid::NewOid(), + .ChunkId = IoHash::FromHexString("372d795bb907a15cab15ab3917854bfef7e7af2c"), + .Policy = CachePolicy::Remote}, + { + .Key = {.Bucket = "badbucket", .Hash = IoHash::FromHexString("e1ce9e1ac8a6f5953dc14c1fa9512b804ed689df")}, + .ChunkId = IoHash::FromHexString("372d795bb907a15cab15ab3917854bfef7e7af2c"), + }}}; + + CbPackage FullRequestPackage; + CHECK(FullRequest.Format(FullRequestPackage)); + GetCacheChunksRequest FullRequestCopy; + CHECK(FullRequestCopy.Parse(FullRequestPackage.GetObject())); + CHECK(FullRequest == FullRequestCopy); + + GetCacheChunksResult EmptyResult; + CbPackage EmptyResponsePackage; + CHECK(EmptyResult.Format(EmptyResponsePackage)); + GetCacheChunksResult EmptyResultCopy; + CHECK(!EmptyResultCopy.Parse(EmptyResponsePackage)); + CHECK(EmptyResult == EmptyResultCopy); + + CompressedBuffer Buffers[3][3] = {{MakeCompressedBuffer(123), MakeCompressedBuffer(321), MakeCompressedBuffer(333)}, + {MakeCompressedBuffer(6123), MakeCompressedBuffer(8321), MakeCompressedBuffer(7333)}, + {MakeCompressedBuffer(5123), MakeCompressedBuffer(2321), MakeCompressedBuffer(2333)}}; + GetCacheChunksResult FullResult = { + .Results = { + CacheValueResult{.RawSize = 0, .RawHash = IoHash::FromBLAKE3(Buffers[0][0].GetRawHash()), .Body = Buffers[0][0]}, + CacheValueResult{.RawSize = 0, .RawHash = IoHash::FromBLAKE3(Buffers[0][1].GetRawHash()), .Body = Buffers[0][1]}, + CacheValueResult{.RawSize = 0, .RawHash = IoHash::FromBLAKE3(Buffers[0][2].GetRawHash()), .Body = Buffers[0][2]}, + CacheValueResult{.RawSize = 0, .RawHash = IoHash::FromBLAKE3(Buffers[2][0].GetRawHash()), .Body = Buffers[2][0]}, + CacheValueResult{.RawSize = 0, .RawHash = IoHash::FromBLAKE3(Buffers[2][1].GetRawHash()), .Body = Buffers[2][1]}, + CacheValueResult{.RawSize = Buffers[2][2].GetRawSize(), .RawHash = IoHash::FromBLAKE3(Buffers[2][2].GetRawHash())}}}; + CbPackage FullResponsePackage; + CHECK(FullResult.Format(FullResponsePackage)); + GetCacheChunksResult FullResultCopy; + CHECK(FullResultCopy.Parse(FullResponsePackage)); + CHECK(FullResult == FullResultCopy); + } + + TEST_CASE("z$service.parse.relative.Uri") + { + HttpRequestData LegacyBucketRequestBecomesNamespaceRequest; + CHECK(HttpRequestParseRelativeUri("test", LegacyBucketRequestBecomesNamespaceRequest)); + CHECK(LegacyBucketRequestBecomesNamespaceRequest.Namespace == "test"); + CHECK(!LegacyBucketRequestBecomesNamespaceRequest.Bucket.has_value()); + CHECK(!LegacyBucketRequestBecomesNamespaceRequest.HashKey.has_value()); + CHECK(!LegacyBucketRequestBecomesNamespaceRequest.ValueContentId.has_value()); + + HttpRequestData LegacyHashKeyRequest; + CHECK(HttpRequestParseRelativeUri("test/0123456789abcdef12340123456789abcdef1234", LegacyHashKeyRequest)); + CHECK(!LegacyHashKeyRequest.Namespace); + CHECK(LegacyHashKeyRequest.Bucket == "test"); + CHECK(LegacyHashKeyRequest.HashKey == IoHash::FromHexString("0123456789abcdef12340123456789abcdef1234")); + CHECK(!LegacyHashKeyRequest.ValueContentId.has_value()); + + HttpRequestData LegacyValueContentIdRequest; + CHECK(HttpRequestParseRelativeUri("test/0123456789abcdef12340123456789abcdef1234/56789abcdef12345678956789abcdef123456789", + LegacyValueContentIdRequest)); + CHECK(!LegacyValueContentIdRequest.Namespace); + CHECK(LegacyValueContentIdRequest.Bucket == "test"); + CHECK(LegacyValueContentIdRequest.HashKey == IoHash::FromHexString("0123456789abcdef12340123456789abcdef1234")); + CHECK(LegacyValueContentIdRequest.ValueContentId == IoHash::FromHexString("56789abcdef12345678956789abcdef123456789")); + + HttpRequestData V2DefaultNamespaceRequest; + CHECK(HttpRequestParseRelativeUri("ue4.ddc", V2DefaultNamespaceRequest)); + CHECK(V2DefaultNamespaceRequest.Namespace == "ue4.ddc"); + CHECK(!V2DefaultNamespaceRequest.Bucket.has_value()); + CHECK(!V2DefaultNamespaceRequest.HashKey.has_value()); + CHECK(!V2DefaultNamespaceRequest.ValueContentId.has_value()); + + HttpRequestData V2NamespaceRequest; + CHECK(HttpRequestParseRelativeUri("nicenamespace", V2NamespaceRequest)); + CHECK(V2NamespaceRequest.Namespace == "nicenamespace"); + CHECK(!V2NamespaceRequest.Bucket.has_value()); + CHECK(!V2NamespaceRequest.HashKey.has_value()); + CHECK(!V2NamespaceRequest.ValueContentId.has_value()); + + HttpRequestData V2BucketRequestWithDefaultNamespace; + CHECK(HttpRequestParseRelativeUri("ue4.ddc/test", V2BucketRequestWithDefaultNamespace)); + CHECK(V2BucketRequestWithDefaultNamespace.Namespace == "ue4.ddc"); + CHECK(V2BucketRequestWithDefaultNamespace.Bucket == "test"); + CHECK(!V2BucketRequestWithDefaultNamespace.HashKey.has_value()); + CHECK(!V2BucketRequestWithDefaultNamespace.ValueContentId.has_value()); + + HttpRequestData V2BucketRequestWithNamespace; + CHECK(HttpRequestParseRelativeUri("nicenamespace/test", V2BucketRequestWithNamespace)); + CHECK(V2BucketRequestWithNamespace.Namespace == "nicenamespace"); + CHECK(V2BucketRequestWithNamespace.Bucket == "test"); + CHECK(!V2BucketRequestWithNamespace.HashKey.has_value()); + CHECK(!V2BucketRequestWithNamespace.ValueContentId.has_value()); + + HttpRequestData V2HashKeyRequest; + CHECK(HttpRequestParseRelativeUri("test/0123456789abcdef12340123456789abcdef1234", V2HashKeyRequest)); + CHECK(!V2HashKeyRequest.Namespace); + CHECK(V2HashKeyRequest.Bucket == "test"); + CHECK(V2HashKeyRequest.HashKey == IoHash::FromHexString("0123456789abcdef12340123456789abcdef1234")); + CHECK(!V2HashKeyRequest.ValueContentId.has_value()); + + HttpRequestData V2ValueContentIdRequest; + CHECK(HttpRequestParseRelativeUri( + "nicenamespace/test/0123456789abcdef12340123456789abcdef1234/56789abcdef12345678956789abcdef123456789", + V2ValueContentIdRequest)); + CHECK(V2ValueContentIdRequest.Namespace == "nicenamespace"); + CHECK(V2ValueContentIdRequest.Bucket == "test"); + CHECK(V2ValueContentIdRequest.HashKey == IoHash::FromHexString("0123456789abcdef12340123456789abcdef1234")); + CHECK(V2ValueContentIdRequest.ValueContentId == IoHash::FromHexString("56789abcdef12345678956789abcdef123456789")); + + HttpRequestData Invalid; + CHECK(!HttpRequestParseRelativeUri("", Invalid)); + CHECK(!HttpRequestParseRelativeUri("/", Invalid)); + CHECK(!HttpRequestParseRelativeUri("bad\2_namespace", Invalid)); + CHECK(!HttpRequestParseRelativeUri("nice/\2\1bucket", Invalid)); + CHECK(!HttpRequestParseRelativeUri("namespace/bucket/0123456789a", Invalid)); + CHECK(!HttpRequestParseRelativeUri("namespace/bucket/0123456789abcdef12340123456789abcdef1234/56789abcdef1234", Invalid)); + CHECK(!HttpRequestParseRelativeUri("namespace/bucket/pppppppp89abcdef12340123456789abcdef1234", Invalid)); + CHECK(!HttpRequestParseRelativeUri("namespace/bucket/0123456789abcdef12340123456789abcdef1234/56789abcd", Invalid)); + CHECK(!HttpRequestParseRelativeUri( + "namespace/bucket/0123456789abcdef12340123456789abcdef1234/ppppppppdef12345678956789abcdef123456789", + Invalid)); + } +#endif +} // namespace cacherequests + +void +cacherequests_forcelink() +{ +} + +} // namespace zen |