aboutsummaryrefslogtreecommitdiff
path: root/zenutil/cache/cacherequests.cpp
diff options
context:
space:
mode:
authorStefan Boberg <[email protected]>2023-05-02 10:01:47 +0200
committerGitHub <[email protected]>2023-05-02 10:01:47 +0200
commit075d17f8ada47e990fe94606c3d21df409223465 (patch)
treee50549b766a2f3c354798a54ff73404217b4c9af /zenutil/cache/cacherequests.cpp
parentfix: bundle shouldn't append content zip to zen (diff)
downloadzen-075d17f8ada47e990fe94606c3d21df409223465.tar.xz
zen-075d17f8ada47e990fe94606c3d21df409223465.zip
moved source directories into `/src` (#264)
* moved source directories into `/src` * updated bundle.lua for new `src` path * moved some docs, icon * removed old test trees
Diffstat (limited to 'zenutil/cache/cacherequests.cpp')
-rw-r--r--zenutil/cache/cacherequests.cpp1643
1 files changed, 0 insertions, 1643 deletions
diff --git a/zenutil/cache/cacherequests.cpp b/zenutil/cache/cacherequests.cpp
deleted file mode 100644
index 4c865ec22..000000000
--- a/zenutil/cache/cacherequests.cpp
+++ /dev/null
@@ -1,1643 +0,0 @@
-// Copyright Epic Games, Inc. All Rights Reserved.
-
-#include <zenutil/cache/cacherequests.h>
-
-#include <zencore/compactbinary.h>
-#include <zencore/compactbinarybuilder.h>
-#include <zencore/compactbinarypackage.h>
-#include <zencore/fmtutils.h>
-#include <zencore/logging.h>
-#include <zencore/zencore.h>
-
-#include <string>
-#include <string_view>
-
-#if ZEN_WITH_TESTS
-# include <zencore/testing.h>
-#endif
-
-namespace zen {
-
-namespace cacherequests {
-
- namespace {
- constinit AsciiSet ValidNamespaceNameCharactersSet{"abcdefghijklmnopqrstuvwxyz0123456789-_.ABCDEFGHIJKLMNOPQRSTUVWXYZ"};
- constinit AsciiSet ValidBucketNameCharactersSet{"abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"};
-
- std::optional<std::string> GetValidNamespaceName(std::string_view Name)
- {
- if (Name.empty())
- {
- ZEN_WARN("Namespace is invalid, empty namespace is not allowed");
- return {};
- }
-
- if (Name.length() > 64)
- {
- ZEN_WARN("Namespace '{}' is invalid, length exceeds 64 characters", Name);
- return {};
- }
-
- if (!AsciiSet::HasOnly(Name, ValidNamespaceNameCharactersSet))
- {
- ZEN_WARN("Namespace '{}' is invalid, invalid characters detected", Name);
- return {};
- }
-
- return ToLower(Name);
- }
-
- std::optional<std::string> GetValidBucketName(std::string_view Name)
- {
- if (Name.empty())
- {
- ZEN_WARN("Bucket name is invalid, empty bucket name is not allowed");
- return {};
- }
-
- if (!AsciiSet::HasOnly(Name, ValidBucketNameCharactersSet))
- {
- ZEN_WARN("Bucket name '{}' is invalid, invalid characters detected", Name);
- return {};
- }
-
- return ToLower(Name);
- }
-
- std::optional<IoHash> GetValidIoHash(std::string_view Hash)
- {
- if (Hash.length() != IoHash::StringLength)
- {
- return {};
- }
-
- IoHash KeyHash;
- if (!ParseHexBytes(Hash.data(), Hash.size(), KeyHash.Hash))
- {
- return {};
- }
- return KeyHash;
- }
-
- std::optional<CacheRecordPolicy> Convert(const OptionalCacheRecordPolicy& Policy)
- {
- return Policy.IsValid() ? Policy.Get() : std::optional<CacheRecordPolicy>{};
- };
- } // namespace
-
- std::optional<std::string> GetRequestNamespace(const CbObjectView& Params)
- {
- CbFieldView NamespaceField = Params["Namespace"];
- if (!NamespaceField)
- {
- return std::string("!default!"); // ZenCacheStore::DefaultNamespace);
- }
-
- if (NamespaceField.HasError())
- {
- return {};
- }
- if (!NamespaceField.IsString())
- {
- return {};
- }
- return GetValidNamespaceName(NamespaceField.AsString());
- }
-
- bool GetRequestCacheKey(const CbObjectView& KeyView, CacheKey& Key)
- {
- CbFieldView BucketField = KeyView["Bucket"];
- if (BucketField.HasError())
- {
- return false;
- }
- if (!BucketField.IsString())
- {
- return false;
- }
- std::optional<std::string> Bucket = GetValidBucketName(BucketField.AsString());
- if (!Bucket.has_value())
- {
- return false;
- }
- CbFieldView HashField = KeyView["Hash"];
- if (HashField.HasError())
- {
- return false;
- }
- if (!HashField.IsHash())
- {
- return false;
- }
- Key.Bucket = *Bucket;
- Key.Hash = HashField.AsHash();
- return true;
- }
-
- void WriteCacheRequestKey(CbObjectWriter& Writer, const CacheKey& Value)
- {
- Writer.BeginObject("Key");
- {
- Writer << "Bucket" << Value.Bucket;
- Writer << "Hash" << Value.Hash;
- }
- Writer.EndObject();
- }
-
- void WriteOptionalCacheRequestPolicy(CbObjectWriter& Writer, std::string_view FieldName, const std::optional<CacheRecordPolicy>& Policy)
- {
- if (Policy)
- {
- Writer.SetName(FieldName);
- Policy->Save(Writer);
- }
- }
-
- std::optional<CachePolicy> GetCachePolicy(CbObjectView ObjectView, std::string_view FieldName)
- {
- std::string_view DefaultPolicyText = ObjectView[FieldName].AsString();
- if (DefaultPolicyText.empty())
- {
- return {};
- }
- return ParseCachePolicy(DefaultPolicyText);
- }
-
- void WriteCachePolicy(CbObjectWriter& Writer, std::string_view FieldName, const std::optional<CachePolicy>& Policy)
- {
- if (Policy)
- {
- Writer << FieldName << WriteToString<128>(*Policy);
- }
- }
-
- bool PutCacheRecordsRequest::Parse(const CbPackage& Package)
- {
- CbObjectView BatchObject = Package.GetObject();
- ZEN_ASSERT(BatchObject["Method"].AsString() == "PutCacheRecords");
- AcceptMagic = BatchObject["AcceptType"].AsUInt32(0);
-
- CbObjectView Params = BatchObject["Params"].AsObjectView();
- std::optional<std::string> RequestNamespace = GetRequestNamespace(Params);
- if (!RequestNamespace)
- {
- return false;
- }
- Namespace = *RequestNamespace;
- DefaultPolicy = GetCachePolicy(Params, "DefaultPolicy").value_or(CachePolicy::Default);
-
- CbArrayView RequestFieldArray = Params["Requests"].AsArrayView();
- Requests.resize(RequestFieldArray.Num());
- for (size_t RequestIndex = 0; CbFieldView RequestField : RequestFieldArray)
- {
- CbObjectView RequestObject = RequestField.AsObjectView();
- CbObjectView RecordObject = RequestObject["Record"].AsObjectView();
- CbObjectView KeyView = RecordObject["Key"].AsObjectView();
-
- PutCacheRecordRequest& Request = Requests[RequestIndex++];
-
- if (!GetRequestCacheKey(KeyView, Request.Key))
- {
- return false;
- }
-
- Request.Policy = Convert(CacheRecordPolicy::Load(RequestObject["Policy"].AsObjectView()));
-
- std::unordered_map<IoHash, size_t, IoHash::Hasher> RawHashToAttachmentIndex;
-
- CbArrayView ValuesArray = RecordObject["Values"].AsArrayView();
- Request.Values.resize(ValuesArray.Num());
- RawHashToAttachmentIndex.reserve(ValuesArray.Num());
- for (size_t Index = 0; CbFieldView Value : ValuesArray)
- {
- CbObjectView ObjectView = Value.AsObjectView();
- IoHash AttachmentHash = ObjectView["RawHash"].AsHash();
- RawHashToAttachmentIndex[AttachmentHash] = Index;
- Request.Values[Index++] = {.Id = ObjectView["Id"].AsObjectId(), .RawHash = AttachmentHash};
- }
-
- RecordObject.IterateAttachments([&](CbFieldView HashView) {
- const IoHash ValueHash = HashView.AsHash();
- if (const CbAttachment* Attachment = Package.FindAttachment(ValueHash))
- {
- if (Attachment->IsCompressedBinary())
- {
- auto It = RawHashToAttachmentIndex.find(ValueHash);
- ZEN_ASSERT(It != RawHashToAttachmentIndex.end());
- PutCacheRecordRequestValue& Value = Request.Values[It->second];
- ZEN_ASSERT(Value.RawHash == ValueHash);
- Value.Body = Attachment->AsCompressedBinary();
- ZEN_ASSERT_SLOW(Value.Body.DecodeRawHash() == Value.RawHash);
- }
- }
- });
- }
-
- return true;
- }
-
- bool PutCacheRecordsRequest::Format(CbPackage& OutPackage) const
- {
- CbObjectWriter Writer;
- Writer << "Method"
- << "PutCacheRecords";
- if (AcceptMagic != 0)
- {
- Writer << "Accept" << AcceptMagic;
- }
-
- Writer.BeginObject("Params");
- {
- Writer << "DefaultPolicy" << WriteToString<128>(DefaultPolicy);
- Writer << "Namespace" << Namespace;
-
- Writer.BeginArray("Requests");
- for (const PutCacheRecordRequest& RecordRequest : Requests)
- {
- Writer.BeginObject();
- {
- Writer.BeginObject("Record");
- {
- WriteCacheRequestKey(Writer, RecordRequest.Key);
- Writer.BeginArray("Values");
- for (const PutCacheRecordRequestValue& Value : RecordRequest.Values)
- {
- Writer.BeginObject();
- {
- Writer.AddObjectId("Id", Value.Id);
- const CompressedBuffer& Buffer = Value.Body;
- if (Buffer)
- {
- IoHash AttachmentHash = Buffer.DecodeRawHash(); // TODO: Slow!
- Writer.AddBinaryAttachment("RawHash", AttachmentHash);
- OutPackage.AddAttachment(CbAttachment(Buffer, AttachmentHash));
- Writer.AddInteger("RawSize", Buffer.DecodeRawSize()); // TODO: Slow!
- }
- else
- {
- if (Value.RawHash == IoHash::Zero)
- {
- return false;
- }
- Writer.AddBinaryAttachment("RawHash", Value.RawHash);
- }
- }
- Writer.EndObject();
- }
- Writer.EndArray();
- }
- Writer.EndObject();
- WriteOptionalCacheRequestPolicy(Writer, "Policy", RecordRequest.Policy);
- }
- Writer.EndObject();
- }
- Writer.EndArray();
- }
- Writer.EndObject();
- OutPackage.SetObject(Writer.Save());
-
- return true;
- }
-
- bool PutCacheRecordsResult::Parse(const CbPackage& Package)
- {
- CbArrayView ResultsArray = Package.GetObject()["Result"].AsArrayView();
- if (!ResultsArray)
- {
- return false;
- }
- CbFieldViewIterator It = ResultsArray.CreateViewIterator();
- while (It.HasValue())
- {
- Success.push_back(It.AsBool());
- It++;
- }
- return true;
- }
-
- bool PutCacheRecordsResult::Format(CbPackage& OutPackage) const
- {
- CbObjectWriter ResponseObject;
- ResponseObject.BeginArray("Result");
- for (bool Value : Success)
- {
- ResponseObject.AddBool(Value);
- }
- ResponseObject.EndArray();
-
- OutPackage.SetObject(ResponseObject.Save());
- return true;
- }
-
- bool GetCacheRecordsRequest::Parse(const CbObjectView& RpcRequest)
- {
- ZEN_ASSERT(RpcRequest["Method"].AsString() == "GetCacheRecords");
- AcceptMagic = RpcRequest["AcceptType"].AsUInt32(0);
- AcceptOptions = RpcRequest["AcceptFlags"].AsUInt16(0);
- ProcessPid = RpcRequest["Pid"].AsInt32(0);
-
- CbObjectView Params = RpcRequest["Params"].AsObjectView();
- std::optional<std::string> RequestNamespace = GetRequestNamespace(Params);
- if (!RequestNamespace)
- {
- return false;
- }
-
- Namespace = *RequestNamespace;
- DefaultPolicy = GetCachePolicy(Params, "DefaultPolicy").value_or(CachePolicy::Default);
-
- CbArrayView RequestsArray = Params["Requests"].AsArrayView();
- Requests.reserve(RequestsArray.Num());
- for (CbFieldView RequestField : RequestsArray)
- {
- CbObjectView RequestObject = RequestField.AsObjectView();
- CbObjectView KeyObject = RequestObject["Key"].AsObjectView();
-
- GetCacheRecordRequest& Request = Requests.emplace_back();
-
- if (!GetRequestCacheKey(KeyObject, Request.Key))
- {
- return false;
- }
-
- Request.Policy = Convert(CacheRecordPolicy::Load(RequestObject["Policy"].AsObjectView()));
- }
- return true;
- }
-
- bool GetCacheRecordsRequest::Parse(const CbPackage& RpcRequest) { return Parse(RpcRequest.GetObject()); }
-
- bool GetCacheRecordsRequest::Format(CbObjectWriter& Writer, const std::span<const size_t> OptionalRecordFilter) const
- {
- Writer << "Method"
- << "GetCacheRecords";
- if (AcceptMagic != 0)
- {
- Writer << "Accept" << AcceptMagic;
- }
- if (AcceptOptions != 0)
- {
- Writer << "AcceptFlags" << AcceptOptions;
- }
- if (ProcessPid != 0)
- {
- Writer << "Pid" << ProcessPid;
- }
-
- Writer.BeginObject("Params");
- {
- Writer << "DefaultPolicy" << WriteToString<128>(DefaultPolicy);
- Writer << "Namespace" << Namespace;
- Writer.BeginArray("Requests");
- if (OptionalRecordFilter.empty())
- {
- for (const GetCacheRecordRequest& RecordRequest : Requests)
- {
- Writer.BeginObject();
- {
- WriteCacheRequestKey(Writer, RecordRequest.Key);
- WriteOptionalCacheRequestPolicy(Writer, "Policy", RecordRequest.Policy);
- }
- Writer.EndObject();
- }
- }
- else
- {
- for (size_t Index : OptionalRecordFilter)
- {
- const GetCacheRecordRequest& RecordRequest = Requests[Index];
- Writer.BeginObject();
- {
- WriteCacheRequestKey(Writer, RecordRequest.Key);
- WriteOptionalCacheRequestPolicy(Writer, "Policy", RecordRequest.Policy);
- }
- Writer.EndObject();
- }
- }
- Writer.EndArray();
- }
- Writer.EndObject();
-
- return true;
- }
-
- bool GetCacheRecordsRequest::Format(CbPackage& OutPackage, const std::span<const size_t> OptionalRecordFilter) const
- {
- CbObjectWriter Writer;
- if (!Format(Writer, OptionalRecordFilter))
- {
- return false;
- }
- OutPackage.SetObject(Writer.Save());
- return true;
- }
-
- bool GetCacheRecordsResult::Parse(const CbPackage& Package, const std::span<const size_t> OptionalRecordResultIndexes)
- {
- CbObject ResponseObject = Package.GetObject();
- CbArrayView ResultsArray = ResponseObject["Result"].AsArrayView();
- if (!ResultsArray)
- {
- return false;
- }
-
- Results.reserve(ResultsArray.Num());
- if (!OptionalRecordResultIndexes.empty() && ResultsArray.Num() != OptionalRecordResultIndexes.size())
- {
- return false;
- }
- for (size_t Index = 0; CbFieldView RecordView : ResultsArray)
- {
- size_t ResultIndex = OptionalRecordResultIndexes.empty() ? Index : OptionalRecordResultIndexes[Index];
- Index++;
-
- if (Results.size() <= ResultIndex)
- {
- Results.resize(ResultIndex + 1);
- }
- if (RecordView.IsNull())
- {
- continue;
- }
- Results[ResultIndex] = GetCacheRecordResult{};
- GetCacheRecordResult& Request = Results[ResultIndex].value();
- CbObjectView RecordObject = RecordView.AsObjectView();
- CbObjectView KeyObject = RecordObject["Key"].AsObjectView();
- if (!GetRequestCacheKey(KeyObject, Request.Key))
- {
- return false;
- }
-
- CbArrayView ValuesArray = RecordObject["Values"].AsArrayView();
- Request.Values.reserve(ValuesArray.Num());
- for (CbFieldView Value : ValuesArray)
- {
- CbObjectView ValueObject = Value.AsObjectView();
- IoHash RawHash = ValueObject["RawHash"].AsHash();
- uint64_t RawSize = ValueObject["RawSize"].AsUInt64();
- Oid Id = ValueObject["Id"].AsObjectId();
- const CbAttachment* Attachment = Package.FindAttachment(RawHash);
- if (!Attachment)
- {
- Request.Values.push_back({.Id = Id, .RawHash = RawHash, .RawSize = RawSize, .Body = {}});
- continue;
- }
- if (!Attachment->IsCompressedBinary())
- {
- return false;
- }
- Request.Values.push_back({.Id = Id, .RawHash = RawHash, .RawSize = RawSize, .Body = Attachment->AsCompressedBinary()});
- }
- }
- return true;
- }
-
- bool GetCacheRecordsResult::Format(CbPackage& OutPackage) const
- {
- CbObjectWriter Writer;
-
- Writer.BeginArray("Result");
- for (const std::optional<GetCacheRecordResult>& RecordResult : Results)
- {
- if (!RecordResult.has_value())
- {
- Writer.AddNull();
- continue;
- }
- Writer.BeginObject();
- WriteCacheRequestKey(Writer, RecordResult->Key);
-
- Writer.BeginArray("Values");
- for (const GetCacheRecordResultValue& Value : RecordResult->Values)
- {
- IoHash AttachmentHash = Value.Body ? Value.Body.DecodeRawHash() : Value.RawHash;
- Writer.BeginObject();
- {
- Writer.AddObjectId("Id", Value.Id);
- Writer.AddHash("RawHash", AttachmentHash);
- Writer.AddInteger("RawSize", Value.Body ? Value.Body.DecodeRawSize() : Value.RawSize);
- }
- Writer.EndObject();
- if (Value.Body)
- {
- OutPackage.AddAttachment(CbAttachment(Value.Body, AttachmentHash));
- }
- }
-
- Writer.EndArray();
- Writer.EndObject();
- }
- Writer.EndArray();
-
- OutPackage.SetObject(Writer.Save());
- return true;
- }
-
- bool PutCacheValuesRequest::Parse(const CbPackage& Package)
- {
- CbObjectView BatchObject = Package.GetObject();
- ZEN_ASSERT(BatchObject["Method"].AsString() == "PutCacheValues");
- AcceptMagic = BatchObject["AcceptType"].AsUInt32(0);
-
- CbObjectView Params = BatchObject["Params"].AsObjectView();
- std::optional<std::string> RequestNamespace = cacherequests::GetRequestNamespace(Params);
- if (!RequestNamespace)
- {
- return false;
- }
-
- Namespace = *RequestNamespace;
- DefaultPolicy = GetCachePolicy(Params, "DefaultPolicy").value_or(CachePolicy::Default);
-
- CbArrayView RequestsArray = Params["Requests"].AsArrayView();
- Requests.reserve(RequestsArray.Num());
- for (CbFieldView RequestField : RequestsArray)
- {
- CbObjectView RequestObject = RequestField.AsObjectView();
- CbObjectView KeyObject = RequestObject["Key"].AsObjectView();
-
- PutCacheValueRequest& Request = Requests.emplace_back();
-
- if (!GetRequestCacheKey(KeyObject, Request.Key))
- {
- return false;
- }
-
- Request.RawHash = RequestObject["RawHash"].AsBinaryAttachment();
- Request.Policy = GetCachePolicy(RequestObject, "Policy");
-
- if (const CbAttachment* Attachment = Package.FindAttachment(Request.RawHash))
- {
- if (!Attachment->IsCompressedBinary())
- {
- return false;
- }
- Request.Body = Attachment->AsCompressedBinary();
- }
- }
- return true;
- }
-
- bool PutCacheValuesRequest::Format(CbPackage& OutPackage) const
- {
- CbObjectWriter Writer;
- Writer << "Method"
- << "PutCacheValues";
- if (AcceptMagic != 0)
- {
- Writer << "Accept" << AcceptMagic;
- }
-
- Writer.BeginObject("Params");
- {
- Writer << "DefaultPolicy" << WriteToString<128>(DefaultPolicy);
- Writer << "Namespace" << Namespace;
-
- Writer.BeginArray("Requests");
- for (const PutCacheValueRequest& ValueRequest : Requests)
- {
- Writer.BeginObject();
- {
- WriteCacheRequestKey(Writer, ValueRequest.Key);
- if (ValueRequest.Body)
- {
- IoHash AttachmentHash = ValueRequest.Body.DecodeRawHash();
- if (ValueRequest.RawHash != IoHash::Zero && AttachmentHash != ValueRequest.RawHash)
- {
- return false;
- }
- Writer.AddBinaryAttachment("RawHash", AttachmentHash);
- OutPackage.AddAttachment(CbAttachment(ValueRequest.Body, AttachmentHash));
- }
- else if (ValueRequest.RawHash != IoHash::Zero)
- {
- Writer.AddBinaryAttachment("RawHash", ValueRequest.RawHash);
- }
- else
- {
- return false;
- }
- WriteCachePolicy(Writer, "Policy", ValueRequest.Policy);
- }
- Writer.EndObject();
- }
- Writer.EndArray();
- }
- Writer.EndObject();
-
- OutPackage.SetObject(Writer.Save());
- return true;
- }
-
- bool PutCacheValuesResult::Parse(const CbPackage& Package)
- {
- CbArrayView ResultsArray = Package.GetObject()["Result"].AsArrayView();
- if (!ResultsArray)
- {
- return false;
- }
- CbFieldViewIterator It = ResultsArray.CreateViewIterator();
- while (It.HasValue())
- {
- Success.push_back(It.AsBool());
- It++;
- }
- return true;
- }
-
- bool PutCacheValuesResult::Format(CbPackage& OutPackage) const
- {
- if (Success.empty())
- {
- return false;
- }
- CbObjectWriter ResponseObject;
- ResponseObject.BeginArray("Result");
- for (bool Value : Success)
- {
- ResponseObject.AddBool(Value);
- }
- ResponseObject.EndArray();
-
- OutPackage.SetObject(ResponseObject.Save());
- return true;
- }
-
- bool GetCacheValuesRequest::Parse(const CbObjectView& BatchObject)
- {
- ZEN_ASSERT(BatchObject["Method"].AsString() == "GetCacheValues");
- AcceptMagic = BatchObject["AcceptType"].AsUInt32(0);
- AcceptOptions = BatchObject["AcceptFlags"].AsUInt16(0);
- ProcessPid = BatchObject["Pid"].AsInt32(0);
-
- CbObjectView Params = BatchObject["Params"].AsObjectView();
- std::optional<std::string> RequestNamespace = cacherequests::GetRequestNamespace(Params);
- if (!RequestNamespace)
- {
- return false;
- }
-
- Namespace = *RequestNamespace;
- DefaultPolicy = GetCachePolicy(Params, "DefaultPolicy").value_or(CachePolicy::Default);
-
- CbArrayView RequestsArray = Params["Requests"].AsArrayView();
- Requests.reserve(RequestsArray.Num());
- for (CbFieldView RequestField : RequestsArray)
- {
- CbObjectView RequestObject = RequestField.AsObjectView();
- CbObjectView KeyObject = RequestObject["Key"].AsObjectView();
-
- GetCacheValueRequest& Request = Requests.emplace_back();
-
- if (!GetRequestCacheKey(KeyObject, Request.Key))
- {
- return false;
- }
-
- Request.Policy = GetCachePolicy(RequestObject, "Policy");
- }
- return true;
- }
-
- bool GetCacheValuesRequest::Format(CbPackage& OutPackage, const std::span<const size_t> OptionalValueFilter) const
- {
- CbObjectWriter Writer;
- Writer << "Method"
- << "GetCacheValues";
- if (AcceptMagic != 0)
- {
- Writer << "Accept" << AcceptMagic;
- }
- if (AcceptOptions != 0)
- {
- Writer << "AcceptFlags" << AcceptOptions;
- }
- if (ProcessPid != 0)
- {
- Writer << "Pid" << ProcessPid;
- }
-
- Writer.BeginObject("Params");
- {
- Writer << "DefaultPolicy" << WriteToString<128>(DefaultPolicy);
- Writer << "Namespace" << Namespace;
-
- Writer.BeginArray("Requests");
- if (OptionalValueFilter.empty())
- {
- for (const GetCacheValueRequest& ValueRequest : Requests)
- {
- Writer.BeginObject();
- {
- WriteCacheRequestKey(Writer, ValueRequest.Key);
- WriteCachePolicy(Writer, "Policy", ValueRequest.Policy);
- }
- Writer.EndObject();
- }
- }
- else
- {
- for (size_t Index : OptionalValueFilter)
- {
- const GetCacheValueRequest& ValueRequest = Requests[Index];
- Writer.BeginObject();
- {
- WriteCacheRequestKey(Writer, ValueRequest.Key);
- WriteCachePolicy(Writer, "Policy", ValueRequest.Policy);
- }
- Writer.EndObject();
- }
- }
- Writer.EndArray();
- }
- Writer.EndObject();
-
- OutPackage.SetObject(Writer.Save());
- return true;
- }
-
- bool CacheValuesResult::Parse(const CbPackage& Package, const std::span<const size_t> OptionalValueResultIndexes)
- {
- CbObject ResponseObject = Package.GetObject();
- CbArrayView ResultsArray = ResponseObject["Result"].AsArrayView();
- if (!ResultsArray)
- {
- return false;
- }
- Results.reserve(ResultsArray.Num());
- if (!OptionalValueResultIndexes.empty() && ResultsArray.Num() != OptionalValueResultIndexes.size())
- {
- return false;
- }
- for (size_t Index = 0; CbFieldView RecordView : ResultsArray)
- {
- size_t ResultIndex = OptionalValueResultIndexes.empty() ? Index : OptionalValueResultIndexes[Index];
- Index++;
-
- if (Results.size() <= ResultIndex)
- {
- Results.resize(ResultIndex + 1);
- }
- if (RecordView.IsNull())
- {
- continue;
- }
-
- CacheValueResult& ValueResult = Results[ResultIndex];
- CbObjectView RecordObject = RecordView.AsObjectView();
-
- CbFieldView RawHashField = RecordObject["RawHash"];
- ValueResult.RawHash = RawHashField.AsHash();
- bool Succeeded = !RawHashField.HasError();
- if (Succeeded)
- {
- const CbAttachment* Attachment = Package.FindAttachment(ValueResult.RawHash);
- ValueResult.Body = Attachment ? Attachment->AsCompressedBinary() : CompressedBuffer();
- if (ValueResult.Body)
- {
- ValueResult.RawSize = ValueResult.Body.DecodeRawSize();
- }
- else
- {
- ValueResult.RawSize = RecordObject["RawSize"].AsUInt64(UINT64_MAX);
- }
- }
- }
- return true;
- }
-
- bool CacheValuesResult::Format(CbPackage& OutPackage) const
- {
- CbObjectWriter ResponseObject;
-
- ResponseObject.BeginArray("Result");
- for (const CacheValueResult& ValueResult : Results)
- {
- ResponseObject.BeginObject();
- if (ValueResult.RawHash != IoHash::Zero)
- {
- ResponseObject.AddHash("RawHash", ValueResult.RawHash);
- if (ValueResult.Body)
- {
- OutPackage.AddAttachment(CbAttachment(ValueResult.Body, ValueResult.RawHash));
- }
- else
- {
- ResponseObject.AddInteger("RawSize", ValueResult.RawSize);
- }
- }
- ResponseObject.EndObject();
- }
- ResponseObject.EndArray();
-
- OutPackage.SetObject(ResponseObject.Save());
- return true;
- }
-
- bool GetCacheChunksRequest::Parse(const CbObjectView& BatchObject)
- {
- ZEN_ASSERT(BatchObject["Method"].AsString() == "GetCacheChunks");
- AcceptMagic = BatchObject["AcceptType"].AsUInt32(0);
- AcceptOptions = BatchObject["AcceptFlags"].AsUInt16(0);
- ProcessPid = BatchObject["Pid"].AsInt32(0);
-
- CbObjectView Params = BatchObject["Params"].AsObjectView();
- std::optional<std::string> RequestNamespace = cacherequests::GetRequestNamespace(Params);
- if (!RequestNamespace)
- {
- return false;
- }
-
- Namespace = *RequestNamespace;
- DefaultPolicy = GetCachePolicy(Params, "DefaultPolicy").value_or(CachePolicy::Default);
-
- CbArrayView RequestsArray = Params["ChunkRequests"].AsArrayView();
- Requests.reserve(RequestsArray.Num());
- for (CbFieldView RequestField : RequestsArray)
- {
- CbObjectView RequestObject = RequestField.AsObjectView();
- CbObjectView KeyObject = RequestObject["Key"].AsObjectView();
-
- GetCacheChunkRequest& Request = Requests.emplace_back();
-
- if (!GetRequestCacheKey(KeyObject, Request.Key))
- {
- return false;
- }
-
- Request.ValueId = RequestObject["ValueId"].AsObjectId();
- Request.ChunkId = RequestObject["ChunkId"].AsHash();
- Request.RawOffset = RequestObject["RawOffset"].AsUInt64();
- Request.RawSize = RequestObject["RawSize"].AsUInt64(UINT64_MAX);
-
- Request.Policy = GetCachePolicy(RequestObject, "Policy");
- }
- return true;
- }
-
- bool GetCacheChunksRequest::Format(CbPackage& OutPackage) const
- {
- CbObjectWriter Writer;
- Writer << "Method"
- << "GetCacheChunks";
- if (AcceptMagic != 0)
- {
- Writer << "Accept" << AcceptMagic;
- }
- if (AcceptOptions != 0)
- {
- Writer << "AcceptFlags" << AcceptOptions;
- }
- if (ProcessPid != 0)
- {
- Writer << "Pid" << ProcessPid;
- }
-
- Writer.BeginObject("Params");
- {
- Writer << "DefaultPolicy" << WriteToString<128>(DefaultPolicy);
- Writer << "Namespace" << Namespace;
-
- Writer.BeginArray("ChunkRequests");
- for (const GetCacheChunkRequest& ValueRequest : Requests)
- {
- Writer.BeginObject();
- {
- WriteCacheRequestKey(Writer, ValueRequest.Key);
-
- Writer.AddObjectId("ValueId", ValueRequest.ValueId);
- Writer.AddHash("ChunkId", ValueRequest.ChunkId);
- Writer.AddInteger("RawOffset", ValueRequest.RawOffset);
- Writer.AddInteger("RawSize", ValueRequest.RawSize);
-
- WriteCachePolicy(Writer, "Policy", ValueRequest.Policy);
- }
- Writer.EndObject();
- }
- Writer.EndArray();
- }
- Writer.EndObject();
-
- OutPackage.SetObject(Writer.Save());
- return true;
- }
-
- bool HttpRequestParseRelativeUri(std::string_view Key, HttpRequestData& Data)
- {
- std::vector<std::string_view> Tokens;
- uint32_t TokenCount = zen::ForEachStrTok(Key, '/', [&](const std::string_view& Token) {
- Tokens.push_back(Token);
- return true;
- });
-
- switch (TokenCount)
- {
- case 1:
- Data.Namespace = GetValidNamespaceName(Tokens[0]);
- return Data.Namespace.has_value();
- case 2:
- {
- std::optional<IoHash> PossibleHashKey = GetValidIoHash(Tokens[1]);
- if (PossibleHashKey.has_value())
- {
- // Legacy bucket/key request
- Data.Bucket = GetValidBucketName(Tokens[0]);
- if (!Data.Bucket.has_value())
- {
- return false;
- }
- Data.HashKey = PossibleHashKey;
- return true;
- }
- Data.Namespace = GetValidNamespaceName(Tokens[0]);
- if (!Data.Namespace.has_value())
- {
- return false;
- }
- Data.Bucket = GetValidBucketName(Tokens[1]);
- if (!Data.Bucket.has_value())
- {
- return false;
- }
- return true;
- }
- case 3:
- {
- std::optional<IoHash> PossibleHashKey = GetValidIoHash(Tokens[1]);
- if (PossibleHashKey.has_value())
- {
- // Legacy bucket/key/valueid request
- Data.Bucket = GetValidBucketName(Tokens[0]);
- if (!Data.Bucket.has_value())
- {
- return false;
- }
- Data.HashKey = PossibleHashKey;
- Data.ValueContentId = GetValidIoHash(Tokens[2]);
- if (!Data.ValueContentId.has_value())
- {
- return false;
- }
- return true;
- }
- Data.Namespace = GetValidNamespaceName(Tokens[0]);
- if (!Data.Namespace.has_value())
- {
- return false;
- }
- Data.Bucket = GetValidBucketName(Tokens[1]);
- if (!Data.Bucket.has_value())
- {
- return false;
- }
- Data.HashKey = GetValidIoHash(Tokens[2]);
- if (!Data.HashKey)
- {
- return false;
- }
- return true;
- }
- case 4:
- {
- Data.Namespace = GetValidNamespaceName(Tokens[0]);
- if (!Data.Namespace.has_value())
- {
- return false;
- }
-
- Data.Bucket = GetValidBucketName(Tokens[1]);
- if (!Data.Bucket.has_value())
- {
- return false;
- }
-
- Data.HashKey = GetValidIoHash(Tokens[2]);
- if (!Data.HashKey.has_value())
- {
- return false;
- }
-
- Data.ValueContentId = GetValidIoHash(Tokens[3]);
- if (!Data.ValueContentId.has_value())
- {
- return false;
- }
- return true;
- }
- default:
- return false;
- }
- }
-
- // bool CacheRecord::Parse(CbObjectView& Reader)
- // {
- // CbObjectView KeyView = Reader["Key"].AsObjectView();
- //
- // if (!GetRequestCacheKey(KeyView, Key))
- // {
- // return false;
- // }
- // CbArrayView ValuesArray = Reader["Values"].AsArrayView();
- // Values.reserve(ValuesArray.Num());
- // for (CbFieldView Value : ValuesArray)
- // {
- // CbObjectView ObjectView = Value.AsObjectView();
- // Values.push_back({.Id = ObjectView["Id"].AsObjectId(),
- // .RawHash = ObjectView["RawHash"].AsHash(),
- // .RawSize = ObjectView["RawSize"].AsUInt64()});
- // }
- // return true;
- // }
- //
- // bool CacheRecord::Format(CbObjectWriter& Writer) const
- // {
- // WriteCacheRequestKey(Writer, Key);
- // Writer.BeginArray("Values");
- // for (const CacheRecordValue& Value : Values)
- // {
- // Writer.BeginObject();
- // {
- // Writer.AddObjectId("Id", Value.Id);
- // Writer.AddHash("RawHash", Value.RawHash);
- // Writer.AddInteger("RawSize", Value.RawSize);
- // }
- // Writer.EndObject();
- // }
- // Writer.EndArray();
- // return true;
- // }
-
-#if ZEN_WITH_TESTS
-
- static bool operator==(const PutCacheRecordRequestValue& Lhs, const PutCacheRecordRequestValue& Rhs)
- {
- const IoHash LhsRawHash = Lhs.RawHash != IoHash::Zero ? Lhs.RawHash : Lhs.Body.DecodeRawHash();
- const IoHash RhsRawHash = Rhs.RawHash != IoHash::Zero ? Rhs.RawHash : Rhs.Body.DecodeRawHash();
- return Lhs.Id == Rhs.Id && LhsRawHash == RhsRawHash &&
- Lhs.Body.GetCompressed().Flatten().GetView().EqualBytes(Rhs.Body.GetCompressed().Flatten().GetView());
- }
-
- static bool operator==(const zen::CacheValuePolicy& Lhs, const zen::CacheValuePolicy& Rhs)
- {
- return (Lhs.Id == Rhs.Id) && (Lhs.Policy == Rhs.Policy);
- }
-
- static bool operator==(const std::span<const zen::CacheValuePolicy>& Lhs, const std::span<const zen::CacheValuePolicy>& Rhs)
- {
- if (Lhs.size() != Lhs.size())
- {
- return false;
- }
- for (size_t Idx = 0; Idx < Lhs.size(); ++Idx)
- {
- if (Lhs[Idx] != Rhs[Idx])
- {
- return false;
- }
- }
- return true;
- }
-
- static bool operator==(const zen::CacheRecordPolicy& Lhs, const zen::CacheRecordPolicy& Rhs)
- {
- return (Lhs.GetRecordPolicy() == Rhs.GetRecordPolicy()) && (Lhs.GetBasePolicy() == Rhs.GetBasePolicy()) &&
- (Lhs.GetValuePolicies() == Rhs.GetValuePolicies());
- }
-
- static bool operator==(const std::optional<CacheRecordPolicy>& Lhs, const std::optional<CacheRecordPolicy>& Rhs)
- {
- return (Lhs.has_value() == Rhs.has_value()) && (!Lhs || (*Lhs == *Rhs));
- }
-
- static bool operator==(const PutCacheRecordRequest& Lhs, const PutCacheRecordRequest& Rhs)
- {
- return (Lhs.Key == Rhs.Key) && (Lhs.Values == Rhs.Values) && (Lhs.Policy == Rhs.Policy);
- }
-
- static bool operator==(const PutCacheRecordsRequest& Lhs, const PutCacheRecordsRequest& Rhs)
- {
- return (Lhs.DefaultPolicy == Rhs.DefaultPolicy) && (Lhs.Namespace == Rhs.Namespace) && (Lhs.Requests == Rhs.Requests);
- }
-
- static bool operator==(const PutCacheRecordsResult& Lhs, const PutCacheRecordsResult& Rhs) { return (Lhs.Success == Rhs.Success); }
-
- static bool operator==(const GetCacheRecordRequest& Lhs, const GetCacheRecordRequest& Rhs)
- {
- return (Lhs.Key == Rhs.Key) && (Lhs.Policy == Rhs.Policy);
- }
-
- static bool operator==(const GetCacheRecordsRequest& Lhs, const GetCacheRecordsRequest& Rhs)
- {
- return (Lhs.DefaultPolicy == Rhs.DefaultPolicy) && (Lhs.Namespace == Rhs.Namespace) && (Lhs.Requests == Rhs.Requests);
- }
-
- static bool operator==(const GetCacheRecordResultValue& Lhs, const GetCacheRecordResultValue& Rhs)
- {
- if ((Lhs.Id != Rhs.Id) || (Lhs.RawHash != Rhs.RawHash) || (Lhs.RawSize != Rhs.RawSize))
- {
- return false;
- }
- if (bool(Lhs.Body) != bool(Rhs.Body))
- {
- return false;
- }
- if (bool(Lhs.Body) && Lhs.Body.DecodeRawHash() != Rhs.Body.DecodeRawHash())
- {
- return false;
- }
- return true;
- }
-
- static bool operator==(const GetCacheRecordResult& Lhs, const GetCacheRecordResult& Rhs)
- {
- return Lhs.Key == Rhs.Key && Lhs.Values == Rhs.Values;
- }
-
- static bool operator==(const std::optional<GetCacheRecordResult>& Lhs, const std::optional<GetCacheRecordResult>& Rhs)
- {
- if (Lhs.has_value() != Rhs.has_value())
- {
- return false;
- }
- return *Lhs == Rhs;
- }
-
- static bool operator==(const GetCacheRecordsResult& Lhs, const GetCacheRecordsResult& Rhs) { return Lhs.Results == Rhs.Results; }
-
- static bool operator==(const PutCacheValueRequest& Lhs, const PutCacheValueRequest& Rhs)
- {
- if ((Lhs.Key != Rhs.Key) || (Lhs.RawHash != Rhs.RawHash))
- {
- return false;
- }
-
- if (bool(Lhs.Body) != bool(Rhs.Body))
- {
- return false;
- }
- if (!Lhs.Body)
- {
- return true;
- }
- return Lhs.Body.GetCompressed().Flatten().GetView().EqualBytes(Rhs.Body.GetCompressed().Flatten().GetView());
- }
-
- static bool operator==(const PutCacheValuesRequest& Lhs, const PutCacheValuesRequest& Rhs)
- {
- return (Lhs.DefaultPolicy == Rhs.DefaultPolicy) && (Lhs.Namespace == Rhs.Namespace) && (Lhs.Requests == Rhs.Requests);
- }
-
- static bool operator==(const PutCacheValuesResult& Lhs, const PutCacheValuesResult& Rhs) { return (Lhs.Success == Rhs.Success); }
-
- static bool operator==(const GetCacheValueRequest& Lhs, const GetCacheValueRequest& Rhs)
- {
- return Lhs.Key == Rhs.Key && Lhs.Policy == Rhs.Policy;
- }
-
- static bool operator==(const GetCacheValuesRequest& Lhs, const GetCacheValuesRequest& Rhs)
- {
- return Lhs.DefaultPolicy == Rhs.DefaultPolicy && Lhs.Namespace == Rhs.Namespace && Lhs.Requests == Rhs.Requests;
- }
-
- static bool operator==(const CacheValueResult& Lhs, const CacheValueResult& Rhs)
- {
- if (Lhs.RawHash != Rhs.RawHash)
- {
- return false;
- };
- if (Lhs.Body)
- {
- if (!Rhs.Body)
- {
- return false;
- }
- return Lhs.Body.GetCompressed().Flatten().GetView().EqualBytes(Rhs.Body.GetCompressed().Flatten().GetView());
- }
- return Lhs.RawSize == Rhs.RawSize;
- }
-
- static bool operator==(const CacheValuesResult& Lhs, const CacheValuesResult& Rhs) { return Lhs.Results == Rhs.Results; }
-
- static bool operator==(const GetCacheChunkRequest& Lhs, const GetCacheChunkRequest& Rhs)
- {
- return Lhs.Key == Rhs.Key && Lhs.ValueId == Rhs.ValueId && Lhs.ChunkId == Rhs.ChunkId && Lhs.RawOffset == Rhs.RawOffset &&
- Lhs.RawSize == Rhs.RawSize && Lhs.Policy == Rhs.Policy;
- }
-
- static bool operator==(const GetCacheChunksRequest& Lhs, const GetCacheChunksRequest& Rhs)
- {
- return Lhs.DefaultPolicy == Rhs.DefaultPolicy && Lhs.Namespace == Rhs.Namespace && Lhs.Requests == Rhs.Requests;
- }
-
- static CompressedBuffer MakeCompressedBuffer(size_t Size) { return CompressedBuffer::Compress(SharedBuffer(IoBuffer(Size))); };
-
- TEST_CASE("cacherequests.put.cache.records")
- {
- PutCacheRecordsRequest EmptyRequest;
- CbPackage EmptyRequestPackage;
- CHECK(EmptyRequest.Format(EmptyRequestPackage));
- PutCacheRecordsRequest EmptyRequestCopy;
- CHECK(!EmptyRequestCopy.Parse(EmptyRequestPackage)); // Namespace is required
-
- PutCacheRecordsRequest FullRequest = {
- .DefaultPolicy = CachePolicy::Remote,
- .Namespace = "the_namespace",
- .Requests = {{.Key = {.Bucket = "thebucket", .Hash = IoHash::FromHexString("177030568fdd461bf4fe5ddbf4d463e514e8178e")},
- .Values = {{.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(2134)},
- {.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(213)},
- {.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(7)}},
- .Policy = CachePolicy::StoreLocal},
- {.Key = {.Bucket = "thebucket", .Hash = IoHash::FromHexString("d1df59fcab06793a5f2c372d795bb907a15cab15")},
- .Values = {{.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(1234)},
- {.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(99)},
- {.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(124)}},
- .Policy = CachePolicy::Store},
- {.Key = {.Bucket = "theotherbucket", .Hash = IoHash::FromHexString("e1ce9e1ac8a6f5953dc14c1fa9512b804ed689df")},
- .Values = {{.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(19)},
- {.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(1248)},
- {.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(823)}}}}};
-
- CbPackage FullRequestPackage;
- CHECK(FullRequest.Format(FullRequestPackage));
- PutCacheRecordsRequest FullRequestCopy;
- CHECK(FullRequestCopy.Parse(FullRequestPackage));
- CHECK(FullRequest == FullRequestCopy);
-
- PutCacheRecordsResult EmptyResult;
- CbPackage EmptyResponsePackage;
- CHECK(EmptyResult.Format(EmptyResponsePackage));
- PutCacheRecordsResult EmptyResultCopy;
- CHECK(!EmptyResultCopy.Parse(EmptyResponsePackage));
- CHECK(EmptyResult == EmptyResultCopy);
-
- PutCacheRecordsResult FullResult = {.Success = {true, false, true, true, false}};
- CbPackage FullResponsePackage;
- CHECK(FullResult.Format(FullResponsePackage));
- PutCacheRecordsResult FullResultCopy;
- CHECK(FullResultCopy.Parse(FullResponsePackage));
- CHECK(FullResult == FullResultCopy);
- }
-
- TEST_CASE("cacherequests.get.cache.records")
- {
- GetCacheRecordsRequest EmptyRequest;
- CbPackage EmptyRequestPackage;
- CHECK(EmptyRequest.Format(EmptyRequestPackage));
- GetCacheRecordsRequest EmptyRequestCopy;
- CHECK(!EmptyRequestCopy.Parse(EmptyRequestPackage)); // Namespace is required
-
- GetCacheRecordsRequest FullRequest = {
- .DefaultPolicy = CachePolicy::StoreLocal,
- .Namespace = "other_namespace",
- .Requests = {{.Key = {.Bucket = "finebucket", .Hash = IoHash::FromHexString("d1df59fcab06793a5f2c372d795bb907a15cab15")},
- .Policy = CachePolicy::Local},
- {.Key = {.Bucket = "badbucket", .Hash = IoHash::FromHexString("177030568fdd461bf4fe5ddbf4d463e514e8178e")},
- .Policy = CachePolicy::Remote},
- {.Key = {.Bucket = "badbucket", .Hash = IoHash::FromHexString("e1ce9e1ac8a6f5953dc14c1fa9512b804ed689df")}}}};
-
- CbPackage FullRequestPackage;
- CHECK(FullRequest.Format(FullRequestPackage));
- GetCacheRecordsRequest FullRequestCopy;
- CHECK(FullRequestCopy.Parse(FullRequestPackage));
- CHECK(FullRequest == FullRequestCopy);
-
- CbPackage PartialRequestPackage;
- CHECK(FullRequest.Format(PartialRequestPackage, std::initializer_list<size_t>{0, 2}));
- GetCacheRecordsRequest PartialRequest = FullRequest;
- PartialRequest.Requests.erase(PartialRequest.Requests.begin() + 1);
- GetCacheRecordsRequest PartialRequestCopy;
- CHECK(PartialRequestCopy.Parse(PartialRequestPackage));
- CHECK(PartialRequest == PartialRequestCopy);
-
- GetCacheRecordsResult EmptyResult;
- CbPackage EmptyResponsePackage;
- CHECK(EmptyResult.Format(EmptyResponsePackage));
- GetCacheRecordsResult EmptyResultCopy;
- CHECK(!EmptyResultCopy.Parse(EmptyResponsePackage));
- CHECK(EmptyResult == EmptyResultCopy);
-
- PutCacheRecordsRequest FullPutRequest = {
- .DefaultPolicy = CachePolicy::Remote,
- .Namespace = "the_namespace",
- .Requests = {{.Key = {.Bucket = "thebucket", .Hash = IoHash::FromHexString("177030568fdd461bf4fe5ddbf4d463e514e8178e")},
- .Values = {{.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(2134)},
- {.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(213)},
- {.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(7)}},
- .Policy = CachePolicy::StoreLocal},
- {.Key = {.Bucket = "thebucket", .Hash = IoHash::FromHexString("d1df59fcab06793a5f2c372d795bb907a15cab15")},
- .Values = {{.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(1234)},
- {.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(99)},
- {.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(124)}},
- .Policy = CachePolicy::Store},
- {.Key = {.Bucket = "theotherbucket", .Hash = IoHash::FromHexString("e1ce9e1ac8a6f5953dc14c1fa9512b804ed689df")},
- .Values = {{.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(19)},
- {.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(1248)},
- {.Id = Oid::NewOid(), .Body = MakeCompressedBuffer(823)}}}}};
-
- CbPackage FullPutRequestPackage;
- CHECK(FullPutRequest.Format(FullPutRequestPackage));
- PutCacheRecordsRequest FullPutRequestCopy;
- CHECK(FullPutRequestCopy.Parse(FullPutRequestPackage));
-
- GetCacheRecordsResult FullResult = {
- {GetCacheRecordResult{.Key = FullPutRequestCopy.Requests[0].Key,
- .Values = {{.Id = FullPutRequestCopy.Requests[0].Values[0].Id,
- .RawHash = FullPutRequestCopy.Requests[0].Values[0].Body.DecodeRawHash(),
- .RawSize = FullPutRequestCopy.Requests[0].Values[0].Body.DecodeRawSize(),
- .Body = FullPutRequestCopy.Requests[0].Values[0].Body},
- {.Id = FullPutRequestCopy.Requests[0].Values[1].Id,
-
- .RawHash = FullPutRequestCopy.Requests[0].Values[1].Body.DecodeRawHash(),
- .RawSize = FullPutRequestCopy.Requests[0].Values[1].Body.DecodeRawSize(),
- .Body = FullPutRequestCopy.Requests[0].Values[1].Body},
- {.Id = FullPutRequestCopy.Requests[0].Values[2].Id,
- .RawHash = FullPutRequestCopy.Requests[0].Values[2].Body.DecodeRawHash(),
- .RawSize = FullPutRequestCopy.Requests[0].Values[2].Body.DecodeRawSize(),
- .Body = FullPutRequestCopy.Requests[0].Values[2].Body}}},
- {}, // Simulate not have!
- GetCacheRecordResult{.Key = FullPutRequestCopy.Requests[2].Key,
- .Values = {{.Id = FullPutRequestCopy.Requests[2].Values[0].Id,
- .RawHash = FullPutRequestCopy.Requests[2].Values[0].Body.DecodeRawHash(),
- .RawSize = FullPutRequestCopy.Requests[2].Values[0].Body.DecodeRawSize(),
- .Body = FullPutRequestCopy.Requests[2].Values[0].Body},
- {.Id = FullPutRequestCopy.Requests[2].Values[1].Id,
- .RawHash = FullPutRequestCopy.Requests[2].Values[1].Body.DecodeRawHash(),
- .RawSize = FullPutRequestCopy.Requests[2].Values[1].Body.DecodeRawSize(),
- .Body = {}}, // Simulate not have
- {.Id = FullPutRequestCopy.Requests[2].Values[2].Id,
- .RawHash = FullPutRequestCopy.Requests[2].Values[2].Body.DecodeRawHash(),
- .RawSize = FullPutRequestCopy.Requests[2].Values[2].Body.DecodeRawSize(),
- .Body = FullPutRequestCopy.Requests[2].Values[2].Body}}}}};
- CbPackage FullResponsePackage;
- CHECK(FullResult.Format(FullResponsePackage));
- GetCacheRecordsResult FullResultCopy;
- CHECK(FullResultCopy.Parse(FullResponsePackage));
- CHECK(FullResult.Results[0] == FullResultCopy.Results[0]);
- CHECK(!FullResultCopy.Results[1]);
- CHECK(FullResult.Results[2] == FullResultCopy.Results[2]);
-
- GetCacheRecordsResult PartialResultCopy;
- CHECK(PartialResultCopy.Parse(FullResponsePackage, std::initializer_list<size_t>{0, 3, 4}));
- CHECK(FullResult.Results[0] == PartialResultCopy.Results[0]);
- CHECK(!PartialResultCopy.Results[1]);
- CHECK(!PartialResultCopy.Results[2]);
- CHECK(!PartialResultCopy.Results[3]);
- CHECK(FullResult.Results[2] == PartialResultCopy.Results[4]);
- }
-
- TEST_CASE("cacherequests.put.cache.values")
- {
- PutCacheValuesRequest EmptyRequest;
- CbPackage EmptyRequestPackage;
- CHECK(EmptyRequest.Format(EmptyRequestPackage));
- PutCacheValuesRequest EmptyRequestCopy;
- CHECK(!EmptyRequestCopy.Parse(EmptyRequestPackage)); // Namespace is required
-
- CompressedBuffer Buffers[3] = {MakeCompressedBuffer(969), MakeCompressedBuffer(3469), MakeCompressedBuffer(9)};
- PutCacheValuesRequest FullRequest = {
- .DefaultPolicy = CachePolicy::StoreLocal,
- .Namespace = "other_namespace",
- .Requests = {{.Key = {.Bucket = "finebucket", .Hash = IoHash::FromHexString("d1df59fcab06793a5f2c372d795bb907a15cab15")},
- .RawHash = Buffers[0].DecodeRawHash(),
- .Body = Buffers[0],
- .Policy = CachePolicy::Local},
- {.Key = {.Bucket = "badbucket", .Hash = IoHash::FromHexString("177030568fdd461bf4fe5ddbf4d463e514e8178e")},
- .RawHash = Buffers[1].DecodeRawHash(),
- .Body = Buffers[1],
- .Policy = CachePolicy::Remote},
- {.Key = {.Bucket = "badbucket", .Hash = IoHash::FromHexString("e1ce9e1ac8a6f5953dc14c1fa9512b804ed689df")},
- .RawHash = Buffers[2].DecodeRawHash()}}};
-
- CbPackage FullRequestPackage;
- CHECK(FullRequest.Format(FullRequestPackage));
- PutCacheValuesRequest FullRequestCopy;
- CHECK(FullRequestCopy.Parse(FullRequestPackage));
- CHECK(FullRequest == FullRequestCopy);
-
- PutCacheValuesResult EmptyResult;
- CbPackage EmptyResponsePackage;
- CHECK(!EmptyResult.Format(EmptyResponsePackage));
-
- PutCacheValuesResult FullResult = {.Success = {true, false, true}};
-
- CbPackage FullResponsePackage;
- CHECK(FullResult.Format(FullResponsePackage));
- PutCacheValuesResult FullResultCopy;
- CHECK(FullResultCopy.Parse(FullResponsePackage));
- CHECK(FullResult == FullResultCopy);
- }
-
- TEST_CASE("cacherequests.get.cache.values")
- {
- GetCacheValuesRequest EmptyRequest;
- CbPackage EmptyRequestPackage;
- CHECK(EmptyRequest.Format(EmptyRequestPackage));
- GetCacheValuesRequest EmptyRequestCopy;
- CHECK(!EmptyRequestCopy.Parse(EmptyRequestPackage.GetObject())); // Namespace is required
-
- GetCacheValuesRequest FullRequest = {
- .DefaultPolicy = CachePolicy::StoreLocal,
- .Namespace = "other_namespace",
- .Requests = {{.Key = {.Bucket = "finebucket", .Hash = IoHash::FromHexString("d1df59fcab06793a5f2c372d795bb907a15cab15")},
- .Policy = CachePolicy::Local},
- {.Key = {.Bucket = "badbucket", .Hash = IoHash::FromHexString("177030568fdd461bf4fe5ddbf4d463e514e8178e")},
- .Policy = CachePolicy::Remote},
- {.Key = {.Bucket = "badbucket", .Hash = IoHash::FromHexString("e1ce9e1ac8a6f5953dc14c1fa9512b804ed689df")}}}};
-
- CbPackage FullRequestPackage;
- CHECK(FullRequest.Format(FullRequestPackage));
- GetCacheValuesRequest FullRequestCopy;
- CHECK(FullRequestCopy.Parse(FullRequestPackage.GetObject()));
- CHECK(FullRequest == FullRequestCopy);
-
- CbPackage PartialRequestPackage;
- CHECK(FullRequest.Format(PartialRequestPackage, std::initializer_list<size_t>{0, 2}));
- GetCacheValuesRequest PartialRequest = FullRequest;
- PartialRequest.Requests.erase(PartialRequest.Requests.begin() + 1);
- GetCacheValuesRequest PartialRequestCopy;
- CHECK(PartialRequestCopy.Parse(PartialRequestPackage.GetObject()));
- CHECK(PartialRequest == PartialRequestCopy);
-
- CacheValuesResult EmptyResult;
- CbPackage EmptyResponsePackage;
- CHECK(EmptyResult.Format(EmptyResponsePackage));
- CacheValuesResult EmptyResultCopy;
- CHECK(!EmptyResultCopy.Parse(EmptyResponsePackage));
- CHECK(EmptyResult == EmptyResultCopy);
-
- CompressedBuffer Buffers[3][3] = {{MakeCompressedBuffer(123), MakeCompressedBuffer(321), MakeCompressedBuffer(333)},
- {MakeCompressedBuffer(6123), MakeCompressedBuffer(8321), MakeCompressedBuffer(7333)},
- {MakeCompressedBuffer(5123), MakeCompressedBuffer(2321), MakeCompressedBuffer(2333)}};
- CacheValuesResult FullResult = {
- .Results = {CacheValueResult{.RawSize = 0, .RawHash = Buffers[0][0].DecodeRawHash(), .Body = Buffers[0][0]},
- CacheValueResult{.RawSize = 0, .RawHash = Buffers[0][1].DecodeRawHash(), .Body = Buffers[0][1]},
- CacheValueResult{.RawSize = 0, .RawHash = Buffers[0][2].DecodeRawHash(), .Body = Buffers[0][2]},
- CacheValueResult{.RawSize = 0, .RawHash = Buffers[2][0].DecodeRawHash(), .Body = Buffers[2][0]},
- CacheValueResult{.RawSize = 0, .RawHash = Buffers[2][1].DecodeRawHash(), .Body = Buffers[2][1]},
- CacheValueResult{.RawSize = Buffers[2][2].DecodeRawSize(), .RawHash = Buffers[2][2].DecodeRawHash()}}};
- CbPackage FullResponsePackage;
- CHECK(FullResult.Format(FullResponsePackage));
- CacheValuesResult FullResultCopy;
- CHECK(FullResultCopy.Parse(FullResponsePackage));
- CHECK(FullResult == FullResultCopy);
-
- CacheValuesResult PartialResultCopy;
- CHECK(PartialResultCopy.Parse(FullResponsePackage, std::initializer_list<size_t>{0, 3, 4, 5, 6, 9}));
- CHECK(PartialResultCopy.Results[0] == FullResult.Results[0]);
- CHECK(PartialResultCopy.Results[1].RawHash == IoHash::Zero);
- CHECK(PartialResultCopy.Results[2].RawHash == IoHash::Zero);
- CHECK(PartialResultCopy.Results[3] == FullResult.Results[1]);
- CHECK(PartialResultCopy.Results[4] == FullResult.Results[2]);
- CHECK(PartialResultCopy.Results[5] == FullResult.Results[3]);
- CHECK(PartialResultCopy.Results[6] == FullResult.Results[4]);
- CHECK(PartialResultCopy.Results[7].RawHash == IoHash::Zero);
- CHECK(PartialResultCopy.Results[8].RawHash == IoHash::Zero);
- CHECK(PartialResultCopy.Results[9] == FullResult.Results[5]);
- }
-
- TEST_CASE("cacherequests.get.cache.chunks")
- {
- GetCacheChunksRequest EmptyRequest;
- CbPackage EmptyRequestPackage;
- CHECK(EmptyRequest.Format(EmptyRequestPackage));
- GetCacheChunksRequest EmptyRequestCopy;
- CHECK(!EmptyRequestCopy.Parse(EmptyRequestPackage.GetObject())); // Namespace is required
-
- GetCacheChunksRequest FullRequest = {
- .DefaultPolicy = CachePolicy::StoreLocal,
- .Namespace = "other_namespace",
- .Requests = {{.Key = {.Bucket = "finebucket", .Hash = IoHash::FromHexString("d1df59fcab06793a5f2c372d795bb907a15cab15")},
- .ValueId = Oid::NewOid(),
- .ChunkId = IoHash::FromHexString("ab3917854bfef7e7af2c372d795bb907a15cab15"),
- .RawOffset = 77,
- .RawSize = 33,
- .Policy = CachePolicy::Local},
- {.Key = {.Bucket = "badbucket", .Hash = IoHash::FromHexString("177030568fdd461bf4fe5ddbf4d463e514e8178e")},
- .ValueId = Oid::NewOid(),
- .ChunkId = IoHash::FromHexString("372d795bb907a15cab15ab3917854bfef7e7af2c"),
- .Policy = CachePolicy::Remote},
- {
- .Key = {.Bucket = "badbucket", .Hash = IoHash::FromHexString("e1ce9e1ac8a6f5953dc14c1fa9512b804ed689df")},
- .ChunkId = IoHash::FromHexString("372d795bb907a15cab15ab3917854bfef7e7af2c"),
- }}};
-
- CbPackage FullRequestPackage;
- CHECK(FullRequest.Format(FullRequestPackage));
- GetCacheChunksRequest FullRequestCopy;
- CHECK(FullRequestCopy.Parse(FullRequestPackage.GetObject()));
- CHECK(FullRequest == FullRequestCopy);
-
- GetCacheChunksResult EmptyResult;
- CbPackage EmptyResponsePackage;
- CHECK(EmptyResult.Format(EmptyResponsePackage));
- GetCacheChunksResult EmptyResultCopy;
- CHECK(!EmptyResultCopy.Parse(EmptyResponsePackage));
- CHECK(EmptyResult == EmptyResultCopy);
-
- CompressedBuffer Buffers[3][3] = {{MakeCompressedBuffer(123), MakeCompressedBuffer(321), MakeCompressedBuffer(333)},
- {MakeCompressedBuffer(6123), MakeCompressedBuffer(8321), MakeCompressedBuffer(7333)},
- {MakeCompressedBuffer(5123), MakeCompressedBuffer(2321), MakeCompressedBuffer(2333)}};
- GetCacheChunksResult FullResult = {
- .Results = {CacheValueResult{.RawSize = 0, .RawHash = Buffers[0][0].DecodeRawHash(), .Body = Buffers[0][0]},
- CacheValueResult{.RawSize = 0, .RawHash = Buffers[0][1].DecodeRawHash(), .Body = Buffers[0][1]},
- CacheValueResult{.RawSize = 0, .RawHash = Buffers[0][2].DecodeRawHash(), .Body = Buffers[0][2]},
- CacheValueResult{.RawSize = 0, .RawHash = Buffers[2][0].DecodeRawHash(), .Body = Buffers[2][0]},
- CacheValueResult{.RawSize = 0, .RawHash = Buffers[2][1].DecodeRawHash(), .Body = Buffers[2][1]},
- CacheValueResult{.RawSize = Buffers[2][2].DecodeRawSize(), .RawHash = Buffers[2][2].DecodeRawHash()}}};
- CbPackage FullResponsePackage;
- CHECK(FullResult.Format(FullResponsePackage));
- GetCacheChunksResult FullResultCopy;
- CHECK(FullResultCopy.Parse(FullResponsePackage));
- CHECK(FullResult == FullResultCopy);
- }
-
- TEST_CASE("z$service.parse.relative.Uri")
- {
- HttpRequestData LegacyBucketRequestBecomesNamespaceRequest;
- CHECK(HttpRequestParseRelativeUri("test", LegacyBucketRequestBecomesNamespaceRequest));
- CHECK(LegacyBucketRequestBecomesNamespaceRequest.Namespace == "test");
- CHECK(!LegacyBucketRequestBecomesNamespaceRequest.Bucket.has_value());
- CHECK(!LegacyBucketRequestBecomesNamespaceRequest.HashKey.has_value());
- CHECK(!LegacyBucketRequestBecomesNamespaceRequest.ValueContentId.has_value());
-
- HttpRequestData LegacyHashKeyRequest;
- CHECK(HttpRequestParseRelativeUri("test/0123456789abcdef12340123456789abcdef1234", LegacyHashKeyRequest));
- CHECK(!LegacyHashKeyRequest.Namespace);
- CHECK(LegacyHashKeyRequest.Bucket == "test");
- CHECK(LegacyHashKeyRequest.HashKey == IoHash::FromHexString("0123456789abcdef12340123456789abcdef1234"));
- CHECK(!LegacyHashKeyRequest.ValueContentId.has_value());
-
- HttpRequestData LegacyValueContentIdRequest;
- CHECK(HttpRequestParseRelativeUri("test/0123456789abcdef12340123456789abcdef1234/56789abcdef12345678956789abcdef123456789",
- LegacyValueContentIdRequest));
- CHECK(!LegacyValueContentIdRequest.Namespace);
- CHECK(LegacyValueContentIdRequest.Bucket == "test");
- CHECK(LegacyValueContentIdRequest.HashKey == IoHash::FromHexString("0123456789abcdef12340123456789abcdef1234"));
- CHECK(LegacyValueContentIdRequest.ValueContentId == IoHash::FromHexString("56789abcdef12345678956789abcdef123456789"));
-
- HttpRequestData V2DefaultNamespaceRequest;
- CHECK(HttpRequestParseRelativeUri("ue4.ddc", V2DefaultNamespaceRequest));
- CHECK(V2DefaultNamespaceRequest.Namespace == "ue4.ddc");
- CHECK(!V2DefaultNamespaceRequest.Bucket.has_value());
- CHECK(!V2DefaultNamespaceRequest.HashKey.has_value());
- CHECK(!V2DefaultNamespaceRequest.ValueContentId.has_value());
-
- HttpRequestData V2NamespaceRequest;
- CHECK(HttpRequestParseRelativeUri("nicenamespace", V2NamespaceRequest));
- CHECK(V2NamespaceRequest.Namespace == "nicenamespace");
- CHECK(!V2NamespaceRequest.Bucket.has_value());
- CHECK(!V2NamespaceRequest.HashKey.has_value());
- CHECK(!V2NamespaceRequest.ValueContentId.has_value());
-
- HttpRequestData V2BucketRequestWithDefaultNamespace;
- CHECK(HttpRequestParseRelativeUri("ue4.ddc/test", V2BucketRequestWithDefaultNamespace));
- CHECK(V2BucketRequestWithDefaultNamespace.Namespace == "ue4.ddc");
- CHECK(V2BucketRequestWithDefaultNamespace.Bucket == "test");
- CHECK(!V2BucketRequestWithDefaultNamespace.HashKey.has_value());
- CHECK(!V2BucketRequestWithDefaultNamespace.ValueContentId.has_value());
-
- HttpRequestData V2BucketRequestWithNamespace;
- CHECK(HttpRequestParseRelativeUri("nicenamespace/test", V2BucketRequestWithNamespace));
- CHECK(V2BucketRequestWithNamespace.Namespace == "nicenamespace");
- CHECK(V2BucketRequestWithNamespace.Bucket == "test");
- CHECK(!V2BucketRequestWithNamespace.HashKey.has_value());
- CHECK(!V2BucketRequestWithNamespace.ValueContentId.has_value());
-
- HttpRequestData V2HashKeyRequest;
- CHECK(HttpRequestParseRelativeUri("test/0123456789abcdef12340123456789abcdef1234", V2HashKeyRequest));
- CHECK(!V2HashKeyRequest.Namespace);
- CHECK(V2HashKeyRequest.Bucket == "test");
- CHECK(V2HashKeyRequest.HashKey == IoHash::FromHexString("0123456789abcdef12340123456789abcdef1234"));
- CHECK(!V2HashKeyRequest.ValueContentId.has_value());
-
- HttpRequestData V2ValueContentIdRequest;
- CHECK(HttpRequestParseRelativeUri(
- "nicenamespace/test/0123456789abcdef12340123456789abcdef1234/56789abcdef12345678956789abcdef123456789",
- V2ValueContentIdRequest));
- CHECK(V2ValueContentIdRequest.Namespace == "nicenamespace");
- CHECK(V2ValueContentIdRequest.Bucket == "test");
- CHECK(V2ValueContentIdRequest.HashKey == IoHash::FromHexString("0123456789abcdef12340123456789abcdef1234"));
- CHECK(V2ValueContentIdRequest.ValueContentId == IoHash::FromHexString("56789abcdef12345678956789abcdef123456789"));
-
- HttpRequestData Invalid;
- CHECK(!HttpRequestParseRelativeUri("", Invalid));
- CHECK(!HttpRequestParseRelativeUri("/", Invalid));
- CHECK(!HttpRequestParseRelativeUri("bad\2_namespace", Invalid));
- CHECK(!HttpRequestParseRelativeUri("nice/\2\1bucket", Invalid));
- CHECK(!HttpRequestParseRelativeUri("namespace/bucket/0123456789a", Invalid));
- CHECK(!HttpRequestParseRelativeUri("namespace/bucket/0123456789abcdef12340123456789abcdef1234/56789abcdef1234", Invalid));
- CHECK(!HttpRequestParseRelativeUri("namespace/bucket/pppppppp89abcdef12340123456789abcdef1234", Invalid));
- CHECK(!HttpRequestParseRelativeUri("namespace/bucket/0123456789abcdef12340123456789abcdef1234/56789abcd", Invalid));
- CHECK(!HttpRequestParseRelativeUri(
- "namespace/bucket/0123456789abcdef12340123456789abcdef1234/ppppppppdef12345678956789abcdef123456789",
- Invalid));
- }
-#endif
-} // namespace cacherequests
-
-void
-cacherequests_forcelink()
-{
-}
-
-} // namespace zen