aboutsummaryrefslogtreecommitdiff
path: root/zenserver/cache/structuredcache.cpp
diff options
context:
space:
mode:
authorDan Engelbrecht <[email protected]>2022-05-12 12:18:21 +0200
committerGitHub <[email protected]>2022-05-12 12:18:21 +0200
commit0fc71fc55feb0bfa98c1e6a24f8d8485859dfc70 (patch)
tree2515a259075a46745e71b62abfadee293eefe9c8 /zenserver/cache/structuredcache.cpp
parentMerge pull request #92 from EpicGames/de/bucket-standalone-temp-file-cleanup (diff)
parentuse string::compare in caseSensitiveCompareStrings (diff)
downloadzen-1.0.1.7.tar.xz
zen-1.0.1.7.zip
Merge pull request #93 from EpicGames/de/namespaces-continuedv1.0.1.7actions_updates
De/namespaces continued
Diffstat (limited to 'zenserver/cache/structuredcache.cpp')
-rw-r--r--zenserver/cache/structuredcache.cpp249
1 files changed, 155 insertions, 94 deletions
diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp
index 0f16f6785..eed7a4420 100644
--- a/zenserver/cache/structuredcache.cpp
+++ b/zenserver/cache/structuredcache.cpp
@@ -140,7 +140,6 @@ HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request)
if (std::all_of(begin(Key), end(Key), [](const char c) { return std::isalnum(c); }))
{
// Bucket reference
-
return HandleCacheBucketRequest(Request, Key);
}
@@ -175,14 +174,18 @@ HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request,
case HttpVerb::kDelete:
// Drop bucket
-
- if (m_CacheStore.DropBucket(ZenCacheStore::DefaultNamespace, Bucket))
- {
- return Request.WriteResponse(HttpResponseCode::OK);
- }
- else
{
- return Request.WriteResponse(HttpResponseCode::NotFound);
+ // TODO: Should add namespace to URI and handle if the namespace is missing for backwards compatability
+ std::string_view Namespace = ZenCacheStore::DefaultNamespace;
+
+ if (m_CacheStore.DropBucket(Namespace, Bucket))
+ {
+ return Request.WriteResponse(HttpResponseCode::OK);
+ }
+ else
+ {
+ return Request.WriteResponse(HttpResponseCode::NotFound);
+ }
}
break;
@@ -226,7 +229,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request
}
if (EnumHasAllFlags(PolicyFromURL, CachePolicy::QueryLocal) &&
- m_CacheStore.Get(ZenCacheStore::DefaultNamespace, Ref.BucketSegment, Ref.HashKey, ClientResultValue))
+ m_CacheStore.Get(Ref.Namespace, Ref.BucketSegment, Ref.HashKey, ClientResultValue))
{
Success = true;
ZenContentType ContentType = ClientResultValue.Value.GetContentType();
@@ -287,7 +290,8 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request
if (Success)
{
- ZEN_DEBUG("HIT - '{}/{}' {} '{}' (LOCAL)",
+ ZEN_DEBUG("HIT - '{}/{}/{}' {} '{}' (LOCAL)",
+ Ref.Namespace,
Ref.BucketSegment,
Ref.HashKey,
NiceBytes(ClientResultValue.Value.Size()),
@@ -306,7 +310,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request
}
else if (!EnumHasAllFlags(PolicyFromURL, CachePolicy::QueryRemote))
{
- ZEN_DEBUG("MISS - '{}/{}' '{}'", Ref.BucketSegment, Ref.HashKey, ToString(AcceptType));
+ ZEN_DEBUG("MISS - '{}/{}/{}' '{}'", Ref.Namespace, Ref.BucketSegment, Ref.HashKey, ToString(AcceptType));
m_CacheStats.MissCount++;
return Request.WriteResponse(HttpResponseCode::NotFound);
}
@@ -324,7 +328,8 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request
metrics::OperationTiming::Scope $(m_UpstreamGetRequestTiming);
- if (GetUpstreamCacheResult UpstreamResult = m_UpstreamCache.GetCacheRecord({Ref.BucketSegment, Ref.HashKey}, AcceptType);
+ if (GetUpstreamCacheResult UpstreamResult =
+ m_UpstreamCache.GetCacheRecord({Ref.Namespace, Ref.BucketSegment, Ref.HashKey}, AcceptType);
UpstreamResult.Success)
{
Success = true;
@@ -340,7 +345,8 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request
if (ValidationResult != CbValidateError::None)
{
Success = false;
- ZEN_WARN("Get - '{}/{}' '{}' FAILED, invalid compact binary object from upstream",
+ ZEN_WARN("Get - '{}/{}/{}' '{}' FAILED, invalid compact binary object from upstream",
+ Ref.Namespace,
Ref.BucketSegment,
Ref.HashKey,
ToString(AcceptType));
@@ -351,7 +357,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request
if (Success && StoreLocal)
{
- m_CacheStore.Put(ZenCacheStore::DefaultNamespace, Ref.BucketSegment, Ref.HashKey, ClientResultValue);
+ m_CacheStore.Put(Ref.Namespace, Ref.BucketSegment, Ref.HashKey, ClientResultValue);
}
}
else if (AcceptType == ZenContentType::kCbPackage)
@@ -405,7 +411,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request
if (StoreLocal)
{
- m_CacheStore.Put(ZenCacheStore::DefaultNamespace, Ref.BucketSegment, Ref.HashKey, CacheValue);
+ m_CacheStore.Put(Ref.Namespace, Ref.BucketSegment, Ref.HashKey, CacheValue);
}
BinaryWriter MemStream;
@@ -434,14 +440,19 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request
else
{
Success = false;
- ZEN_WARN("Get - '{}/{}' '{}' FAILED, invalid upstream package", Ref.BucketSegment, Ref.HashKey, ToString(AcceptType));
+ ZEN_WARN("Get - '{}/{}/{}' '{}' FAILED, invalid upstream package",
+ Ref.Namespace,
+ Ref.BucketSegment,
+ Ref.HashKey,
+ ToString(AcceptType));
}
}
}
if (Success)
{
- ZEN_DEBUG("HIT - '{}/{}' {} '{}' (UPSTREAM)",
+ ZEN_DEBUG("HIT - '{}/{}/{}' {} '{}' (UPSTREAM)",
+ Ref.Namespace,
Ref.BucketSegment,
Ref.HashKey,
NiceBytes(ClientResultValue.Value.Size()),
@@ -463,7 +474,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request
}
else
{
- ZEN_DEBUG("MISS - '{}/{}' '{}'", Ref.BucketSegment, Ref.HashKey, ToString(AcceptType));
+ ZEN_DEBUG("MISS - '{}/{}/{}' '{}'", Ref.Namespace, Ref.BucketSegment, Ref.HashKey, ToString(AcceptType));
m_CacheStats.MissCount++;
AsyncRequest.WriteResponse(HttpResponseCode::NotFound);
}
@@ -486,12 +497,12 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request
if (ContentType == HttpContentType::kBinary || ContentType == HttpContentType::kCompressedBinary)
{
- ZEN_DEBUG("PUT - '{}/{}' {} '{}'", Ref.BucketSegment, Ref.HashKey, NiceBytes(Body.Size()), ToString(ContentType));
- m_CacheStore.Put(ZenCacheStore::DefaultNamespace, Ref.BucketSegment, Ref.HashKey, {.Value = Body});
+ ZEN_DEBUG("PUT - '{}/{}/{}' {} '{}'", Ref.Namespace, Ref.BucketSegment, Ref.HashKey, NiceBytes(Body.Size()), ToString(ContentType));
+ m_CacheStore.Put(Ref.Namespace, Ref.BucketSegment, Ref.HashKey, {.Value = Body});
if (EnumHasAllFlags(PolicyFromURL, CachePolicy::StoreRemote))
{
- m_UpstreamCache.EnqueueUpstream({.Type = ContentType, .Key = {Ref.BucketSegment, Ref.HashKey}});
+ m_UpstreamCache.EnqueueUpstream({.Type = ContentType, .Key = {Ref.Namespace, Ref.BucketSegment, Ref.HashKey}});
}
Request.WriteResponse(HttpResponseCode::Created);
@@ -502,7 +513,11 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request
if (ValidationResult != CbValidateError::None)
{
- ZEN_WARN("PUT - '{}/{}' '{}' FAILED, invalid compact binary", Ref.BucketSegment, Ref.HashKey, ToString(ContentType));
+ ZEN_WARN("PUT - '{}/{}/{}' '{}' FAILED, invalid compact binary",
+ Ref.Namespace,
+ Ref.BucketSegment,
+ Ref.HashKey,
+ ToString(ContentType));
return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Compact binary validation failed"sv);
}
@@ -520,7 +535,8 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request
TotalCount++;
});
- ZEN_DEBUG("PUT - '{}/{}' {} '{}' attachments '{}/{}' (valid/total)",
+ ZEN_DEBUG("PUT - '{}/{}/{}' {} '{}' attachments '{}/{}' (valid/total)",
+ Ref.Namespace,
Ref.BucketSegment,
Ref.HashKey,
NiceBytes(Body.Size()),
@@ -529,14 +545,14 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request
ValidAttachments.size());
Body.SetContentType(ZenContentType::kCbObject);
- m_CacheStore.Put(ZenCacheStore::DefaultNamespace, Ref.BucketSegment, Ref.HashKey, {.Value = Body});
+ m_CacheStore.Put(Ref.Namespace, Ref.BucketSegment, Ref.HashKey, {.Value = Body});
const bool IsPartialRecord = TotalCount != static_cast<int32_t>(ValidAttachments.size());
if (EnumHasAllFlags(Policy, CachePolicy::StoreRemote) && !IsPartialRecord)
{
m_UpstreamCache.EnqueueUpstream({.Type = ZenContentType::kCbObject,
- .Key = {Ref.BucketSegment, Ref.HashKey},
+ .Key = {Ref.Namespace, Ref.BucketSegment, Ref.HashKey},
.ValueContentIds = std::move(ValidAttachments)});
}
@@ -548,7 +564,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request
if (!Package.TryLoad(Body))
{
- ZEN_WARN("PUT - '{}/{}' '{}' FAILED, invalid package", Ref.BucketSegment, Ref.HashKey, ToString(ContentType));
+ ZEN_WARN("PUT - '{}/{}/{}' '{}' FAILED, invalid package", Ref.Namespace, Ref.BucketSegment, Ref.HashKey, ToString(ContentType));
return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid package"sv);
}
CachePolicy Policy = PolicyFromURL;
@@ -578,7 +594,8 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request
}
else
{
- ZEN_WARN("PUT - '{}/{}' '{}' FAILED, attachment '{}' is not compressed",
+ ZEN_WARN("PUT - '{}/{}/{}' '{}' FAILED, attachment '{}' is not compressed",
+ Ref.Namespace,
Ref.BucketSegment,
Ref.HashKey,
ToString(HttpContentType::kCbPackage),
@@ -599,7 +616,8 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request
return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid attachment(s)"sv);
}
- ZEN_DEBUG("PUT - '{}/{}' {} '{}', attachments '{}/{}/{}' (new/valid/total)",
+ ZEN_DEBUG("PUT - '{}/{}/{}' {} '{}', attachments '{}/{}/{}' (new/valid/total)",
+ Ref.Namespace,
Ref.BucketSegment,
Ref.HashKey,
NiceBytes(Body.GetSize()),
@@ -612,14 +630,14 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request
CacheValue.Value = CacheRecord.GetBuffer().AsIoBuffer();
CacheValue.Value.SetContentType(ZenContentType::kCbObject);
- m_CacheStore.Put(ZenCacheStore::DefaultNamespace, Ref.BucketSegment, Ref.HashKey, CacheValue);
+ m_CacheStore.Put(Ref.Namespace, Ref.BucketSegment, Ref.HashKey, CacheValue);
const bool IsPartialRecord = Count.Valid != Count.Total;
if (EnumHasAllFlags(Policy, CachePolicy::StoreRemote) && !IsPartialRecord)
{
m_UpstreamCache.EnqueueUpstream({.Type = ZenContentType::kCbPackage,
- .Key = {Ref.BucketSegment, Ref.HashKey},
+ .Key = {Ref.Namespace, Ref.BucketSegment, Ref.HashKey},
.ValueContentIds = std::move(ValidAttachments)});
}
@@ -661,7 +679,7 @@ HttpStructuredCacheService::HandleGetCacheValue(zen::HttpServerRequest& Request,
if (QueryUpstream)
{
- if (auto UpstreamResult = m_UpstreamCache.GetCacheValue({Ref.BucketSegment, Ref.HashKey}, Ref.ValueContentId);
+ if (auto UpstreamResult = m_UpstreamCache.GetCacheValue({Ref.Namespace, Ref.BucketSegment, Ref.HashKey}, Ref.ValueContentId);
UpstreamResult.Success)
{
if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(UpstreamResult.Value)))
@@ -679,7 +697,8 @@ HttpStructuredCacheService::HandleGetCacheValue(zen::HttpServerRequest& Request,
if (!Value)
{
- ZEN_DEBUG("MISS - '{}/{}/{}' '{}' in {}",
+ ZEN_DEBUG("MISS - '{}/{}/{}/{}' '{}' in {}",
+ Ref.Namespace,
Ref.BucketSegment,
Ref.HashKey,
Ref.ValueContentId,
@@ -689,7 +708,8 @@ HttpStructuredCacheService::HandleGetCacheValue(zen::HttpServerRequest& Request,
return Request.WriteResponse(HttpResponseCode::NotFound);
}
- ZEN_DEBUG("HIT - '{}/{}/{}' {} '{}' ({}) in {}",
+ ZEN_DEBUG("HIT - '{}/{}/{}/{}' {} '{}' ({}) in {}",
+ Ref.Namespace,
Ref.BucketSegment,
Ref.HashKey,
Ref.ValueContentId,
@@ -747,7 +767,8 @@ HttpStructuredCacheService::HandlePutCacheValue(zen::HttpServerRequest& Request,
CidStore::InsertResult Result = m_CidStore.AddChunk(Compressed);
- ZEN_DEBUG("PUT - '{}/{}/{}' {} '{}' ({}) in {}",
+ ZEN_DEBUG("PUT - '{}/{}/{}/{}' {} '{}' ({}) in {}",
+ Ref.Namespace,
Ref.BucketSegment,
Ref.HashKey,
Ref.ValueContentId,
@@ -772,8 +793,15 @@ HttpStructuredCacheService::ValidateKeyUri(HttpServerRequest& Request, CacheRef&
return false;
}
+ OutRef.Namespace = ToLower(ZenCacheStore::DefaultNamespace); // TODO: Should add namespace to URI and handle if the namespace is
+ // missing for backwards compatability
OutRef.BucketSegment = ToLower(Key.substr(0, BucketSplitOffset));
+ if (!std::all_of(begin(OutRef.Namespace), end(OutRef.Namespace), [](const char c) { return std::isalnum(c); }))
+ {
+ return false;
+ }
+
if (!std::all_of(begin(OutRef.BucketSegment), end(OutRef.BucketSegment), [](const char c) { return std::isalnum(c); }))
{
return false;
@@ -907,12 +935,14 @@ HttpStructuredCacheService::HandleRpcPutCacheRecords(zen::HttpServerRequest& Req
std::vector<bool> Results;
for (CbFieldView RequestField : Params["Requests"sv])
{
- CbObjectView RequestObject = RequestField.AsObjectView();
- CbObjectView RecordObject = RequestObject["Record"sv].AsObjectView();
- CbObjectView KeyView = RecordObject["Key"sv].AsObjectView();
- CbFieldView BucketField = KeyView["Bucket"sv];
- CbFieldView HashField = KeyView["Hash"sv];
- CacheKey Key = CacheKey::Create(BucketField.AsString(), HashField.AsHash());
+ CbObjectView RequestObject = RequestField.AsObjectView();
+ CbObjectView RecordObject = RequestObject["Record"sv].AsObjectView();
+ CbObjectView KeyView = RecordObject["Key"sv].AsObjectView();
+ CbFieldView NamespaceField = KeyView["Namespace"sv];
+ CbFieldView BucketField = KeyView["Bucket"sv];
+ CbFieldView HashField = KeyView["Hash"sv];
+ CacheKey Key =
+ CacheKey::Create(NamespaceField.AsString(ZenCacheStore::DefaultNamespace), BucketField.AsString(), HashField.AsHash());
if (BucketField.HasError() || HashField.HasError() || Key.Bucket.empty())
{
return Request.WriteResponse(HttpResponseCode::BadRequest);
@@ -981,7 +1011,8 @@ HttpStructuredCacheService::PutCacheRecord(PutRequestData& Request, const CbPack
}
else
{
- ZEN_WARN("PUT - '{}/{}' '{}' FAILED, attachment '{}' is not compressed",
+ ZEN_WARN("PUT - '{}/{}/{}' '{}' FAILED, attachment '{}' is not compressed",
+ Request.Key.Namespace,
Request.Key.Bucket,
Request.Key.Hash,
ToString(HttpContentType::kCbPackage),
@@ -1002,7 +1033,8 @@ HttpStructuredCacheService::PutCacheRecord(PutRequestData& Request, const CbPack
return PutResult::Invalid;
}
- ZEN_DEBUG("PUT - '{}/{}' {}, attachments '{}/{}/{}' (new/valid/total)",
+ ZEN_DEBUG("PUT - '{}/{}/{}' {}, attachments '{}/{}/{}' (new/valid/total)",
+ Request.Key.Namespace,
Request.Key.Bucket,
Request.Key.Hash,
NiceBytes(TransferredSize),
@@ -1014,7 +1046,7 @@ HttpStructuredCacheService::PutCacheRecord(PutRequestData& Request, const CbPack
CacheValue.Value = IoBuffer(Record.GetSize());
Record.CopyTo(MutableMemoryView(CacheValue.Value.MutableData(), CacheValue.Value.GetSize()));
CacheValue.Value.SetContentType(ZenContentType::kCbObject);
- m_CacheStore.Put(ZenCacheStore::DefaultNamespace, Request.Key.Bucket, Request.Key.Hash, CacheValue);
+ m_CacheStore.Put(Request.Key.Namespace, Request.Key.Bucket, Request.Key.Hash, CacheValue);
const bool IsPartialRecord = Count.Valid != Count.Total;
@@ -1080,13 +1112,14 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Htt
for (CbFieldView RequestField : RequestsArray)
{
- RecordRequestData& Request = Requests.emplace_back();
- CbObjectView RequestObject = RequestField.AsObjectView();
- CbObjectView KeyObject = RequestObject["Key"sv].AsObjectView();
- CbFieldView BucketField = KeyObject["Bucket"sv];
- CbFieldView HashField = KeyObject["Hash"sv];
- CacheKey& Key = Request.Upstream.Key;
- Key = CacheKey::Create(BucketField.AsString(), HashField.AsHash());
+ RecordRequestData& Request = Requests.emplace_back();
+ CbObjectView RequestObject = RequestField.AsObjectView();
+ CbObjectView KeyObject = RequestObject["Key"sv].AsObjectView();
+ CbFieldView NamespaceField = KeyObject["Namespace"sv];
+ CbFieldView BucketField = KeyObject["Bucket"sv];
+ CbFieldView HashField = KeyObject["Hash"sv];
+ CacheKey& Key = Request.Upstream.Key;
+ Key = CacheKey::Create(NamespaceField.AsString(ZenCacheStore::DefaultNamespace), BucketField.AsString(), HashField.AsHash());
if (HashField.HasError() || Key.Bucket.empty())
{
return HttpRequest.WriteResponse(HttpResponseCode::BadRequest);
@@ -1100,7 +1133,7 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Htt
ZenCacheValue RecordCacheValue;
if (EnumHasAllFlags(Policy.GetRecordPolicy(), CachePolicy::QueryLocal) &&
- m_CacheStore.Get(ZenCacheStore::DefaultNamespace, Key.Bucket, Key.Hash, RecordCacheValue))
+ m_CacheStore.Get(Key.Namespace, Key.Bucket, Key.Hash, RecordCacheValue))
{
Request.RecordCacheValue = std::move(RecordCacheValue.Value);
if (Request.RecordCacheValue.GetContentType() != ZenContentType::kCbObject)
@@ -1231,7 +1264,7 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Htt
Request.RecordObject = ObjectBuffer;
if (EnumHasAllFlags(Request.DownstreamPolicy.GetRecordPolicy(), CachePolicy::StoreLocal))
{
- m_CacheStore.Put(ZenCacheStore::DefaultNamespace, Key.Bucket, Key.Hash, {.Value = {Request.RecordCacheValue}});
+ m_CacheStore.Put(Key.Namespace, Key.Bucket, Key.Hash, {.Value = {Request.RecordCacheValue}});
}
ParseValues(Request);
Request.UsedUpstream = true;
@@ -1269,7 +1302,11 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Htt
}
else
{
- ZEN_DEBUG("Uncompressed value '{}' from upstream cache record '{}/{}'", Value.ContentId, Key.Bucket, Key.Hash);
+ ZEN_DEBUG("Uncompressed value '{}' from upstream cache record '{}/{}/{}'",
+ Value.ContentId,
+ Key.Namespace,
+ Key.Bucket,
+ Key.Hash);
}
}
if (!Value.Exists && !EnumHasAllFlags(ValuePolicy, CachePolicy::SkipData))
@@ -1306,7 +1343,8 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Htt
}
}
- ZEN_DEBUG("HIT - '{}/{}' {}{}{}",
+ ZEN_DEBUG("HIT - '{}/{}/{}' {}{}{}",
+ Key.Namespace,
Key.Bucket,
Key.Hash,
NiceBytes(Request.RecordCacheValue.Size()),
@@ -1322,11 +1360,11 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Htt
if (!EnumHasAnyFlags(Request.DownstreamPolicy.GetRecordPolicy(), CachePolicy::Query))
{
// If they requested no query, do not record this as a miss
- ZEN_DEBUG("DISABLEDQUERY - '{}/{}'", Key.Bucket, Key.Hash);
+ ZEN_DEBUG("DISABLEDQUERY - '{}/{}/{}'", Key.Namespace, Key.Bucket, Key.Hash);
}
else
{
- ZEN_DEBUG("MISS - '{}/{}' {}", Key.Bucket, Key.Hash, Request.RecordObject ? ""sv : "(PARTIAL)"sv);
+ ZEN_DEBUG("MISS - '{}/{}/{}' {}", Key.Namespace, Key.Bucket, Key.Hash, Request.RecordObject ? ""sv : "(PARTIAL)"sv);
m_CacheStats.MissCount++;
}
}
@@ -1357,11 +1395,13 @@ HttpStructuredCacheService::HandleRpcPutCacheValues(zen::HttpServerRequest& Requ
std::vector<bool> Results;
for (CbFieldView RequestField : Params["Requests"sv])
{
- CbObjectView RequestObject = RequestField.AsObjectView();
- CbObjectView KeyView = RequestObject["Key"sv].AsObjectView();
- CbFieldView BucketField = KeyView["Bucket"sv];
- CbFieldView HashField = KeyView["Hash"sv];
- CacheKey Key = CacheKey::Create(BucketField.AsString(), HashField.AsHash());
+ CbObjectView RequestObject = RequestField.AsObjectView();
+ CbObjectView KeyView = RequestObject["Key"sv].AsObjectView();
+ CbFieldView NamespaceField = KeyView["Namespace"sv];
+ CbFieldView BucketField = KeyView["Bucket"sv];
+ CbFieldView HashField = KeyView["Hash"sv];
+ CacheKey Key =
+ CacheKey::Create(NamespaceField.AsString(ZenCacheStore::DefaultNamespace), BucketField.AsString(), HashField.AsHash());
if (BucketField.HasError() || HashField.HasError() || Key.Bucket.empty())
{
return Request.WriteResponse(HttpResponseCode::BadRequest);
@@ -1388,21 +1428,21 @@ HttpStructuredCacheService::HandleRpcPutCacheValues(zen::HttpServerRequest& Requ
{
IoBuffer Value = Chunk.GetCompressed().Flatten().AsIoBuffer();
Value.SetContentType(ZenContentType::kCompressedBinary);
- m_CacheStore.Put(ZenCacheStore::DefaultNamespace, Key.Bucket, Key.Hash, {.Value = Value});
+ m_CacheStore.Put(Key.Namespace, Key.Bucket, Key.Hash, {.Value = Value});
TransferredSize = Chunk.GetCompressedSize();
}
Succeeded = true;
}
else
{
- ZEN_WARN("PUTCACHEVALUES - '{}/{}/{}' FAILED, value is not compressed", Key.Bucket, Key.Hash, RawHash);
+ ZEN_WARN("PUTCACHEVALUES - '{}/{}/{}/{}' FAILED, value is not compressed", Key.Namespace, Key.Bucket, Key.Hash, RawHash);
return Request.WriteResponse(HttpResponseCode::BadRequest);
}
}
else if (EnumHasAllFlags(Policy, CachePolicy::QueryLocal))
{
ZenCacheValue ExistingValue;
- if (m_CacheStore.Get(ZenCacheStore::DefaultNamespace, Key.Bucket, Key.Hash, ExistingValue) &&
+ if (m_CacheStore.Get(Key.Namespace, Key.Bucket, Key.Hash, ExistingValue) &&
IsCompressedBinary(ExistingValue.Value.GetContentType()))
{
Succeeded = true;
@@ -1416,7 +1456,12 @@ HttpStructuredCacheService::HandleRpcPutCacheValues(zen::HttpServerRequest& Requ
m_UpstreamCache.EnqueueUpstream({.Type = ZenContentType::kCompressedBinary, .Key = Key});
}
Results.push_back(Succeeded);
- ZEN_DEBUG("PUTCACHEVALUES - '{}/{}' {}, '{}'", Key.Bucket, Key.Hash, NiceBytes(TransferredSize), Succeeded ? "Added"sv : "Invalid");
+ ZEN_DEBUG("PUTCACHEVALUES - '{}/{}/{}' {}, '{}'",
+ Key.Namespace,
+ Key.Bucket,
+ Key.Hash,
+ NiceBytes(TransferredSize),
+ Succeeded ? "Added"sv : "Invalid");
}
if (Results.empty())
{
@@ -1466,12 +1511,14 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& Http
{
Stopwatch Timer;
- RequestData& Request = Requests.emplace_back();
- CbObjectView RequestObject = RequestField.AsObjectView();
- CbObjectView KeyObject = RequestObject["Key"sv].AsObjectView();
- CbFieldView BucketField = KeyObject["Bucket"sv];
- CbFieldView HashField = KeyObject["Hash"sv];
- Request.Key = CacheKey::Create(BucketField.AsString(), HashField.AsHash());
+ RequestData& Request = Requests.emplace_back();
+ CbObjectView RequestObject = RequestField.AsObjectView();
+ CbObjectView KeyObject = RequestObject["Key"sv].AsObjectView();
+ CbFieldView NamespaceField = KeyObject["Namespace"sv];
+ CbFieldView BucketField = KeyObject["Bucket"sv];
+ CbFieldView HashField = KeyObject["Hash"sv];
+ Request.Key =
+ CacheKey::Create(NamespaceField.AsString(ZenCacheStore::DefaultNamespace), BucketField.AsString(), HashField.AsHash());
if (BucketField.HasError() || HashField.HasError() || Request.Key.Bucket.empty())
{
return HttpRequest.WriteResponse(HttpResponseCode::BadRequest);
@@ -1486,15 +1533,15 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& Http
ZenCacheValue CacheValue;
if (EnumHasAllFlags(Policy, CachePolicy::QueryLocal))
{
- if (m_CacheStore.Get(ZenCacheStore::DefaultNamespace, Key.Bucket, Key.Hash, CacheValue) &&
- IsCompressedBinary(CacheValue.Value.GetContentType()))
+ if (m_CacheStore.Get(Key.Namespace, Key.Bucket, Key.Hash, CacheValue) && IsCompressedBinary(CacheValue.Value.GetContentType()))
{
Result = CompressedBuffer::FromCompressed(SharedBuffer(CacheValue.Value));
}
}
if (Result)
{
- ZEN_DEBUG("GETCACHEVALUES HIT - '{}/{}' {} ({}) in {}",
+ ZEN_DEBUG("GETCACHEVALUES HIT - '{}/{}/{}' {} ({}) in {}",
+ Key.Namespace,
Key.Bucket,
Key.Hash,
NiceBytes(Result.GetCompressed().GetSize()),
@@ -1509,11 +1556,12 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& Http
else if (!EnumHasAnyFlags(Policy, CachePolicy::Query))
{
// If they requested no query, do not record this as a miss
- ZEN_DEBUG("GETCACHEVALUES DISABLEDQUERY - '{}/{}'", Key.Bucket, Key.Hash);
+ ZEN_DEBUG("GETCACHEVALUES DISABLEDQUERY - '{}/{}/{}'", Key.Namespace, Key.Bucket, Key.Hash);
}
else
{
- ZEN_DEBUG("GETCACHEVALUES MISS - '{}/{}' ({}) in {}",
+ ZEN_DEBUG("GETCACHEVALUES MISS - '{}/{}/{}' ({}) in {}",
+ Key.Namespace,
Key.Bucket,
Key.Hash,
"LOCAL"sv,
@@ -1531,7 +1579,7 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& Http
for (size_t Index : RemoteRequestIndexes)
{
RequestData& Request = Requests[Index];
- RequestedRecordsData.push_back({{Request.Key.Bucket, Request.Key.Hash}});
+ RequestedRecordsData.push_back({{Request.Key.Namespace, Request.Key.Bucket, Request.Key.Hash}});
CacheChunkRequests.push_back(&RequestedRecordsData.back());
}
Stopwatch Timer;
@@ -1551,11 +1599,9 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& Http
// that we copy data from upstream even when SkipData and !StoreLocal are true means that it is too expensive
// for us to keep the data only on the upstream server.
// if (EnumHasAllFlags(Policy, CachePolicy::StoreLocal))
- m_CacheStore.Put(ZenCacheStore::DefaultNamespace,
- Request.Key.Bucket,
- Request.Key.Hash,
- ZenCacheValue{Params.Value});
- ZEN_DEBUG("GETCACHEVALUES HIT - '{}/{}' {} ({}) in {}",
+ m_CacheStore.Put(Request.Key.Namespace, Request.Key.Bucket, Request.Key.Hash, ZenCacheValue{Params.Value});
+ ZEN_DEBUG("GETCACHEVALUES HIT - '{}/{}/{}' {} ({}) in {}",
+ ChunkRequest.Key.Namespace,
ChunkRequest.Key.Bucket,
ChunkRequest.Key.Hash,
NiceBytes(Request.Result.GetCompressed().GetSize()),
@@ -1566,7 +1612,8 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& Http
return;
}
}
- ZEN_DEBUG("GETCACHEVALUES MISS - '{}/{}' ({}) in {}",
+ ZEN_DEBUG("GETCACHEVALUES MISS - '{}/{}/{}' ({}) in {}",
+ ChunkRequest.Key.Namespace,
ChunkRequest.Key.Bucket,
ChunkRequest.Key.Hash,
"UPSTREAM"sv,
@@ -1723,9 +1770,12 @@ HttpStructuredCacheService::ParseGetCacheChunksRequest(std::vector<CacheKeyReque
ChunkRequest& Request = Requests.emplace_back();
Request.Key = &RequestKey;
- CbObjectView KeyObject = RequestObject["Key"sv].AsObjectView();
- CbFieldView HashField = KeyObject["Hash"sv];
- RequestKey.Key = CacheKey::Create(KeyObject["Bucket"sv].AsString(), HashField.AsHash());
+ CbObjectView KeyObject = RequestObject["Key"sv].AsObjectView();
+ CbFieldView HashField = KeyObject["Hash"sv];
+ CbFieldView NamespaceField = KeyObject["Namespace"sv];
+ RequestKey.Key = CacheKey::Create(NamespaceField.AsString(ZenCacheStore::DefaultNamespace),
+ KeyObject["Bucket"sv].AsString(),
+ HashField.AsHash());
if (RequestKey.Key.Bucket.empty() || HashField.HasError())
{
ZEN_WARN("GetCacheChunks: Invalid key in ChunkRequest.");
@@ -1767,9 +1817,11 @@ HttpStructuredCacheService::ParseGetCacheChunksRequest(std::vector<CacheKeyReque
}
else
{
- ZEN_WARN("GetCacheChunks: Keys in ChunkRequest are not sorted: {}/{} came after {}/{}.",
+ ZEN_WARN("GetCacheChunks: Keys in ChunkRequest are not sorted: {}/{}/{} came after {}/{}/{}.",
+ RequestKey.Key.Namespace,
RequestKey.Key.Bucket,
RequestKey.Key.Hash,
+ PreviousRecordKey->Key.Namespace,
PreviousRecordKey->Key.Bucket,
PreviousRecordKey->Key.Hash);
return false;
@@ -1810,7 +1862,7 @@ HttpStructuredCacheService::GetLocalCacheRecords(std::vector<CacheKeyRequest>&
if (!Record.Exists && EnumHasAllFlags(Record.DownstreamPolicy, CachePolicy::QueryLocal))
{
ZenCacheValue CacheValue;
- if (m_CacheStore.Get(ZenCacheStore::DefaultNamespace, RecordKey.Key.Bucket, RecordKey.Key.Hash, CacheValue))
+ if (m_CacheStore.Get(RecordKey.Key.Namespace, RecordKey.Key.Bucket, RecordKey.Key.Hash, CacheValue))
{
Record.Exists = true;
Record.CacheValue = std::move(CacheValue.Value);
@@ -1845,7 +1897,7 @@ HttpStructuredCacheService::GetLocalCacheRecords(std::vector<CacheKeyRequest>&
if (EnumHasAllFlags(Record.DownstreamPolicy, CachePolicy::StoreLocal))
{
- m_CacheStore.Put(ZenCacheStore::DefaultNamespace, Key.Bucket, Key.Hash, {.Value = Record.CacheValue});
+ m_CacheStore.Put(Key.Namespace, Key.Bucket, Key.Hash, {.Value = Record.CacheValue});
}
};
m_UpstreamCache.GetCacheRecords(UpstreamRecordRequests, std::move(OnCacheRecordGetComplete));
@@ -1942,7 +1994,7 @@ HttpStructuredCacheService::GetLocalCacheValues(std::vector<cache::detail::Chunk
if (!Request->Exists && EnumHasAllFlags(Request->DownstreamPolicy, CachePolicy::QueryLocal))
{
ZenCacheValue CacheValue;
- if (m_CacheStore.Get(ZenCacheStore::DefaultNamespace, Request->Key->Key.Bucket, Request->Key->Key.Hash, CacheValue))
+ if (m_CacheStore.Get(Request->Key->Key.Namespace, Request->Key->Key.Bucket, Request->Key->Key.Hash, CacheValue))
{
if (IsCompressedBinary(CacheValue.Value.GetContentType()))
{
@@ -2011,7 +2063,7 @@ HttpStructuredCacheService::GetUpstreamCacheChunks(std::vector<CacheChunkRequest
}
else
{
- m_CacheStore.Put(ZenCacheStore::DefaultNamespace, Key.Key.Bucket, Key.Key.Hash, {.Value = Params.Value});
+ m_CacheStore.Put(Key.Key.Namespace, Key.Key.Bucket, Key.Key.Hash, {.Value = Params.Value});
}
}
if (!EnumHasAllFlags(Request.DownstreamPolicy, CachePolicy::SkipData))
@@ -2058,7 +2110,8 @@ HttpStructuredCacheService::WriteGetCacheChunksResponse(std::vector<cache::detai
Writer.AddInteger("RawSize"sv, Request.TotalSize);
}
- ZEN_DEBUG("HIT - '{}/{}/{}' {} '{}' ({})",
+ ZEN_DEBUG("HIT - '{}/{}/{}/{}' {} '{}' ({})",
+ Request.Key->Key.Namespace,
Request.Key->Key.Bucket,
Request.Key->Key.Hash,
Request.Key->ValueId,
@@ -2069,11 +2122,19 @@ HttpStructuredCacheService::WriteGetCacheChunksResponse(std::vector<cache::detai
}
else if (!EnumHasAnyFlags(Request.DownstreamPolicy, CachePolicy::Query))
{
- ZEN_DEBUG("SKIP - '{}/{}/{}'", Request.Key->Key.Bucket, Request.Key->Key.Hash, Request.Key->ValueId);
+ ZEN_DEBUG("SKIP - '{}/{}/{}/{}'",
+ Request.Key->Key.Namespace,
+ Request.Key->Key.Bucket,
+ Request.Key->Key.Hash,
+ Request.Key->ValueId);
}
else
{
- ZEN_DEBUG("MISS - '{}/{}/{}'", Request.Key->Key.Bucket, Request.Key->Key.Hash, Request.Key->ValueId);
+ ZEN_DEBUG("MISS - '{}/{}/{}/{}'",
+ Request.Key->Key.Namespace,
+ Request.Key->Key.Bucket,
+ Request.Key->Key.Hash,
+ Request.Key->ValueId);
m_CacheStats.MissCount++;
}
}