From eadca136e5db2895379dda0f1a66f019d3914a82 Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Fri, 29 Oct 2021 11:14:12 +0200 Subject: First pass batch request. --- zenserver/cache/structuredcache.cpp | 72 +++++++++++++++++++++++++++++++++++++ zenserver/cache/structuredcache.h | 1 + 2 files changed, 73 insertions(+) (limited to 'zenserver') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 35cb02cbb..172e122e4 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -210,6 +210,13 @@ HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request) { std::string_view Key = Request.RelativeUri(); + if (Key == "$batch") + { + const auto QueryParams = Request.GetQueryParams(); + CachePolicy Policy = ParseCachePolicy(QueryParams); + return HandleBatchRequest(Request, Policy); + } + if (std::all_of(begin(Key), end(Key), [](const char c) { return std::isalnum(c); })) { // Bucket reference @@ -872,6 +879,71 @@ HttpStructuredCacheService::ValidateKeyUri(HttpServerRequest& Request, CacheRef& return true; } +void +HttpStructuredCacheService::HandleBatchRequest(zen::HttpServerRequest& Request, CachePolicy Policy) +{ + ZEN_UNUSED(Policy); + + switch (auto Verb = Request.RequestVerb()) + { + using enum HttpVerb; + + case kGet: + { + const HttpContentType ContentType = Request.RequestContentType(); + const HttpContentType AcceptType = Request.AcceptContentType(); + + if (ContentType != HttpContentType::kCbObject || AcceptType != HttpContentType::kCbPackage) + { + return Request.WriteResponse(HttpResponseCode::BadRequest); + } + + CbObject BatchRequest = zen::LoadCompactBinaryObject(Request.ReadPayload()); + + CbPackage Package; + CbObjectWriter BatchResponse; + + BatchResponse.BeginArray("records"sv); + + for (CbFieldView QueryView : BatchRequest["records"sv]) + { + CbObjectView Query = QueryView.AsObjectView(); + const std::string_view Bucket = Query["bucket"sv].AsString(); + const IoHash CacheKey = Query["key"sv].AsHash(); + + ZenCacheValue CacheValue; + const bool Hit = m_CacheStore.Get(Bucket, CacheKey, CacheValue); + + if (Hit) + { + CbObjectView CacheRecord(CacheValue.Value.Data()); + + CacheRecord.IterateAttachments([this, &Package](CbFieldView AttachmentHash) { + if (IoBuffer Chunk = m_CidStore.FindChunkByCid(AttachmentHash.AsHash())) + { + Package.AddAttachment(CbAttachment(CompressedBuffer::FromCompressed(SharedBuffer(Chunk)))); + } + }); + + BatchResponse << CacheRecord; + } + } + + BatchResponse.EndArray(); + Package.SetObject(BatchResponse.Save()); + + BinaryWriter MemStream; + Package.Save(MemStream); + + Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, IoBuffer(IoBuffer::Wrap, MemStream.GetData(), MemStream.GetSize())); + } + break; + default: + Request.WriteResponse(HttpResponseCode::BadRequest); + break; + } +} + void HttpStructuredCacheService::HandleStatsRequest(zen::HttpServerRequest& Request) { diff --git a/zenserver/cache/structuredcache.h b/zenserver/cache/structuredcache.h index ad7253f79..d6b4944fd 100644 --- a/zenserver/cache/structuredcache.h +++ b/zenserver/cache/structuredcache.h @@ -89,6 +89,7 @@ private: void HandleCachePayloadRequest(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy Policy); void HandleGetCachePayload(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy Policy); void HandlePutCachePayload(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy Policy); + void HandleBatchRequest(zen::HttpServerRequest& Request, CachePolicy Policy); void HandleCacheBucketRequest(zen::HttpServerRequest& Request, std::string_view Bucket); virtual void HandleStatsRequest(zen::HttpServerRequest& Request) override; virtual void HandleStatusRequest(zen::HttpServerRequest& Request) override; -- cgit v1.2.3 From 20317e28bec68fcd11e00026cd8308aefc0ce58b Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Mon, 1 Nov 2021 10:45:26 +0100 Subject: Fixed possible undefined use error --- zenserver/cache/structuredcachestore.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'zenserver') diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index e1d77a088..45241eb68 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -160,7 +160,7 @@ ZenCacheMemoryLayer::Get(std::string_view InBucket, const IoHash& HashKey, ZenCa return false; } - CacheBucket* Bucket = Bucket = &it->second; + CacheBucket* Bucket = &it->second; _.ReleaseNow(); -- cgit v1.2.3 From 584855c4684204aa3cb0867ffb3fdbce61a936b3 Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Mon, 1 Nov 2021 12:44:12 +0100 Subject: Moved declaration of ZenDiskCacheLayer::CacheBucket in the .h GCC fails to compile if CacheBucket is only forward-declared within the class, issuing an "incomplete type" error in relation to the m_Buckets unordered map. --- zenserver/cache/structuredcachestore.cpp | 122 ++++++++----------------------- zenserver/cache/structuredcachestore.h | 86 +++++++++++++++++++++- 2 files changed, 116 insertions(+), 92 deletions(-) (limited to 'zenserver') diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index 45241eb68..119062833 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -18,7 +18,6 @@ #include #include #include -#include #include #include #include @@ -317,107 +316,48 @@ ZenCacheMemoryLayer::CacheBucket::Put(const IoHash& HashKey, const ZenCacheValue ////////////////////////////////////////////////////////////////////////// -#pragma pack(push) -#pragma pack(1) +inline DiskLocation::DiskLocation() = default; -struct DiskLocation +inline DiskLocation::DiskLocation(uint64_t Offset, uint64_t ValueSize, uint32_t IndexSize, uint64_t Flags) +: OffsetAndFlags(CombineOffsetAndFlags(Offset, Flags)) +, LowerSize(ValueSize & 0xFFFFffff) +, IndexDataSize(IndexSize) { - inline DiskLocation() = default; - - inline DiskLocation(uint64_t Offset, uint64_t ValueSize, uint32_t IndexSize, uint64_t Flags) - : OffsetAndFlags(CombineOffsetAndFlags(Offset, Flags)) - , LowerSize(ValueSize & 0xFFFFffff) - , IndexDataSize(IndexSize) - { - } - - static const uint64_t kOffsetMask = 0x0000'ffFF'ffFF'ffFFull; - static const uint64_t kSizeMask = 0x00FF'0000'0000'0000ull; - static const uint64_t kFlagsMask = 0xff00'0000'0000'0000ull; - static const uint64_t kStandaloneFile = 0x8000'0000'0000'0000ull; - static const uint64_t kStructured = 0x4000'0000'0000'0000ull; - static const uint64_t kTombStone = 0x2000'0000'0000'0000ull; - - static uint64_t CombineOffsetAndFlags(uint64_t Offset, uint64_t Flags) { return Offset | Flags; } +} - inline uint64_t Offset() const { return OffsetAndFlags & kOffsetMask; } - inline uint64_t Size() const { return LowerSize; } - inline uint64_t IsFlagSet(uint64_t Flag) const { return OffsetAndFlags & Flag; } - inline ZenContentType GetContentType() const - { - ZenContentType ContentType = ZenContentType::kBinary; +inline uint64_t DiskLocation::CombineOffsetAndFlags(uint64_t Offset, uint64_t Flags) +{ + return Offset | Flags; +} - if (IsFlagSet(DiskLocation::kStructured)) - { - ContentType = ZenContentType::kCbObject; - } +inline uint64_t DiskLocation::Offset() const +{ + return OffsetAndFlags & kOffsetMask; +} - return ContentType; - } +inline uint64_t DiskLocation::Size() const +{ + return LowerSize; +} -private: - uint64_t OffsetAndFlags = 0; - uint32_t LowerSize = 0; - uint32_t IndexDataSize = 0; -}; +inline uint64_t DiskLocation::IsFlagSet(uint64_t Flag) const +{ + return OffsetAndFlags & Flag; +} -struct DiskIndexEntry +inline ZenContentType DiskLocation::GetContentType() const { - IoHash Key; - DiskLocation Location; -}; + ZenContentType ContentType = ZenContentType::kBinary; -#pragma pack(pop) + if (IsFlagSet(DiskLocation::kStructured)) + { + ContentType = ZenContentType::kCbObject; + } -static_assert(sizeof(DiskIndexEntry) == 36); + return ContentType; +} -struct ZenCacheDiskLayer::CacheBucket -{ - CacheBucket(); - ~CacheBucket(); - - void OpenOrCreate(std::filesystem::path BucketDir, bool AllowCreate = true); - static bool Delete(std::filesystem::path BucketDir); - - bool Get(const IoHash& HashKey, ZenCacheValue& OutValue); - void Put(const IoHash& HashKey, const ZenCacheValue& Value); - void Drop(); - void Flush(); - void Scrub(ScrubContext& Ctx); - void GarbageCollect(GcContext& GcCtx); - - inline bool IsOk() const { return m_IsOk; } - -private: - std::filesystem::path m_BucketDir; - Oid m_BucketId; - bool m_IsOk = false; - uint64_t m_LargeObjectThreshold = 64 * 1024; - - // These files are used to manage storage of small objects for this bucket - - BasicFile m_SobsFile; - TCasLogFile m_SlogFile; - - RwLock m_IndexLock; - tsl::robin_map m_Index; - uint64_t m_WriteCursor = 0; - - void BuildPath(WideStringBuilderBase& Path, const IoHash& HashKey); - void PutStandaloneCacheValue(const IoHash& HashKey, const ZenCacheValue& Value); - bool GetStandaloneCacheValue(const DiskLocation& Loc, const IoHash& HashKey, ZenCacheValue& OutValue); - bool GetInlineCacheValue(const DiskLocation& Loc, ZenCacheValue& OutValue); - - // These locks are here to avoid contention on file creation, therefore it's sufficient - // that we take the same lock for the same hash - // - // These locks are small and should really be spaced out so they don't share cache lines, - // but we don't currently access them at particularly high frequency so it should not be - // an issue in practice - - RwLock m_ShardedLocks[256]; - inline RwLock& LockForHash(const IoHash& Hash) { return m_ShardedLocks[Hash.Hash[19]]; } -}; +////////////////////////////////////////////////////////////////////////// ZenCacheDiskLayer::CacheBucket::CacheBucket() { diff --git a/zenserver/cache/structuredcachestore.h b/zenserver/cache/structuredcachestore.h index 760f4995c..6521d8393 100644 --- a/zenserver/cache/structuredcachestore.h +++ b/zenserver/cache/structuredcachestore.h @@ -7,7 +7,9 @@ #include #include #include +#include #include +#include #pragma warning(push) #pragma warning(disable : 4127) @@ -98,6 +100,42 @@ private: Configuration m_Configuration; }; +#pragma pack(push) +#pragma pack(1) + +struct DiskLocation +{ + static const uint64_t kOffsetMask = 0x0000'ffFF'ffFF'ffFFull; + static const uint64_t kSizeMask = 0x00FF'0000'0000'0000ull; + static const uint64_t kFlagsMask = 0xff00'0000'0000'0000ull; + static const uint64_t kStandaloneFile = 0x8000'0000'0000'0000ull; + static const uint64_t kStructured = 0x4000'0000'0000'0000ull; + static const uint64_t kTombStone = 0x2000'0000'0000'0000ull; + + DiskLocation(); + DiskLocation(uint64_t Offset, uint64_t ValueSize, uint32_t IndexSize, uint64_t Flags); + static uint64_t CombineOffsetAndFlags(uint64_t Offset, uint64_t Flags); + uint64_t Offset() const; + uint64_t Size() const; + uint64_t IsFlagSet(uint64_t Flag) const; + ZenContentType GetContentType() const; + +private: + uint64_t OffsetAndFlags = 0; + uint32_t LowerSize = 0; + uint32_t IndexDataSize = 0; +}; + +struct DiskIndexEntry +{ + IoHash Key; + DiskLocation Location; +}; + +#pragma pack(pop) + +static_assert(sizeof(DiskIndexEntry) == 36); + class ZenCacheDiskLayer { public: @@ -117,7 +155,53 @@ private: /** A cache bucket manages a single directory containing metadata and data for that bucket */ - struct CacheBucket; + struct CacheBucket + { + CacheBucket(); + ~CacheBucket(); + + void OpenOrCreate(std::filesystem::path BucketDir, bool AllowCreate = true); + static bool Delete(std::filesystem::path BucketDir); + + bool Get(const IoHash& HashKey, ZenCacheValue& OutValue); + void Put(const IoHash& HashKey, const ZenCacheValue& Value); + void Drop(); + void Flush(); + void Scrub(ScrubContext& Ctx); + void GarbageCollect(GcContext& GcCtx); + + inline bool IsOk() const { return m_IsOk; } + + private: + std::filesystem::path m_BucketDir; + Oid m_BucketId; + bool m_IsOk = false; + uint64_t m_LargeObjectThreshold = 64 * 1024; + + // These files are used to manage storage of small objects for this bucket + + BasicFile m_SobsFile; + TCasLogFile m_SlogFile; + + RwLock m_IndexLock; + tsl::robin_map m_Index; + uint64_t m_WriteCursor = 0; + + void BuildPath(WideStringBuilderBase& Path, const IoHash& HashKey); + void PutStandaloneCacheValue(const IoHash& HashKey, const ZenCacheValue& Value); + bool GetStandaloneCacheValue(const DiskLocation& Loc, const IoHash& HashKey, ZenCacheValue& OutValue); + bool GetInlineCacheValue(const DiskLocation& Loc, ZenCacheValue& OutValue); + + // These locks are here to avoid contention on file creation, therefore it's sufficient + // that we take the same lock for the same hash + // + // These locks are small and should really be spaced out so they don't share cache lines, + // but we don't currently access them at particularly high frequency so it should not be + // an issue in practice + + RwLock m_ShardedLocks[256]; + inline RwLock& LockForHash(const IoHash& Hash) { return m_ShardedLocks[Hash.Hash[19]]; } + }; std::filesystem::path m_RootDir; RwLock m_Lock; -- cgit v1.2.3 From 934881f5572778e17b973d29dfb0ad760f29025c Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Mon, 1 Nov 2021 14:36:28 +0100 Subject: Upload cache record before blobs and call finalize when processing upstream to Jupiter. --- zenserver/diag/formatters.h | 20 ++-- zenserver/upstream/jupiter.cpp | 10 +- zenserver/upstream/upstreamcache.cpp | 195 ++++++++++++++++++----------------- 3 files changed, 124 insertions(+), 101 deletions(-) (limited to 'zenserver') diff --git a/zenserver/diag/formatters.h b/zenserver/diag/formatters.h index 42f928efe..759df58d3 100644 --- a/zenserver/diag/formatters.h +++ b/zenserver/diag/formatters.h @@ -2,6 +2,11 @@ #pragma once +#include +#include +#include +#include + ZEN_THIRD_PARTY_INCLUDES_START #include #include @@ -17,7 +22,7 @@ struct fmt::formatter { using namespace std::literals; - if (Response.status_code == 200) + if (Response.status_code == 200 || Response.status_code == 201) { return fmt::format_to(Ctx.out(), "Url: {}, Status: {}, Bytes: {}/{} (Up/Down), Elapsed: {}s", @@ -32,18 +37,21 @@ struct fmt::formatter const auto It = Response.header.find("Content-Type"); const std::string_view ContentType = It != Response.header.end() ? It->second : ""sv; - const bool IsBinary = ContentType == "application/x-ue-cb"sv || ContentType == "application/x-ue-comp"sv || - ContentType == "application/octet-stream"; - - if (IsBinary) + if (ContentType == "application/x-ue-cb"sv) { + zen::IoBuffer Body(zen::IoBuffer::Wrap, Response.text.data(), Response.text.size()); + zen::CbObjectView Obj(Body.Data()); + zen::ExtendableStringBuilder<256> Sb; + std::string_view Json = Obj.ToJson(Sb).ToView(); + return fmt::format_to(Ctx.out(), - "Url: {}, Status: {}, Bytes: {}/{} (Up/Down), Elapsed: {}s, Reason: '{}'", + "Url: {}, Status: {}, Bytes: {}/{} (Up/Down), Elapsed: {}s, Response: '{}', Reason: '{}'", Response.url.str(), Response.status_code, Response.uploaded_bytes, Response.downloaded_bytes, Response.elapsed, + Json, Response.reason); } else diff --git a/zenserver/upstream/jupiter.cpp b/zenserver/upstream/jupiter.cpp index 556a2124d..7a36b5841 100644 --- a/zenserver/upstream/jupiter.cpp +++ b/zenserver/upstream/jupiter.cpp @@ -87,12 +87,12 @@ CloudCacheSession::GetDerivedData(std::string_view BucketId, std::string_view Ke } ExtendableStringBuilder<256> Uri; - Uri << m_CacheClient->ServiceUrl() << "/api/v1/c/ddc/" << m_CacheClient->DdcNamespace() << "/" << BucketId << "/" << Key << ".raw"; + Uri << m_CacheClient->ServiceUrl() << "/api/v1/c/ddc/" << m_CacheClient->DdcNamespace() << "/" << BucketId << "/" << Key; cpr::Session& Session = m_SessionState->Session; Session.SetOption(cpr::Url{Uri.c_str()}); - Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}}); + Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}, {"Accept", "application/octet-stream"}}); cpr::Response Response = Session.Get(); ZEN_DEBUG("GET {}", Response); @@ -137,6 +137,7 @@ CloudCacheSession::GetRef(std::string_view BucketId, const IoHash& Key, ZenConte Session.SetOption(cpr::Url{Uri.c_str()}); Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}, {"Accept", ContentType}}); + Session.SetOption(cpr::Body{}); cpr::Response Response = Session.Get(); ZEN_DEBUG("GET {}", Response); @@ -172,6 +173,7 @@ CloudCacheSession::GetBlob(const IoHash& Key) Session.SetOption(cpr::Url{Uri.c_str()}); Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}, {"Accept", "application/octet-stream"}}); + Session.SetOption(cpr::Body{}); cpr::Response Response = Session.Get(); ZEN_DEBUG("GET {}", Response); @@ -203,6 +205,7 @@ CloudCacheSession::GetCompressedBlob(const IoHash& Key) Session.SetOption(cpr::Url{Uri.c_str()}); Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}, {"Accept", "application/x-ue-comp"}}); + Session.SetOption(cpr::Body{}); cpr::Response Response = Session.Get(); ZEN_DEBUG("GET {}", Response); @@ -234,6 +237,7 @@ CloudCacheSession::GetObject(const IoHash& Key) Session.SetOption(cpr::Url{Uri.c_str()}); Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}, {"Accept", "application/x-ue-cb"}}); + Session.SetOption(cpr::Body{}); cpr::Response Response = Session.Get(); ZEN_DEBUG("GET {}", Response); @@ -383,6 +387,7 @@ CloudCacheSession::FinalizeRef(std::string_view BucketId, const IoHash& Key, con Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}, {"X-Jupiter-IoHash", RefHash.ToHexString()}, {"Content-Type", "application/x-ue-cb"}}); + Session.SetBody(cpr::Body{}); cpr::Response Response = Session.Post(); ZEN_DEBUG("POST {}", Response); @@ -546,6 +551,7 @@ CloudCacheSession::RefExists(std::string_view BucketId, const IoHash& Key) Session.SetOption(cpr::Url{Uri.c_str()}); Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}}); + Session.SetOption(cpr::Body{}); cpr::Response Response = Session.Head(); ZEN_DEBUG("HEAD {}", Response); diff --git a/zenserver/upstream/upstreamcache.cpp b/zenserver/upstream/upstreamcache.cpp index 168449d05..c0cd858b6 100644 --- a/zenserver/upstream/upstreamcache.cpp +++ b/zenserver/upstream/upstreamcache.cpp @@ -175,8 +175,10 @@ namespace detail { IoBuffer RecordValue, std::span Payloads) override { + using namespace fmt::literals; + ZEN_ASSERT(CacheRecord.PayloadIds.size() == Payloads.size()); - const uint32_t MaxAttempts = 3; + const size_t MaxAttempts = 3; try { @@ -204,117 +206,112 @@ namespace detail { } else { - bool Success = false; int64_t TotalBytes = 0ull; double TotalElapsedSeconds = 0.0; - for (size_t Idx = 0, Count = Payloads.size(); Idx < Count; Idx++) - { - Success = false; - for (int32_t Attempt = 0; Attempt < MaxAttempts; Attempt++) + const auto PutBlobs = [&](std::span PayloadIds, std::string& OutReason) -> bool { + for (const IoHash& PayloadId : PayloadIds) { - if (CloudCacheResult Result = Session.PutCompressedBlob(CacheRecord.PayloadIds[Idx], Payloads[Idx]); - Result.Success) + const auto It = std::find(std::begin(CacheRecord.PayloadIds), std::end(CacheRecord.PayloadIds), PayloadId); + + if (It == std::end(CacheRecord.PayloadIds)) { - TotalBytes += Result.Bytes; - TotalElapsedSeconds += Result.ElapsedSeconds; - Success = true; - break; + OutReason = "payload '{}' MISSING from local cache"_format(PayloadId); + return false; } - } - if (!Success) - { - return {.Reason = "Failed to upload payload", - .Bytes = TotalBytes, - .ElapsedSeconds = TotalElapsedSeconds, - .Success = false}; + const size_t Idx = std::distance(std::begin(CacheRecord.PayloadIds), It); + + CloudCacheResult BlobResult; + for (size_t Attempt = 0; Attempt < MaxAttempts && !BlobResult.Success; Attempt++) + { + BlobResult = Session.PutCompressedBlob(CacheRecord.PayloadIds[Idx], Payloads[Idx]); + } + + if (!BlobResult.Success) + { + OutReason = "upload payload '{}' FAILED, reason '{}'"_format(PayloadId, BlobResult.Reason); + return false; + } + + TotalBytes += BlobResult.Bytes; + TotalElapsedSeconds += BlobResult.ElapsedSeconds; } + + return true; + }; + + PutRefResult RefResult; + for (int32_t Attempt = 0; Attempt < MaxAttempts && !RefResult.Success; Attempt++) + { + RefResult = + Session.PutRef(CacheRecord.CacheKey.Bucket, CacheRecord.CacheKey.Hash, RecordValue, ZenContentType::kCbObject); } - Success = false; - for (int32_t Attempt = 0; Attempt < MaxAttempts; Attempt++) + if (!RefResult.Success) { - if (PutRefResult Result = Session.PutRef(CacheRecord.CacheKey.Bucket, - CacheRecord.CacheKey.Hash, - RecordValue, - ZenContentType::kCbObject); - Result.Success) - { - TotalBytes += Result.Bytes; - TotalElapsedSeconds += Result.ElapsedSeconds; - Success = true; + return {.Reason = "upload cache record '{}/{}' FAILED, reason '{}'"_format(CacheRecord.CacheKey.Bucket, + CacheRecord.CacheKey.Hash, + RefResult.Reason), + .Success = false}; + } - if (!Result.Needs.empty()) - { - for (const IoHash& NeededHash : Result.Needs) - { - Success = false; - - if (auto It = - std::find(std::begin(CacheRecord.PayloadIds), std::end(CacheRecord.PayloadIds), NeededHash); - It != std::end(CacheRecord.PayloadIds)) - { - const size_t Idx = It - std::begin(CacheRecord.PayloadIds); - - if (CloudCacheResult BlobResult = - Session.PutCompressedBlob(CacheRecord.PayloadIds[Idx], Payloads[Idx]); - BlobResult.Success) - { - TotalBytes += BlobResult.Bytes; - TotalElapsedSeconds += BlobResult.ElapsedSeconds; - Success = true; - } - else - { - ZEN_WARN("upload missing payload '{}/{}/{}' FAILED", - CacheRecord.CacheKey.Bucket, - CacheRecord.CacheKey.Hash, - NeededHash); - } - } - else - { - ZEN_WARN("needed payload '{}/{}/{}' MISSING", - CacheRecord.CacheKey.Bucket, - CacheRecord.CacheKey.Hash, - NeededHash); - } - } + TotalBytes += RefResult.Bytes; + TotalElapsedSeconds += RefResult.ElapsedSeconds; - const IoHash RefHash = IoHash::HashBuffer(RecordValue); + std::string Reason; + if (!PutBlobs(RefResult.Needs, Reason)) + { + return {.Reason = std::move(Reason), .Success = false}; + } - if (FinalizeRefResult FinalizeResult = - Session.FinalizeRef(CacheRecord.CacheKey.Bucket, CacheRecord.CacheKey.Hash, RefHash); - FinalizeResult.Success) - { - TotalBytes += FinalizeResult.Bytes; - TotalElapsedSeconds += FinalizeResult.ElapsedSeconds; - Success = true; - - for (const IoHash& MissingHash : FinalizeResult.Needs) - { - ZEN_WARN("finalize '{}/{}' FAILED, missing '{}'", - CacheRecord.CacheKey.Bucket, - CacheRecord.CacheKey.Hash, - MissingHash); - } - } - else - { - ZEN_WARN("finalize '{}/{}' FAILED", CacheRecord.CacheKey.Bucket, CacheRecord.CacheKey.Hash); - Success = false; - } - } + const IoHash RefHash = IoHash::HashBuffer(RecordValue); + FinalizeRefResult FinalizeResult = Session.FinalizeRef(CacheRecord.CacheKey.Bucket, CacheRecord.CacheKey.Hash, RefHash); + + if (!FinalizeResult.Success) + { + return {.Reason = "finalize cache record '{}/{}' FAILED, reason '{}'"_format(CacheRecord.CacheKey.Bucket, + CacheRecord.CacheKey.Hash, + FinalizeResult.Reason), + .Success = false}; + } + + if (!FinalizeResult.Needs.empty()) + { + if (!PutBlobs(FinalizeResult.Needs, Reason)) + { + return {.Reason = std::move(Reason), .Success = false}; + } + + FinalizeResult = Session.FinalizeRef(CacheRecord.CacheKey.Bucket, CacheRecord.CacheKey.Hash, RefHash); + + if (!FinalizeResult.Success) + { + return {.Reason = "finalize '{}/{}' FAILED, reason '{}'"_format(CacheRecord.CacheKey.Bucket, + CacheRecord.CacheKey.Hash, + FinalizeResult.Reason), + .Success = false}; + } - if (Success) + if (!FinalizeResult.Needs.empty()) + { + ExtendableStringBuilder<256> Sb; + for (const IoHash& MissingHash : FinalizeResult.Needs) { - break; + Sb << MissingHash.ToHexString() << ","; } + + return {.Reason = "finalize '{}/{}' FAILED, still needs payload(s) '{}'"_format(CacheRecord.CacheKey.Bucket, + CacheRecord.CacheKey.Hash, + Sb.ToString()), + .Success = false}; } } - return {.Bytes = TotalBytes, .ElapsedSeconds = TotalElapsedSeconds, .Success = Success}; + TotalBytes += FinalizeResult.Bytes; + TotalElapsedSeconds += FinalizeResult.ElapsedSeconds; + + return {.Bytes = TotalBytes, .ElapsedSeconds = TotalElapsedSeconds, .Success = true}; } } catch (std::exception& Err) @@ -890,6 +887,15 @@ private: { const PutUpstreamCacheResult Result = Endpoint->PutCacheRecord(CacheRecord, CacheValue.Value, std::span(Payloads)); m_Stats.Add(m_Log, *Endpoint, Result, m_Endpoints); + + if (!Result.Success) + { + ZEN_WARN("upload cache record '{}/{}' FAILED, endpoint '{}', reason '{}'", + CacheRecord.CacheKey.Bucket, + CacheRecord.CacheKey.Hash, + Endpoint->DisplayName(), + Result.Reason); + } } } } @@ -907,7 +913,10 @@ private: } catch (std::exception& e) { - ZEN_WARN("process upstream ({}/{}) FAILED '{}'", CacheRecord.CacheKey.Bucket, CacheRecord.CacheKey.Hash, e.what()); + ZEN_WARN("upload cache record '{}/{}' FAILED, reason '{}'", + CacheRecord.CacheKey.Bucket, + CacheRecord.CacheKey.Hash, + e.what()); } } -- cgit v1.2.3 From 4b8d4c0e375c729e38bdaadfebf0eaf14f08f5f9 Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Tue, 2 Nov 2021 10:50:18 +0100 Subject: Added upstream batch API. --- zenserver/cache/cachekey.cpp | 105 ++++++++++++++++ zenserver/cache/cachekey.h | 52 ++++++++ zenserver/cache/structuredcache.cpp | 225 ++++++++++++++++------------------- zenserver/upstream/upstreamcache.cpp | 41 +++++++ zenserver/upstream/upstreamcache.h | 14 +++ zenserver/zenserver.vcxproj | 2 + zenserver/zenserver.vcxproj.filters | 6 + 7 files changed, 323 insertions(+), 122 deletions(-) create mode 100644 zenserver/cache/cachekey.cpp create mode 100644 zenserver/cache/cachekey.h (limited to 'zenserver') diff --git a/zenserver/cache/cachekey.cpp b/zenserver/cache/cachekey.cpp new file mode 100644 index 000000000..94ef7fd12 --- /dev/null +++ b/zenserver/cache/cachekey.cpp @@ -0,0 +1,105 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "cachekey.h" + +#include + +namespace zen { + +using namespace std::literals; + +namespace detail { namespace cacheopt { + constexpr std::string_view Local = "local"sv; + constexpr std::string_view Remote = "remote"sv; + constexpr std::string_view Data = "data"sv; + constexpr std::string_view Meta = "meta"sv; + constexpr std::string_view Value = "value"sv; + constexpr std::string_view Attachments = "attachments"sv; +}} // namespace detail::cacheopt + +CachePolicy +ParseQueryCachePolicy(std::string_view QueryPolicy, CachePolicy Default) +{ + if (QueryPolicy.empty()) + { + return Default; + } + + CachePolicy Result = CachePolicy::None; + + ForEachStrTok(QueryPolicy, ',', [&Result](const std::string_view& Token) { + if (Token == detail::cacheopt::Local) + { + Result |= CachePolicy::QueryLocal; + } + if (Token == detail::cacheopt::Remote) + { + Result |= CachePolicy::QueryRemote; + } + return true; + }); + + return Result; +} + +CachePolicy +ParseStoreCachePolicy(std::string_view StorePolicy, CachePolicy Default) +{ + if (StorePolicy.empty()) + { + return Default; + } + + CachePolicy Result = CachePolicy::None; + + ForEachStrTok(StorePolicy, ',', [&Result](const std::string_view& Token) { + if (Token == detail::cacheopt::Local) + { + Result |= CachePolicy::StoreLocal; + } + if (Token == detail::cacheopt::Remote) + { + Result |= CachePolicy::StoreRemote; + } + return true; + }); + + return Result; +} + +CachePolicy +ParseSkipCachePolicy(std::string_view SkipPolicy, CachePolicy Default) +{ + if (SkipPolicy.empty()) + { + return Default; + } + + CachePolicy Result = CachePolicy::None; + + ForEachStrTok(SkipPolicy, ',', [&Result](const std::string_view& Token) { + if (Token == detail::cacheopt::Meta) + { + Result |= CachePolicy::SkipMeta; + } + if (Token == detail::cacheopt::Value) + { + Result |= CachePolicy::SkipValue; + } + if (Token == detail::cacheopt::Attachments) + { + Result |= CachePolicy::SkipAttachments; + } + if (Token == detail::cacheopt::Data) + { + Result |= CachePolicy::SkipData; + } + return true; + }); + + return Result; +} + +CacheKey CacheKey::None = CacheKey{.Bucket = std::string(), .Hash = IoHash()}; + +} // namespace zen diff --git a/zenserver/cache/cachekey.h b/zenserver/cache/cachekey.h new file mode 100644 index 000000000..eba063699 --- /dev/null +++ b/zenserver/cache/cachekey.h @@ -0,0 +1,52 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include +#include + +namespace zen { + +enum class CachePolicy : uint8_t +{ + None = 0, + QueryLocal = 1 << 0, + QueryRemote = 1 << 1, + Query = QueryLocal | QueryRemote, + StoreLocal = 1 << 2, + StoreRemote = 1 << 3, + Store = StoreLocal | StoreRemote, + SkipMeta = 1 << 4, + SkipValue = 1 << 5, + SkipAttachments = 1 << 6, + SkipData = SkipMeta | SkipValue | SkipAttachments, + SkipLocalCopy = 1 << 7, + Local = QueryLocal | StoreLocal, + Remote = QueryRemote | StoreRemote, + Default = Query | Store, + Disable = None, +}; + +gsl_DEFINE_ENUM_BITMASK_OPERATORS(CachePolicy); + +CachePolicy ParseQueryCachePolicy(std::string_view QueryPolicy, CachePolicy Default = CachePolicy::Query); + +CachePolicy ParseStoreCachePolicy(std::string_view StorePolicy, CachePolicy Default = CachePolicy::Store); + +CachePolicy ParseSkipCachePolicy(std::string_view SkipPolicy, CachePolicy Default = CachePolicy::None); + +struct CacheKey +{ + std::string Bucket; + IoHash Hash; + + static CacheKey Create(std::string_view Bucket, const IoHash& Hash) + { + return {.Bucket = ToLower(Bucket), .Hash = Hash}; + } + + static CacheKey None; +}; + +} // namespace zen diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 172e122e4..59cc74d53 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -12,6 +12,7 @@ #include #include +#include "cachekey.h" #include "monitoring/httpstats.h" #include "structuredcache.h" #include "structuredcachestore.h" @@ -36,111 +37,12 @@ using namespace std::literals; ////////////////////////////////////////////////////////////////////////// -namespace detail { namespace cacheopt { - constexpr std::string_view Local = "local"sv; - constexpr std::string_view Remote = "remote"sv; - constexpr std::string_view Data = "data"sv; - constexpr std::string_view Meta = "meta"sv; - constexpr std::string_view Value = "value"sv; - constexpr std::string_view Attachments = "attachments"sv; -}} // namespace detail::cacheopt - -////////////////////////////////////////////////////////////////////////// - -enum class CachePolicy : uint8_t -{ - None = 0, - QueryLocal = 1 << 0, - QueryRemote = 1 << 1, - Query = QueryLocal | QueryRemote, - StoreLocal = 1 << 2, - StoreRemote = 1 << 3, - Store = StoreLocal | StoreRemote, - SkipMeta = 1 << 4, - SkipValue = 1 << 5, - SkipAttachments = 1 << 6, - SkipData = SkipMeta | SkipValue | SkipAttachments, - SkipLocalCopy = 1 << 7, - Local = QueryLocal | StoreLocal, - Remote = QueryRemote | StoreRemote, - Default = Query | Store, - Disable = None, -}; - -gsl_DEFINE_ENUM_BITMASK_OPERATORS(CachePolicy); - CachePolicy ParseCachePolicy(const HttpServerRequest::QueryParams& QueryParams) { - CachePolicy QueryPolicy = CachePolicy::Query; - - { - std::string_view Opts = QueryParams.GetValue("query"sv); - if (!Opts.empty()) - { - QueryPolicy = CachePolicy::None; - ForEachStrTok(Opts, ',', [&QueryPolicy](const std::string_view& Opt) { - if (Opt == detail::cacheopt::Local) - { - QueryPolicy |= CachePolicy::QueryLocal; - } - if (Opt == detail::cacheopt::Remote) - { - QueryPolicy |= CachePolicy::QueryRemote; - } - return true; - }); - } - } - - CachePolicy StorePolicy = CachePolicy::Store; - - { - std::string_view Opts = QueryParams.GetValue("store"sv); - if (!Opts.empty()) - { - StorePolicy = CachePolicy::None; - ForEachStrTok(Opts, ',', [&StorePolicy](const std::string_view& Opt) { - if (Opt == detail::cacheopt::Local) - { - StorePolicy |= CachePolicy::StoreLocal; - } - if (Opt == detail::cacheopt::Remote) - { - StorePolicy |= CachePolicy::StoreRemote; - } - return true; - }); - } - } - - CachePolicy SkipPolicy = CachePolicy::None; - - { - std::string_view Opts = QueryParams.GetValue("skip"sv); - if (!Opts.empty()) - { - ForEachStrTok(Opts, ',', [&SkipPolicy](const std::string_view& Opt) { - if (Opt == detail::cacheopt::Meta) - { - SkipPolicy |= CachePolicy::SkipMeta; - } - if (Opt == detail::cacheopt::Value) - { - SkipPolicy |= CachePolicy::SkipValue; - } - if (Opt == detail::cacheopt::Attachments) - { - SkipPolicy |= CachePolicy::SkipAttachments; - } - if (Opt == detail::cacheopt::Data) - { - SkipPolicy |= CachePolicy::SkipData; - } - return true; - }); - } - } + const CachePolicy QueryPolicy = zen::ParseQueryCachePolicy(QueryParams.GetValue("query"sv)); + const CachePolicy StorePolicy = zen::ParseStoreCachePolicy(QueryParams.GetValue("store"sv)); + const CachePolicy SkipPolicy = zen::ParseSkipCachePolicy(QueryParams.GetValue("skip"sv)); return QueryPolicy | StorePolicy | SkipPolicy; } @@ -882,7 +784,11 @@ HttpStructuredCacheService::ValidateKeyUri(HttpServerRequest& Request, CacheRef& void HttpStructuredCacheService::HandleBatchRequest(zen::HttpServerRequest& Request, CachePolicy Policy) { - ZEN_UNUSED(Policy); + using namespace fmt::literals; + + const bool SkipData = (Policy & CachePolicy::SkipData) == CachePolicy::SkipData; + const bool SkipAttachments = (Policy & CachePolicy::SkipAttachments) == CachePolicy::SkipAttachments; + const bool QueryUpstream = m_UpstreamCache && ((Policy & CachePolicy::QueryRemote) == CachePolicy::QueryRemote); switch (auto Verb = Request.RequestVerb()) { @@ -898,44 +804,119 @@ HttpStructuredCacheService::HandleBatchRequest(zen::HttpServerRequest& Request, return Request.WriteResponse(HttpResponseCode::BadRequest); } - CbObject BatchRequest = zen::LoadCompactBinaryObject(Request.ReadPayload()); + CbObject BatchRequest = zen::LoadCompactBinaryObject(Request.ReadPayload()); + const std::string_view Method = BatchRequest["method"sv].AsString(); - CbPackage Package; - CbObjectWriter BatchResponse; + if (Method != "getcacherecords"sv) + { + return Request.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Invalid method '{}'"_format(Method)); + } - BatchResponse.BeginArray("records"sv); + CbObjectView Params = BatchRequest["params"sv].AsObjectView(); - for (CbFieldView QueryView : BatchRequest["records"sv]) + std::vector CacheKeys; + std::vector CacheValues; + std::vector Payloads; + std::vector Missing; + + for (CbFieldView QueryView : Params["cachekeys"sv]) { - CbObjectView Query = QueryView.AsObjectView(); - const std::string_view Bucket = Query["bucket"sv].AsString(); - const IoHash CacheKey = Query["key"sv].AsHash(); + CbObjectView Query = QueryView.AsObjectView(); + CacheKeys.push_back(CacheKey::Create(Query["bucket"sv].AsString(), Query["key"sv].AsHash())); + } - ZenCacheValue CacheValue; - const bool Hit = m_CacheStore.Get(Bucket, CacheKey, CacheValue); + CacheValues.resize(CacheKeys.size()); - if (Hit) + for (size_t Idx = 0; const CacheKey& Key : CacheKeys) + { + ZenCacheValue CacheValue; + if (m_CacheStore.Get(Key.Bucket, Key.Hash, CacheValue)) { CbObjectView CacheRecord(CacheValue.Value.Data()); - CacheRecord.IterateAttachments([this, &Package](CbFieldView AttachmentHash) { - if (IoBuffer Chunk = m_CidStore.FindChunkByCid(AttachmentHash.AsHash())) + if (!SkipAttachments) + { + CacheRecord.IterateAttachments([this, &Payloads](CbFieldView AttachmentHash) { + if (IoBuffer Chunk = m_CidStore.FindChunkByCid(AttachmentHash.AsHash())) + { + Payloads.push_back(Chunk); + } + }); + } + + CacheValues[Idx] = CacheValue.Value; + } + else + { + Missing.push_back(Idx); + } + + ++Idx; + } + + if (QueryUpstream) + { + auto UpstreamResult = m_UpstreamCache->GetCacheRecords( + CacheKeys, + Missing, + [this, &CacheValues, &Payloads, SkipAttachments](size_t KeyIndex, IoBuffer UpstreamValue) { + CbPackage UpstreamPackage; + if (UpstreamValue && UpstreamPackage.TryLoad(UpstreamValue)) { - Package.AddAttachment(CbAttachment(CompressedBuffer::FromCompressed(SharedBuffer(Chunk)))); + CbObjectView CacheRecord = UpstreamPackage.GetObject(); + + CacheRecord.IterateAttachments([&](CbFieldView AttachmentHash) { + if (const CbAttachment* Attachment = UpstreamPackage.FindAttachment(AttachmentHash.AsHash())) + { + if (CompressedBuffer Chunk = Attachment->AsCompressedBinary()) + { + m_CidStore.AddChunk(Chunk); + + if (!SkipAttachments) + { + Payloads.push_back(Chunk.GetCompressed().Flatten().AsIoBuffer()); + } + } + } + }); + + CacheValues[KeyIndex] = IoBufferBuilder::MakeCloneFromMemory(CacheRecord.GetView()); } }); + } + + CbObjectWriter BatchResponse; - BatchResponse << CacheRecord; + BatchResponse.BeginArray("result"sv); + for (const IoBuffer& Value : CacheValues) + { + if (Value) + { + BatchResponse << CbObjectView(Value.Data()); + } + else + { + BatchResponse.AddNull(); } } - BatchResponse.EndArray(); + + CbPackage Package; Package.SetObject(BatchResponse.Save()); - + + for (const IoBuffer& Payload : Payloads) + { + Package.AddAttachment(CbAttachment(CompressedBuffer::FromCompressed(SharedBuffer(Payload)))); + } + BinaryWriter MemStream; Package.Save(MemStream); - - Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, IoBuffer(IoBuffer::Wrap, MemStream.GetData(), MemStream.GetSize())); + + Request.WriteResponse(HttpResponseCode::OK, + HttpContentType::kCbPackage, + IoBuffer(IoBuffer::Wrap, MemStream.GetData(), MemStream.GetSize())); } break; default: diff --git a/zenserver/upstream/upstreamcache.cpp b/zenserver/upstream/upstreamcache.cpp index c0cd858b6..437b29cd7 100644 --- a/zenserver/upstream/upstreamcache.cpp +++ b/zenserver/upstream/upstreamcache.cpp @@ -780,6 +780,47 @@ public: return {}; } + virtual GetUpstreamCacheBatchResult GetCacheRecords(std::span CacheKeys, + std::span KeyIndex, + OnCacheGetComplete OnComplete) override + { + if (!m_Options.ReadUpstream) + { + return {.Missing = std::vector(KeyIndex.begin(), KeyIndex.end())}; + } + + GetUpstreamCacheBatchResult Result; + + for (size_t Idx : KeyIndex) + { + const UpstreamCacheKey CacheKey = {CacheKeys[Idx].Bucket, CacheKeys[Idx].Hash}; + + GetUpstreamCacheResult CacheResult; + for (auto& Endpoint : m_Endpoints) + { + if (Endpoint->IsHealthy()) + { + CacheResult = Endpoint->GetCacheRecord(CacheKey, ZenContentType::kCbPackage); + if (CacheResult.Success) + { + break; + } + } + } + + if (CacheResult.Success) + { + OnComplete(Idx, CacheResult.Value); + } + else + { + Result.Missing.push_back(Idx); + } + } + + return Result; + } + virtual GetUpstreamCacheResult GetCachePayload(UpstreamPayloadKey PayloadKey) override { if (m_Options.ReadUpstream) diff --git a/zenserver/upstream/upstreamcache.h b/zenserver/upstream/upstreamcache.h index edc995da6..04554f210 100644 --- a/zenserver/upstream/upstreamcache.h +++ b/zenserver/upstream/upstreamcache.h @@ -2,12 +2,15 @@ #pragma once +#include "cache/cachekey.h" + #include #include #include #include #include +#include #include namespace zen { @@ -62,6 +65,11 @@ struct GetUpstreamCacheResult bool Success = false; }; +struct GetUpstreamCacheBatchResult +{ + std::vector Missing; +}; + struct PutUpstreamCacheResult { std::string Reason; @@ -88,6 +96,8 @@ struct UpstreamEndpointStats std::atomic SecondsDown{}; }; +using OnCacheGetComplete = std::function; + /** * The upstream endpont is responsible for handling upload/downloading of cache records. */ @@ -129,6 +139,10 @@ public: virtual GetUpstreamCacheResult GetCacheRecord(UpstreamCacheKey CacheKey, ZenContentType Type) = 0; + virtual GetUpstreamCacheBatchResult GetCacheRecords(std::span CacheKeys, + std::span KeyIndex, + OnCacheGetComplete OnComplete) = 0; + virtual GetUpstreamCacheResult GetCachePayload(UpstreamPayloadKey PayloadKey) = 0; struct EnqueueResult diff --git a/zenserver/zenserver.vcxproj b/zenserver/zenserver.vcxproj index d954d3f8d..d90dd009a 100644 --- a/zenserver/zenserver.vcxproj +++ b/zenserver/zenserver.vcxproj @@ -105,6 +105,7 @@ + @@ -131,6 +132,7 @@ + diff --git a/zenserver/zenserver.vcxproj.filters b/zenserver/zenserver.vcxproj.filters index 04c6267ba..ae5411afb 100644 --- a/zenserver/zenserver.vcxproj.filters +++ b/zenserver/zenserver.vcxproj.filters @@ -41,6 +41,9 @@ + + cache + @@ -76,6 +79,9 @@ + + cache + -- cgit v1.2.3 From c32504578c67f5edf0faf4a458750f67ae53ab3a Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Wed, 3 Nov 2021 13:16:35 +0100 Subject: Type consistency around signed/unsigned comparison --- zenserver/upstream/upstreamcache.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'zenserver') diff --git a/zenserver/upstream/upstreamcache.cpp b/zenserver/upstream/upstreamcache.cpp index c0cd858b6..a260cb40d 100644 --- a/zenserver/upstream/upstreamcache.cpp +++ b/zenserver/upstream/upstreamcache.cpp @@ -178,7 +178,7 @@ namespace detail { using namespace fmt::literals; ZEN_ASSERT(CacheRecord.PayloadIds.size() == Payloads.size()); - const size_t MaxAttempts = 3; + const int32_t MaxAttempts = 3; try { @@ -223,7 +223,7 @@ namespace detail { const size_t Idx = std::distance(std::begin(CacheRecord.PayloadIds), It); CloudCacheResult BlobResult; - for (size_t Attempt = 0; Attempt < MaxAttempts && !BlobResult.Success; Attempt++) + for (int32_t Attempt = 0; Attempt < MaxAttempts && !BlobResult.Success; Attempt++) { BlobResult = Session.PutCompressedBlob(CacheRecord.PayloadIds[Idx], Payloads[Idx]); } @@ -479,7 +479,7 @@ namespace detail { std::span Payloads) override { ZEN_ASSERT(CacheRecord.PayloadIds.size() == Payloads.size()); - const uint32_t MaxAttempts = 3; + const int32_t MaxAttempts = 3; try { -- cgit v1.2.3 From 4f503210dca72fdaeee61693626ef6085e93e030 Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Mon, 8 Nov 2021 16:00:10 +0100 Subject: Added batched get chunk(s). --- zenserver/cache/cachekey.h | 18 +- zenserver/cache/structuredcache.cpp | 353 +++++++++++++++++++++++++----------- zenserver/cache/structuredcache.h | 2 + 3 files changed, 263 insertions(+), 110 deletions(-) (limited to 'zenserver') diff --git a/zenserver/cache/cachekey.h b/zenserver/cache/cachekey.h index eba063699..012a01292 100644 --- a/zenserver/cache/cachekey.h +++ b/zenserver/cache/cachekey.h @@ -4,6 +4,7 @@ #include #include +#include #include namespace zen { @@ -40,13 +41,18 @@ struct CacheKey { std::string Bucket; IoHash Hash; - - static CacheKey Create(std::string_view Bucket, const IoHash& Hash) - { - return {.Bucket = ToLower(Bucket), .Hash = Hash}; - } - + + static CacheKey Create(std::string_view Bucket, const IoHash& Hash) { return {.Bucket = ToLower(Bucket), .Hash = Hash}; } + static CacheKey None; }; +struct CacheChunk +{ + CacheKey Key; + IoHash Id; + uint64_t RawOffset = 0ull; + uint64_t RawSize = ~uint64_t(0); +}; + } // namespace zen diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 59cc74d53..a1b0e1549 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -231,8 +231,8 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request if (ValidCount != AttachmentCount) { - Success = false; - ZEN_WARN("GET - '{}/{}' '{}' FAILED, found '{}' of '{}' attachments", + // Success = false; + ZEN_WARN("GET - '{}/{}' '{}' is partial, found '{}' of '{}' attachments", Ref.BucketSegment, Ref.HashKey, ToString(AcceptType), @@ -554,12 +554,12 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request } }); - const bool AttachmentsValid = ValidAttachments.size() == Attachments.size(); + const bool IsPartialCacheRecord = ValidAttachments.size() != Attachments.size(); - if (!AttachmentsValid) - { - return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid attachments"sv); - } + // if (!AttachmentsValid) + //{ + // return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid attachments"sv); + //} ZEN_DEBUG("PUT - '{}/{}' {} '{}', {}/{} new attachments", Ref.BucketSegment, @@ -574,7 +574,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request m_CacheStore.Put(Ref.BucketSegment, Ref.HashKey, {.Value = CacheRecord.GetBuffer().AsIoBuffer()}); - if (StoreUpstream) + if (StoreUpstream && !IsPartialCacheRecord) { ZEN_ASSERT(m_UpstreamCache); auto Result = m_UpstreamCache->EnqueueUpstream({.Type = ZenContentType::kCbPackage, @@ -786,15 +786,11 @@ HttpStructuredCacheService::HandleBatchRequest(zen::HttpServerRequest& Request, { using namespace fmt::literals; - const bool SkipData = (Policy & CachePolicy::SkipData) == CachePolicy::SkipData; - const bool SkipAttachments = (Policy & CachePolicy::SkipAttachments) == CachePolicy::SkipAttachments; - const bool QueryUpstream = m_UpstreamCache && ((Policy & CachePolicy::QueryRemote) == CachePolicy::QueryRemote); - switch (auto Verb = Request.RequestVerb()) { using enum HttpVerb; - case kGet: + case kPost: { const HttpContentType ContentType = Request.RequestContentType(); const HttpContentType AcceptType = Request.AcceptContentType(); @@ -807,122 +803,271 @@ HttpStructuredCacheService::HandleBatchRequest(zen::HttpServerRequest& Request, CbObject BatchRequest = zen::LoadCompactBinaryObject(Request.ReadPayload()); const std::string_view Method = BatchRequest["method"sv].AsString(); - if (Method != "getcacherecords"sv) + if (Method == "get-cache-records"sv) { - return Request.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - "Invalid method '{}'"_format(Method)); + HandleBatchGetCacheRecords(Request, BatchRequest, Policy); } + else if (Method == "get-cache-chunks"sv) + { + HandleBatchGetCacheChunks(Request, BatchRequest, Policy); + } + else + { + Request.WriteResponse(HttpResponseCode::BadRequest); + } + } + break; + default: + Request.WriteResponse(HttpResponseCode::BadRequest); + break; + } +} - CbObjectView Params = BatchRequest["params"sv].AsObjectView(); +void +HttpStructuredCacheService::HandleBatchGetCacheRecords(zen::HttpServerRequest& Request, CbObjectView BatchRequest, CachePolicy Policy) +{ + using namespace fmt::literals; - std::vector CacheKeys; - std::vector CacheValues; - std::vector Payloads; - std::vector Missing; + const bool SkipData = (Policy & CachePolicy::SkipData) == CachePolicy::SkipData; + const bool SkipAttachments = (Policy & CachePolicy::SkipAttachments) == CachePolicy::SkipAttachments; + const bool QueryUpstream = m_UpstreamCache && ((Policy & CachePolicy::QueryRemote) == CachePolicy::QueryRemote); - for (CbFieldView QueryView : Params["cachekeys"sv]) - { - CbObjectView Query = QueryView.AsObjectView(); - CacheKeys.push_back(CacheKey::Create(Query["bucket"sv].AsString(), Query["key"sv].AsHash())); - } + const std::string_view Method = BatchRequest["method"sv].AsString(); + ZEN_ASSERT(Method == "get-cache-records"sv); - CacheValues.resize(CacheKeys.size()); + CbObjectView Params = BatchRequest["params"sv].AsObjectView(); - for (size_t Idx = 0; const CacheKey& Key : CacheKeys) - { - ZenCacheValue CacheValue; - if (m_CacheStore.Get(Key.Bucket, Key.Hash, CacheValue)) - { - CbObjectView CacheRecord(CacheValue.Value.Data()); + std::vector CacheKeys; + std::vector CacheValues; + std::vector Payloads; + std::vector Missing; - if (!SkipAttachments) - { - CacheRecord.IterateAttachments([this, &Payloads](CbFieldView AttachmentHash) { - if (IoBuffer Chunk = m_CidStore.FindChunkByCid(AttachmentHash.AsHash())) - { - Payloads.push_back(Chunk); - } - }); - } + for (CbFieldView QueryView : Params["cachekeys"sv]) + { + CbObjectView Query = QueryView.AsObjectView(); + CacheKeys.push_back(CacheKey::Create(Query["bucket"sv].AsString(), Query["hash"sv].AsHash())); + } - CacheValues[Idx] = CacheValue.Value; - } - else + CacheValues.resize(CacheKeys.size()); + + for (size_t Idx = 0; const CacheKey& Key : CacheKeys) + { + ZenCacheValue CacheValue; + if (m_CacheStore.Get(Key.Bucket, Key.Hash, CacheValue)) + { + CbObjectView CacheRecord(CacheValue.Value.Data()); + + if (!SkipAttachments) + { + CacheRecord.IterateAttachments([this, &Payloads](CbFieldView AttachmentHash) { + if (IoBuffer Chunk = m_CidStore.FindChunkByCid(AttachmentHash.AsHash())) { - Missing.push_back(Idx); + Payloads.push_back(Chunk); } + }); + } - ++Idx; - } + CacheValues[Idx] = CacheValue.Value; + } + else + { + Missing.push_back(Idx); + } + + ++Idx; + } - if (QueryUpstream) + if (!Missing.empty() && QueryUpstream) + { + auto UpstreamResult = m_UpstreamCache->GetCacheRecords( + CacheKeys, + Missing, + [this, &CacheValues, &Payloads, SkipAttachments](size_t KeyIndex, IoBuffer UpstreamValue) { + CbPackage UpstreamPackage; + if (UpstreamValue && UpstreamPackage.TryLoad(UpstreamValue)) { - auto UpstreamResult = m_UpstreamCache->GetCacheRecords( - CacheKeys, - Missing, - [this, &CacheValues, &Payloads, SkipAttachments](size_t KeyIndex, IoBuffer UpstreamValue) { - CbPackage UpstreamPackage; - if (UpstreamValue && UpstreamPackage.TryLoad(UpstreamValue)) + CbObjectView CacheRecord = UpstreamPackage.GetObject(); + + CacheRecord.IterateAttachments([&](CbFieldView AttachmentHash) { + if (const CbAttachment* Attachment = UpstreamPackage.FindAttachment(AttachmentHash.AsHash())) + { + if (CompressedBuffer Chunk = Attachment->AsCompressedBinary()) { - CbObjectView CacheRecord = UpstreamPackage.GetObject(); - - CacheRecord.IterateAttachments([&](CbFieldView AttachmentHash) { - if (const CbAttachment* Attachment = UpstreamPackage.FindAttachment(AttachmentHash.AsHash())) - { - if (CompressedBuffer Chunk = Attachment->AsCompressedBinary()) - { - m_CidStore.AddChunk(Chunk); - - if (!SkipAttachments) - { - Payloads.push_back(Chunk.GetCompressed().Flatten().AsIoBuffer()); - } - } - } - }); - - CacheValues[KeyIndex] = IoBufferBuilder::MakeCloneFromMemory(CacheRecord.GetView()); + m_CidStore.AddChunk(Chunk); + + if (!SkipAttachments) + { + Payloads.push_back(Chunk.GetCompressed().Flatten().AsIoBuffer()); + } } - }); + } + }); + + CacheValues[KeyIndex] = IoBufferBuilder::MakeCloneFromMemory(CacheRecord.GetView()); } + }); + } - CbObjectWriter BatchResponse; + CbObjectWriter BatchResponse; - BatchResponse.BeginArray("result"sv); - for (const IoBuffer& Value : CacheValues) - { - if (Value) - { - BatchResponse << CbObjectView(Value.Data()); - } - else - { - BatchResponse.AddNull(); - } - } - BatchResponse.EndArray(); + BatchResponse.BeginArray("result"sv); + for (const IoBuffer& Value : CacheValues) + { + if (Value) + { + CbObjectView Record(Value.Data()); + BatchResponse << Record; + } + else + { + BatchResponse.AddNull(); + } + } + BatchResponse.EndArray(); - CbPackage Package; - Package.SetObject(BatchResponse.Save()); + CbPackage Package; + Package.SetObject(BatchResponse.Save()); - for (const IoBuffer& Payload : Payloads) - { - Package.AddAttachment(CbAttachment(CompressedBuffer::FromCompressed(SharedBuffer(Payload)))); - } + for (const IoBuffer& Payload : Payloads) + { + Package.AddAttachment(CbAttachment(CompressedBuffer::FromCompressed(SharedBuffer(Payload)))); + } + + BinaryWriter MemStream; + Package.Save(MemStream); + + Request.WriteResponse(HttpResponseCode::OK, + HttpContentType::kCbPackage, + IoBuffer(IoBuffer::Wrap, MemStream.GetData(), MemStream.GetSize())); +} + +void +HttpStructuredCacheService::HandleBatchGetCacheChunks(zen::HttpServerRequest& Request, CbObjectView BatchRequest, CachePolicy Policy) +{ + using namespace fmt::literals; - BinaryWriter MemStream; - Package.Save(MemStream); + const bool SkipData = (Policy & CachePolicy::SkipData) == CachePolicy::SkipData; + const bool SkipAttachments = (Policy & CachePolicy::SkipAttachments) == CachePolicy::SkipAttachments; + const bool QueryUpstream = m_UpstreamCache && ((Policy & CachePolicy::QueryRemote) == CachePolicy::QueryRemote); + + const std::string_view Method = BatchRequest["method"sv].AsString(); + ZEN_ASSERT(Method == "get-cache-chunks"sv); - Request.WriteResponse(HttpResponseCode::OK, - HttpContentType::kCbPackage, - IoBuffer(IoBuffer::Wrap, MemStream.GetData(), MemStream.GetSize())); + CbObjectView Params = BatchRequest["params"sv].AsObjectView(); + + const auto GetChunkIdFromPayloadId = [](CbObjectView Record, const Oid& PayloadId) -> IoHash { + if (CbObjectView ValueObject = Record["Value"].AsObjectView()) + { + const Oid Id = ValueObject["Id"].AsObjectId(); + if (Id == PayloadId) + { + return ValueObject["RawHash"sv].AsHash(); } - break; - default: - Request.WriteResponse(HttpResponseCode::BadRequest); - break; + } + + for (CbFieldView AttachmentView : Record["Attachments"sv]) + { + CbObjectView AttachmentObject = AttachmentView.AsObjectView(); + const Oid Id = AttachmentObject["Id"].AsObjectId(); + + if (Id == PayloadId) + { + return AttachmentObject["RawHash"sv].AsHash(); + } + } + + return IoHash::Zero; + }; + + std::vector CacheChunks; + std::vector ChunkPolicies; + std::vector Missing; + + for (CbFieldView ChunkView : Params["cachechunks"sv]) + { + CbObjectView Chunk = ChunkView.AsObjectView(); + CbObjectView CacheKeyObject = Chunk["key"sv].AsObjectView(); + const CacheKey Key = CacheKey::Create(CacheKeyObject["bucket"sv].AsString(), CacheKeyObject["hash"sv].AsHash()); + const Oid PayloadId = Chunk["id"sv].AsObjectId(); + const uint64_t RawOffset = Chunk["rawoffset"sv].AsUInt64(); + const uint64_t RawSize = Chunk["rawsize"sv].AsUInt64(); + const uint32_t ChunkPolicy = Chunk["policy"sv].AsUInt32(); + + IoHash ChunkId = IoHash::Zero; + + ZenCacheValue CacheValue; + if (m_CacheStore.Get(Key.Bucket, Key.Hash, CacheValue)) + { + ChunkId = GetChunkIdFromPayloadId(CbObjectView(CacheValue.Value.Data()), PayloadId); + } + + CacheChunks.emplace_back(Key, ChunkId, RawOffset, RawSize); + ChunkPolicies.emplace_back(static_cast(ChunkPolicy)); + } + + if (CacheChunks.empty()) + { + return Request.WriteResponse(HttpResponseCode::BadRequest); + } + + std::vector Chunks; + Chunks.resize(CacheChunks.size()); + + for (size_t Idx = 0; CacheChunk & CacheChunk : CacheChunks) + { + if (IoBuffer Chunk = m_CidStore.FindChunkByCid(CacheChunk.Id)) + { + ZEN_DEBUG("HIT - '{}/{}/{}' {} '{}' ({})", + CacheChunk.Key.Bucket, + CacheChunk.Key.Hash, + CacheChunk.Id, + NiceBytes(Chunk.Size()), + ToString(Chunk.GetContentType()), + "LOCAL"); + + Chunks[Idx] = Chunk; + } + else + { + ZEN_DEBUG("MISS - '{}/{}/{}' '{}'", + CacheChunk.Key.Bucket, + CacheChunk.Key.Hash, + CacheChunk.Id, + ToString(HttpContentType::kCompressedBinary)); + + CacheChunk.Id = IoHash::Zero; + Missing.push_back(Idx); + } + ++Idx; + } + + CbPackage Package; + CbObjectWriter BatchResponse; + + BatchResponse.BeginArray("result"sv); + + for (size_t Idx = 0; Idx < Chunks.size(); ++Idx) + { + if (Chunks[Idx]) + { + BatchResponse << CacheChunks[Idx].Id; + Package.AddAttachment(CbAttachment(CompressedBuffer::FromCompressed(SharedBuffer(std::move(Chunks[Idx]))))); + } + else + { + BatchResponse << IoHash::Zero; + } } + BatchResponse.EndArray(); + + Package.SetObject(BatchResponse.Save()); + + BinaryWriter MemStream; + Package.Save(MemStream); + + Request.WriteResponse(HttpResponseCode::OK, + HttpContentType::kCbPackage, + IoBuffer(IoBuffer::Wrap, MemStream.GetData(), MemStream.GetSize())); } void diff --git a/zenserver/cache/structuredcache.h b/zenserver/cache/structuredcache.h index d6b4944fd..fe4453c2b 100644 --- a/zenserver/cache/structuredcache.h +++ b/zenserver/cache/structuredcache.h @@ -90,6 +90,8 @@ private: void HandleGetCachePayload(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy Policy); void HandlePutCachePayload(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy Policy); void HandleBatchRequest(zen::HttpServerRequest& Request, CachePolicy Policy); + void HandleBatchGetCacheRecords(zen::HttpServerRequest& Request, CbObjectView BatchRequest, CachePolicy Policy); + void HandleBatchGetCacheChunks(zen::HttpServerRequest& Request, CbObjectView BatchRequest, CachePolicy Policy); void HandleCacheBucketRequest(zen::HttpServerRequest& Request, std::string_view Bucket); virtual void HandleStatsRequest(zen::HttpServerRequest& Request) override; virtual void HandleStatusRequest(zen::HttpServerRequest& Request) override; -- cgit v1.2.3 From e0d54396fa3ba0f5466a4ea1f2810721c18fa55f Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Tue, 9 Nov 2021 13:20:00 +0100 Subject: Sort cache keys when resolving payload ID's. --- zenserver/cache/cachekey.cpp | 2 +- zenserver/cache/cachekey.h | 64 ++++++++++++-- zenserver/cache/structuredcache.cpp | 156 ++++++++++++++++++++++++++--------- zenserver/cache/structuredcache.h | 2 +- zenserver/upstream/upstreamcache.cpp | 48 ++++++++++- zenserver/upstream/upstreamcache.h | 4 + 6 files changed, 228 insertions(+), 48 deletions(-) (limited to 'zenserver') diff --git a/zenserver/cache/cachekey.cpp b/zenserver/cache/cachekey.cpp index 94ef7fd12..57993d424 100644 --- a/zenserver/cache/cachekey.cpp +++ b/zenserver/cache/cachekey.cpp @@ -100,6 +100,6 @@ ParseSkipCachePolicy(std::string_view SkipPolicy, CachePolicy Default) return Result; } -CacheKey CacheKey::None = CacheKey{.Bucket = std::string(), .Hash = IoHash()}; +const CacheKey CacheKey::Empty = CacheKey{.Bucket = std::string(), .Hash = IoHash()}; } // namespace zen diff --git a/zenserver/cache/cachekey.h b/zenserver/cache/cachekey.h index 012a01292..5b67b0261 100644 --- a/zenserver/cache/cachekey.h +++ b/zenserver/cache/cachekey.h @@ -44,15 +44,67 @@ struct CacheKey static CacheKey Create(std::string_view Bucket, const IoHash& Hash) { return {.Bucket = ToLower(Bucket), .Hash = Hash}; } - static CacheKey None; + static const CacheKey Empty; }; -struct CacheChunk +inline bool +operator==(const CacheKey& A, const CacheKey& B) { - CacheKey Key; - IoHash Id; - uint64_t RawOffset = 0ull; - uint64_t RawSize = ~uint64_t(0); + return A.Bucket == B.Bucket && A.Hash == B.Hash; +} + +inline bool +operator!=(const CacheKey& A, const CacheKey& B) +{ + return A.Bucket != B.Bucket || A.Hash != B.Hash; +} + +inline bool +operator<(const CacheKey& A, const CacheKey& B) +{ + const std::string& BucketA = A.Bucket; + const std::string& BucketB = B.Bucket; + return BucketA == BucketB ? A.Hash < B.Hash : BucketA < BucketB; +} + +struct CacheChunkRequest +{ + CacheKey Key; + IoHash ChunkId; + Oid PayloadId; + uint64_t RawOffset = 0ull; + uint64_t RawSize = ~uint64_t(0); + CachePolicy Policy = CachePolicy::Default; }; +inline bool +operator<(const CacheChunkRequest& A, const CacheChunkRequest& B) +{ + if (A.Key < B.Key) + { + return true; + } + if (B.Key < A.Key) + { + return false; + } + if (A.ChunkId < B.ChunkId) + { + return true; + } + if (B.ChunkId < A.ChunkId) + { + return false; + } + if (A.PayloadId < B.PayloadId) + { + return true; + } + if (B.PayloadId < A.PayloadId) + { + return false; + } + return A.RawOffset < B.RawOffset; +} + } // namespace zen diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index a1b0e1549..decad2f04 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -809,7 +809,7 @@ HttpStructuredCacheService::HandleBatchRequest(zen::HttpServerRequest& Request, } else if (Method == "get-cache-chunks"sv) { - HandleBatchGetCacheChunks(Request, BatchRequest, Policy); + HandleBatchGetCachePayloads(Request, BatchRequest, Policy); } else { @@ -848,6 +848,11 @@ HttpStructuredCacheService::HandleBatchGetCacheRecords(zen::HttpServerRequest& R CacheKeys.push_back(CacheKey::Create(Query["bucket"sv].AsString(), Query["hash"sv].AsHash())); } + if (CacheKeys.empty()) + { + return Request.WriteResponse(HttpResponseCode::BadRequest); + } + CacheValues.resize(CacheKeys.size()); for (size_t Idx = 0; const CacheKey& Key : CacheKeys) @@ -867,7 +872,14 @@ HttpStructuredCacheService::HandleBatchGetCacheRecords(zen::HttpServerRequest& R }); } + ZEN_DEBUG("HIT - '{}/{}' {} '{}' (LOCAL)", + Key.Bucket, + Key.Hash, + NiceBytes(CacheValue.Value.Size()), + ToString(CacheValue.Value.GetContentType())); + CacheValues[Idx] = CacheValue.Value; + m_CacheStats.HitCount++; } else { @@ -882,8 +894,9 @@ HttpStructuredCacheService::HandleBatchGetCacheRecords(zen::HttpServerRequest& R auto UpstreamResult = m_UpstreamCache->GetCacheRecords( CacheKeys, Missing, - [this, &CacheValues, &Payloads, SkipAttachments](size_t KeyIndex, IoBuffer UpstreamValue) { - CbPackage UpstreamPackage; + [this, &CacheKeys, &CacheValues, &Payloads, SkipAttachments](size_t KeyIndex, IoBuffer UpstreamValue) { + const CacheKey& Key = CacheKeys[KeyIndex]; + CbPackage UpstreamPackage; if (UpstreamValue && UpstreamPackage.TryLoad(UpstreamValue)) { CbObjectView CacheRecord = UpstreamPackage.GetObject(); @@ -903,9 +916,25 @@ HttpStructuredCacheService::HandleBatchGetCacheRecords(zen::HttpServerRequest& R } }); + ZEN_DEBUG("HIT - '{}/{}' {} '{}' (UPSTREAM)", + Key.Bucket, + Key.Hash, + NiceBytes(UpstreamValue.Size()), + ToString(UpstreamValue.GetContentType())); + CacheValues[KeyIndex] = IoBufferBuilder::MakeCloneFromMemory(CacheRecord.GetView()); + m_CacheStats.UpstreamHitCount++; } }); + + Missing = std::move(UpstreamResult.Missing); + } + + for (size_t Idx : Missing) + { + const CacheKey& Key = CacheKeys[Idx]; + ZEN_DEBUG("MISS - '{}/{}'", Key.Bucket, Key.Hash); + m_CacheStats.MissCount++; } CbObjectWriter BatchResponse; @@ -942,19 +971,10 @@ HttpStructuredCacheService::HandleBatchGetCacheRecords(zen::HttpServerRequest& R } void -HttpStructuredCacheService::HandleBatchGetCacheChunks(zen::HttpServerRequest& Request, CbObjectView BatchRequest, CachePolicy Policy) +HttpStructuredCacheService::HandleBatchGetCachePayloads(zen::HttpServerRequest& Request, CbObjectView BatchRequest, CachePolicy Policy) { using namespace fmt::literals; - const bool SkipData = (Policy & CachePolicy::SkipData) == CachePolicy::SkipData; - const bool SkipAttachments = (Policy & CachePolicy::SkipAttachments) == CachePolicy::SkipAttachments; - const bool QueryUpstream = m_UpstreamCache && ((Policy & CachePolicy::QueryRemote) == CachePolicy::QueryRemote); - - const std::string_view Method = BatchRequest["method"sv].AsString(); - ZEN_ASSERT(Method == "get-cache-chunks"sv); - - CbObjectView Params = BatchRequest["params"sv].AsObjectView(); - const auto GetChunkIdFromPayloadId = [](CbObjectView Record, const Oid& PayloadId) -> IoHash { if (CbObjectView ValueObject = Record["Value"].AsObjectView()) { @@ -979,68 +999,126 @@ HttpStructuredCacheService::HandleBatchGetCacheChunks(zen::HttpServerRequest& Re return IoHash::Zero; }; - std::vector CacheChunks; - std::vector ChunkPolicies; - std::vector Missing; + const bool SkipData = (Policy & CachePolicy::SkipData) == CachePolicy::SkipData; + const bool SkipAttachments = (Policy & CachePolicy::SkipAttachments) == CachePolicy::SkipAttachments; + const bool QueryUpstream = m_UpstreamCache && ((Policy & CachePolicy::QueryRemote) == CachePolicy::QueryRemote); + + const std::string_view Method = BatchRequest["method"sv].AsString(); + ZEN_ASSERT(Method == "get-cache-chunks"sv); + + CbObjectView Params = BatchRequest["params"sv].AsObjectView(); + + std::vector ChunkRequests; + std::vector Missing; - for (CbFieldView ChunkView : Params["cachechunks"sv]) + for (CbFieldView ChunkView : Params["chunkrequests"sv]) { CbObjectView Chunk = ChunkView.AsObjectView(); CbObjectView CacheKeyObject = Chunk["key"sv].AsObjectView(); const CacheKey Key = CacheKey::Create(CacheKeyObject["bucket"sv].AsString(), CacheKeyObject["hash"sv].AsHash()); - const Oid PayloadId = Chunk["id"sv].AsObjectId(); + const IoHash ChunkId = IoHash::Zero; + const Oid PayloadId = Chunk["payloadid"sv].AsObjectId(); const uint64_t RawOffset = Chunk["rawoffset"sv].AsUInt64(); const uint64_t RawSize = Chunk["rawsize"sv].AsUInt64(); const uint32_t ChunkPolicy = Chunk["policy"sv].AsUInt32(); - IoHash ChunkId = IoHash::Zero; + ChunkRequests.emplace_back(Key, ChunkId, PayloadId, RawOffset, RawSize, static_cast(ChunkPolicy)); + } - ZenCacheValue CacheValue; - if (m_CacheStore.Get(Key.Bucket, Key.Hash, CacheValue)) + std::stable_sort(ChunkRequests.begin(), ChunkRequests.end()); + + CacheKey CurrentKey = CacheKey::Empty; + IoBuffer CurrentRecordBuffer; + + for (CacheChunkRequest& ChunkRequest : ChunkRequests) + { + if (ChunkRequest.Key != CurrentKey) { - ChunkId = GetChunkIdFromPayloadId(CbObjectView(CacheValue.Value.Data()), PayloadId); + CurrentKey = ChunkRequest.Key; + + ZenCacheValue CacheValue; + if (m_CacheStore.Get(ChunkRequest.Key.Bucket, ChunkRequest.Key.Hash, CacheValue)) + { + CurrentRecordBuffer = CacheValue.Value; + } } - CacheChunks.emplace_back(Key, ChunkId, RawOffset, RawSize); - ChunkPolicies.emplace_back(static_cast(ChunkPolicy)); + if (CurrentRecordBuffer) + { + ChunkRequest.ChunkId = GetChunkIdFromPayloadId(CbObjectView(CurrentRecordBuffer.GetData()), ChunkRequest.PayloadId); + } } - if (CacheChunks.empty()) + if (ChunkRequests.empty()) { return Request.WriteResponse(HttpResponseCode::BadRequest); } std::vector Chunks; - Chunks.resize(CacheChunks.size()); + Chunks.resize(ChunkRequests.size()); - for (size_t Idx = 0; CacheChunk & CacheChunk : CacheChunks) + for (size_t Idx = 0; const CacheChunkRequest& ChunkRequest : ChunkRequests) { - if (IoBuffer Chunk = m_CidStore.FindChunkByCid(CacheChunk.Id)) + if (IoBuffer Chunk = m_CidStore.FindChunkByCid(ChunkRequest.ChunkId)) { ZEN_DEBUG("HIT - '{}/{}/{}' {} '{}' ({})", - CacheChunk.Key.Bucket, - CacheChunk.Key.Hash, - CacheChunk.Id, + ChunkRequest.Key.Bucket, + ChunkRequest.Key.Hash, + ChunkRequest.ChunkId, NiceBytes(Chunk.Size()), ToString(Chunk.GetContentType()), "LOCAL"); Chunks[Idx] = Chunk; + m_CacheStats.HitCount++; } else { - ZEN_DEBUG("MISS - '{}/{}/{}' '{}'", - CacheChunk.Key.Bucket, - CacheChunk.Key.Hash, - CacheChunk.Id, - ToString(HttpContentType::kCompressedBinary)); - - CacheChunk.Id = IoHash::Zero; Missing.push_back(Idx); } ++Idx; } + if (!Missing.empty() && QueryUpstream) + { + auto UpstreamResult = m_UpstreamCache->GetCachePayloads( + ChunkRequests, + Missing, + [this, &ChunkRequests, &Chunks](size_t ChunkIndex, IoBuffer UpstreamValue) { + const CacheChunkRequest& ChunkRequest = ChunkRequests[ChunkIndex]; + + if (CompressedBuffer CompressedChunk = CompressedBuffer::FromCompressed(SharedBuffer(UpstreamValue))) + { + auto InsertResult = m_CidStore.AddChunk(CompressedChunk); + IoBuffer Chunk = CompressedChunk.GetCompressed().Flatten().AsIoBuffer(); + + ZEN_DEBUG("HIT - '{}/{}/{}' {} '{}' ({})", + ChunkRequest.Key.Bucket, + ChunkRequest.Key.Hash, + ChunkRequest.ChunkId, + NiceBytes(Chunk.GetSize()), + ToString(Chunk.GetContentType()), + "UPSTREAM"); + + Chunks[ChunkIndex] = Chunk; + m_CacheStats.UpstreamHitCount++; + } + else + { + ZEN_WARN("got uncompressed upstream cache payload"); + } + }); + + Missing = std::move(UpstreamResult.Missing); + } + + for (size_t Idx : Missing) + { + const CacheChunkRequest& ChunkRequest = ChunkRequests[Idx]; + ZEN_DEBUG("MISS - '{}/{}/{}'", ChunkRequest.Key.Bucket, ChunkRequest.Key.Hash, ChunkRequest.ChunkId); + m_CacheStats.MissCount++; + } + CbPackage Package; CbObjectWriter BatchResponse; @@ -1050,7 +1128,7 @@ HttpStructuredCacheService::HandleBatchGetCacheChunks(zen::HttpServerRequest& Re { if (Chunks[Idx]) { - BatchResponse << CacheChunks[Idx].Id; + BatchResponse << ChunkRequests[Idx].ChunkId; Package.AddAttachment(CbAttachment(CompressedBuffer::FromCompressed(SharedBuffer(std::move(Chunks[Idx]))))); } else diff --git a/zenserver/cache/structuredcache.h b/zenserver/cache/structuredcache.h index fe4453c2b..1ff4f28c9 100644 --- a/zenserver/cache/structuredcache.h +++ b/zenserver/cache/structuredcache.h @@ -91,7 +91,7 @@ private: void HandlePutCachePayload(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy Policy); void HandleBatchRequest(zen::HttpServerRequest& Request, CachePolicy Policy); void HandleBatchGetCacheRecords(zen::HttpServerRequest& Request, CbObjectView BatchRequest, CachePolicy Policy); - void HandleBatchGetCacheChunks(zen::HttpServerRequest& Request, CbObjectView BatchRequest, CachePolicy Policy); + void HandleBatchGetCachePayloads(zen::HttpServerRequest& Request, CbObjectView BatchRequest, CachePolicy Policy); void HandleCacheBucketRequest(zen::HttpServerRequest& Request, std::string_view Bucket); virtual void HandleStatsRequest(zen::HttpServerRequest& Request) override; virtual void HandleStatusRequest(zen::HttpServerRequest& Request) override; diff --git a/zenserver/upstream/upstreamcache.cpp b/zenserver/upstream/upstreamcache.cpp index 437b29cd7..7ef0caf62 100644 --- a/zenserver/upstream/upstreamcache.cpp +++ b/zenserver/upstream/upstreamcache.cpp @@ -782,7 +782,7 @@ public: virtual GetUpstreamCacheBatchResult GetCacheRecords(std::span CacheKeys, std::span KeyIndex, - OnCacheGetComplete OnComplete) override + OnCacheGetComplete OnComplete) override final { if (!m_Options.ReadUpstream) { @@ -801,6 +801,52 @@ public: if (Endpoint->IsHealthy()) { CacheResult = Endpoint->GetCacheRecord(CacheKey, ZenContentType::kCbPackage); + m_Stats.Add(m_Log, *Endpoint, CacheResult, m_Endpoints); + + if (CacheResult.Success) + { + break; + } + } + } + + if (CacheResult.Success) + { + OnComplete(Idx, CacheResult.Value); + } + else + { + Result.Missing.push_back(Idx); + } + } + + return Result; + } + + virtual GetUpstreamCacheBatchResult GetCachePayloads(std::span CacheChunkRequests, + std::span ChunkIndex, + OnCacheGetComplete OnComplete) override final + { + if (!m_Options.ReadUpstream) + { + return {.Missing = std::vector(ChunkIndex.begin(), ChunkIndex.end())}; + } + + GetUpstreamCacheBatchResult Result; + + for (size_t Idx : ChunkIndex) + { + const CacheChunkRequest& Chunk = CacheChunkRequests[Idx]; + UpstreamPayloadKey PayloadKey{{Chunk.Key.Bucket, Chunk.Key.Hash}, Chunk.ChunkId}; + + GetUpstreamCacheResult CacheResult; + for (auto& Endpoint : m_Endpoints) + { + if (Endpoint->IsHealthy()) + { + CacheResult = Endpoint->GetCachePayload(PayloadKey); + m_Stats.Add(m_Log, *Endpoint, CacheResult, m_Endpoints); + if (CacheResult.Success) { break; diff --git a/zenserver/upstream/upstreamcache.h b/zenserver/upstream/upstreamcache.h index 04554f210..a7bae302d 100644 --- a/zenserver/upstream/upstreamcache.h +++ b/zenserver/upstream/upstreamcache.h @@ -145,6 +145,10 @@ public: virtual GetUpstreamCacheResult GetCachePayload(UpstreamPayloadKey PayloadKey) = 0; + virtual GetUpstreamCacheBatchResult GetCachePayloads(std::span CacheChunkRequests, + std::span RequestIndex, + OnCacheGetComplete OnComplete) = 0; + struct EnqueueResult { bool Success = false; -- cgit v1.2.3 From 424be141e88b04b4de7ab5def2c29b03f5f72d48 Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Wed, 10 Nov 2021 08:47:57 +0100 Subject: Handle cache record policy. --- zenserver/cache/cachekey.cpp | 39 ++++++++++++++++++++++++++++++++++++ zenserver/cache/cachekey.h | 23 +++++++++++++++++++++ zenserver/cache/structuredcache.cpp | 40 ++++++++++++++++++++----------------- zenserver/cache/structuredcache.h | 6 +++--- 4 files changed, 87 insertions(+), 21 deletions(-) (limited to 'zenserver') diff --git a/zenserver/cache/cachekey.cpp b/zenserver/cache/cachekey.cpp index 57993d424..85d57745d 100644 --- a/zenserver/cache/cachekey.cpp +++ b/zenserver/cache/cachekey.cpp @@ -2,6 +2,7 @@ #include "cachekey.h" +#include #include namespace zen { @@ -100,6 +101,44 @@ ParseSkipCachePolicy(std::string_view SkipPolicy, CachePolicy Default) return Result; } +CachePolicy +CacheRecordPolicy::GetPayloadPolicy(const Oid& PayloadId) const +{ + if (const auto It = m_PayloadPolicies.find(PayloadId); It != m_PayloadPolicies.end()) + { + return It->second; + } + + return m_DefaultPayloadPolicy; +} + +bool +CacheRecordPolicy::FromCompactBinary(CbObjectView RecordPolicyObject, CacheRecordPolicy& OutRecordPolicy) +{ + using namespace std::literals; + + const uint32_t RecordPolicy = RecordPolicyObject["recordpolicy"sv].AsUInt32(static_cast(CachePolicy::Default)); + const uint32_t DefaultPayloadPolicy = + RecordPolicyObject["defaultpayloadpolicy"sv].AsUInt32(static_cast(CachePolicy::Default)); + + OutRecordPolicy.m_RecordPolicy = static_cast(RecordPolicy); + OutRecordPolicy.m_DefaultPayloadPolicy = static_cast(DefaultPayloadPolicy); + + for (CbFieldView PayloadPolicyView : RecordPolicyObject["payloadpolicies"sv]) + { + CbObjectView PayloadPolicyObject = PayloadPolicyView.AsObjectView(); + const Oid PayloadId = PayloadPolicyObject["payloadid"sv].AsObjectId(); + const uint32_t PayloadPolicy = PayloadPolicyObject["policy"sv].AsUInt32(); + + if (PayloadId != Oid::Zero && PayloadPolicy != 0) + { + OutRecordPolicy.m_PayloadPolicies.emplace(PayloadId, static_cast(PayloadPolicy)); + } + } + + return true; +} + const CacheKey CacheKey::Empty = CacheKey{.Bucket = std::string(), .Hash = IoHash()}; } // namespace zen diff --git a/zenserver/cache/cachekey.h b/zenserver/cache/cachekey.h index 5b67b0261..6ce5d3aab 100644 --- a/zenserver/cache/cachekey.h +++ b/zenserver/cache/cachekey.h @@ -7,8 +7,12 @@ #include #include +#include + namespace zen { +class CbObjectView; + enum class CachePolicy : uint8_t { None = 0, @@ -37,6 +41,25 @@ CachePolicy ParseStoreCachePolicy(std::string_view StorePolicy, CachePolicy Defa CachePolicy ParseSkipCachePolicy(std::string_view SkipPolicy, CachePolicy Default = CachePolicy::None); +class CacheRecordPolicy +{ +public: + CacheRecordPolicy() = default; + + CachePolicy GetRecordPolicy() const { return m_RecordPolicy; } + CachePolicy GetPayloadPolicy(const Oid& PayloadId) const; + CachePolicy GetDefaultPayloadPolicy() const { return m_DefaultPayloadPolicy; } + + static bool FromCompactBinary(CbObjectView RecordPolicyObject, CacheRecordPolicy& OutRecordPolicy); + +private: + using PayloadPolicyMap = std::unordered_map; + + CachePolicy m_RecordPolicy = CachePolicy::Default; + CachePolicy m_DefaultPayloadPolicy = CachePolicy::Default; + PayloadPolicyMap m_PayloadPolicies; +}; + struct CacheKey { std::string Bucket; diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index decad2f04..073192c12 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -114,9 +114,7 @@ HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request) if (Key == "$batch") { - const auto QueryParams = Request.GetQueryParams(); - CachePolicy Policy = ParseCachePolicy(QueryParams); - return HandleBatchRequest(Request, Policy); + return HandleBatchRequest(Request); } if (std::all_of(begin(Key), end(Key), [](const char c) { return std::isalnum(c); })) @@ -782,10 +780,8 @@ HttpStructuredCacheService::ValidateKeyUri(HttpServerRequest& Request, CacheRef& } void -HttpStructuredCacheService::HandleBatchRequest(zen::HttpServerRequest& Request, CachePolicy Policy) +HttpStructuredCacheService::HandleBatchRequest(zen::HttpServerRequest& Request) { - using namespace fmt::literals; - switch (auto Verb = Request.RequestVerb()) { using enum HttpVerb; @@ -805,11 +801,11 @@ HttpStructuredCacheService::HandleBatchRequest(zen::HttpServerRequest& Request, if (Method == "get-cache-records"sv) { - HandleBatchGetCacheRecords(Request, BatchRequest, Policy); + HandleBatchGetCacheRecords(Request, BatchRequest); } else if (Method == "get-cache-chunks"sv) { - HandleBatchGetCachePayloads(Request, BatchRequest, Policy); + HandleBatchGetCachePayloads(Request, BatchRequest); } else { @@ -824,19 +820,21 @@ HttpStructuredCacheService::HandleBatchRequest(zen::HttpServerRequest& Request, } void -HttpStructuredCacheService::HandleBatchGetCacheRecords(zen::HttpServerRequest& Request, CbObjectView BatchRequest, CachePolicy Policy) +HttpStructuredCacheService::HandleBatchGetCacheRecords(zen::HttpServerRequest& Request, CbObjectView BatchRequest) { using namespace fmt::literals; - const bool SkipData = (Policy & CachePolicy::SkipData) == CachePolicy::SkipData; - const bool SkipAttachments = (Policy & CachePolicy::SkipAttachments) == CachePolicy::SkipAttachments; - const bool QueryUpstream = m_UpstreamCache && ((Policy & CachePolicy::QueryRemote) == CachePolicy::QueryRemote); - const std::string_view Method = BatchRequest["method"sv].AsString(); ZEN_ASSERT(Method == "get-cache-records"sv); CbObjectView Params = BatchRequest["params"sv].AsObjectView(); + CacheRecordPolicy Policy; + CacheRecordPolicy::FromCompactBinary(Params["policy"sv].AsObjectView(), Policy); + + const bool SkipAttachments = (Policy.GetRecordPolicy() & CachePolicy::SkipAttachments) == CachePolicy::SkipAttachments; + const bool QueryUpstream = m_UpstreamCache && ((Policy.GetRecordPolicy() & CachePolicy::QueryRemote) == CachePolicy::QueryRemote); + std::vector CacheKeys; std::vector CacheValues; std::vector Payloads; @@ -855,6 +853,11 @@ HttpStructuredCacheService::HandleBatchGetCacheRecords(zen::HttpServerRequest& R CacheValues.resize(CacheKeys.size()); + if (!SkipAttachments) + { + Payloads.reserve(CacheKeys.size()); + } + for (size_t Idx = 0; const CacheKey& Key : CacheKeys) { ZenCacheValue CacheValue; @@ -971,7 +974,7 @@ HttpStructuredCacheService::HandleBatchGetCacheRecords(zen::HttpServerRequest& R } void -HttpStructuredCacheService::HandleBatchGetCachePayloads(zen::HttpServerRequest& Request, CbObjectView BatchRequest, CachePolicy Policy) +HttpStructuredCacheService::HandleBatchGetCachePayloads(zen::HttpServerRequest& Request, CbObjectView BatchRequest) { using namespace fmt::literals; @@ -999,15 +1002,16 @@ HttpStructuredCacheService::HandleBatchGetCachePayloads(zen::HttpServerRequest& return IoHash::Zero; }; - const bool SkipData = (Policy & CachePolicy::SkipData) == CachePolicy::SkipData; - const bool SkipAttachments = (Policy & CachePolicy::SkipAttachments) == CachePolicy::SkipAttachments; - const bool QueryUpstream = m_UpstreamCache && ((Policy & CachePolicy::QueryRemote) == CachePolicy::QueryRemote); - const std::string_view Method = BatchRequest["method"sv].AsString(); ZEN_ASSERT(Method == "get-cache-chunks"sv); CbObjectView Params = BatchRequest["params"sv].AsObjectView(); + CacheRecordPolicy Policy; + CacheRecordPolicy::FromCompactBinary(Params["policy"sv].AsObjectView(), Policy); + + const bool QueryUpstream = m_UpstreamCache && ((Policy.GetRecordPolicy() & CachePolicy::QueryRemote) == CachePolicy::QueryRemote); + std::vector ChunkRequests; std::vector Missing; diff --git a/zenserver/cache/structuredcache.h b/zenserver/cache/structuredcache.h index 1ff4f28c9..9efcc05fa 100644 --- a/zenserver/cache/structuredcache.h +++ b/zenserver/cache/structuredcache.h @@ -89,9 +89,9 @@ private: void HandleCachePayloadRequest(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy Policy); void HandleGetCachePayload(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy Policy); void HandlePutCachePayload(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy Policy); - void HandleBatchRequest(zen::HttpServerRequest& Request, CachePolicy Policy); - void HandleBatchGetCacheRecords(zen::HttpServerRequest& Request, CbObjectView BatchRequest, CachePolicy Policy); - void HandleBatchGetCachePayloads(zen::HttpServerRequest& Request, CbObjectView BatchRequest, CachePolicy Policy); + void HandleBatchRequest(zen::HttpServerRequest& Request); + void HandleBatchGetCacheRecords(zen::HttpServerRequest& Request, CbObjectView BatchRequest); + void HandleBatchGetCachePayloads(zen::HttpServerRequest& Request, CbObjectView BatchRequest); void HandleCacheBucketRequest(zen::HttpServerRequest& Request, std::string_view Bucket); virtual void HandleStatsRequest(zen::HttpServerRequest& Request) override; virtual void HandleStatusRequest(zen::HttpServerRequest& Request) override; -- cgit v1.2.3 From 6b5748c8424cdac3c7a39ef87a9bc33ffa69256f Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Wed, 10 Nov 2021 10:58:30 +0100 Subject: Changed RPC keys to camel case. --- zenserver/cache/cachekey.cpp | 10 ++--- zenserver/cache/structuredcache.cpp | 76 ++++++++++++++++++------------------- 2 files changed, 43 insertions(+), 43 deletions(-) (limited to 'zenserver') diff --git a/zenserver/cache/cachekey.cpp b/zenserver/cache/cachekey.cpp index 85d57745d..eca2d95d5 100644 --- a/zenserver/cache/cachekey.cpp +++ b/zenserver/cache/cachekey.cpp @@ -117,18 +117,18 @@ CacheRecordPolicy::FromCompactBinary(CbObjectView RecordPolicyObject, CacheRecor { using namespace std::literals; - const uint32_t RecordPolicy = RecordPolicyObject["recordpolicy"sv].AsUInt32(static_cast(CachePolicy::Default)); + const uint32_t RecordPolicy = RecordPolicyObject["RecordPolicy"sv].AsUInt32(static_cast(CachePolicy::Default)); const uint32_t DefaultPayloadPolicy = - RecordPolicyObject["defaultpayloadpolicy"sv].AsUInt32(static_cast(CachePolicy::Default)); + RecordPolicyObject["DefaultPayloadPolicy"sv].AsUInt32(static_cast(CachePolicy::Default)); OutRecordPolicy.m_RecordPolicy = static_cast(RecordPolicy); OutRecordPolicy.m_DefaultPayloadPolicy = static_cast(DefaultPayloadPolicy); - for (CbFieldView PayloadPolicyView : RecordPolicyObject["payloadpolicies"sv]) + for (CbFieldView PayloadPolicyView : RecordPolicyObject["PayloadPolicies"sv]) { CbObjectView PayloadPolicyObject = PayloadPolicyView.AsObjectView(); - const Oid PayloadId = PayloadPolicyObject["payloadid"sv].AsObjectId(); - const uint32_t PayloadPolicy = PayloadPolicyObject["policy"sv].AsUInt32(); + const Oid PayloadId = PayloadPolicyObject["Id"sv].AsObjectId(); + const uint32_t PayloadPolicy = PayloadPolicyObject["Policy"sv].AsUInt32(); if (PayloadId != Oid::Zero && PayloadPolicy != 0) { diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 073192c12..45f626ff5 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -797,13 +797,13 @@ HttpStructuredCacheService::HandleBatchRequest(zen::HttpServerRequest& Request) } CbObject BatchRequest = zen::LoadCompactBinaryObject(Request.ReadPayload()); - const std::string_view Method = BatchRequest["method"sv].AsString(); + const std::string_view Method = BatchRequest["Method"sv].AsString(); - if (Method == "get-cache-records"sv) + if (Method == "GetCacheRecords"sv) { HandleBatchGetCacheRecords(Request, BatchRequest); } - else if (Method == "get-cache-chunks"sv) + else if (Method == "GetCachePayloads"sv) { HandleBatchGetCachePayloads(Request, BatchRequest); } @@ -824,13 +824,13 @@ HttpStructuredCacheService::HandleBatchGetCacheRecords(zen::HttpServerRequest& R { using namespace fmt::literals; - const std::string_view Method = BatchRequest["method"sv].AsString(); - ZEN_ASSERT(Method == "get-cache-records"sv); + const std::string_view Method = BatchRequest["Method"sv].AsString(); + ZEN_ASSERT(Method == "GetCacheRecords"sv); - CbObjectView Params = BatchRequest["params"sv].AsObjectView(); + CbObjectView Params = BatchRequest["Params"sv].AsObjectView(); CacheRecordPolicy Policy; - CacheRecordPolicy::FromCompactBinary(Params["policy"sv].AsObjectView(), Policy); + CacheRecordPolicy::FromCompactBinary(Params["Policy"sv].AsObjectView(), Policy); const bool SkipAttachments = (Policy.GetRecordPolicy() & CachePolicy::SkipAttachments) == CachePolicy::SkipAttachments; const bool QueryUpstream = m_UpstreamCache && ((Policy.GetRecordPolicy() & CachePolicy::QueryRemote) == CachePolicy::QueryRemote); @@ -840,10 +840,10 @@ HttpStructuredCacheService::HandleBatchGetCacheRecords(zen::HttpServerRequest& R std::vector Payloads; std::vector Missing; - for (CbFieldView QueryView : Params["cachekeys"sv]) + for (CbFieldView QueryView : Params["CacheKeys"sv]) { CbObjectView Query = QueryView.AsObjectView(); - CacheKeys.push_back(CacheKey::Create(Query["bucket"sv].AsString(), Query["hash"sv].AsHash())); + CacheKeys.push_back(CacheKey::Create(Query["Bucket"sv].AsString(), Query["Hash"sv].AsHash())); } if (CacheKeys.empty()) @@ -858,7 +858,7 @@ HttpStructuredCacheService::HandleBatchGetCacheRecords(zen::HttpServerRequest& R Payloads.reserve(CacheKeys.size()); } - for (size_t Idx = 0; const CacheKey& Key : CacheKeys) + for (size_t KeyIndex = 0; const CacheKey& Key : CacheKeys) { ZenCacheValue CacheValue; if (m_CacheStore.Get(Key.Bucket, Key.Hash, CacheValue)) @@ -881,15 +881,15 @@ HttpStructuredCacheService::HandleBatchGetCacheRecords(zen::HttpServerRequest& R NiceBytes(CacheValue.Value.Size()), ToString(CacheValue.Value.GetContentType())); - CacheValues[Idx] = CacheValue.Value; + CacheValues[KeyIndex] = CacheValue.Value; m_CacheStats.HitCount++; } else { - Missing.push_back(Idx); + Missing.push_back(KeyIndex); } - ++Idx; + ++KeyIndex; } if (!Missing.empty() && QueryUpstream) @@ -933,16 +933,16 @@ HttpStructuredCacheService::HandleBatchGetCacheRecords(zen::HttpServerRequest& R Missing = std::move(UpstreamResult.Missing); } - for (size_t Idx : Missing) + for (size_t KeyIndex : Missing) { - const CacheKey& Key = CacheKeys[Idx]; + const CacheKey& Key = CacheKeys[KeyIndex]; ZEN_DEBUG("MISS - '{}/{}'", Key.Bucket, Key.Hash); m_CacheStats.MissCount++; } CbObjectWriter BatchResponse; - BatchResponse.BeginArray("result"sv); + BatchResponse.BeginArray("Result"sv); for (const IoBuffer& Value : CacheValues) { if (Value) @@ -1002,29 +1002,29 @@ HttpStructuredCacheService::HandleBatchGetCachePayloads(zen::HttpServerRequest& return IoHash::Zero; }; - const std::string_view Method = BatchRequest["method"sv].AsString(); - ZEN_ASSERT(Method == "get-cache-chunks"sv); + const std::string_view Method = BatchRequest["Method"sv].AsString(); + ZEN_ASSERT(Method == "GetCachePayloads"sv); - CbObjectView Params = BatchRequest["params"sv].AsObjectView(); + CbObjectView Params = BatchRequest["Params"sv].AsObjectView(); CacheRecordPolicy Policy; - CacheRecordPolicy::FromCompactBinary(Params["policy"sv].AsObjectView(), Policy); + CacheRecordPolicy::FromCompactBinary(Params["Policy"sv].AsObjectView(), Policy); const bool QueryUpstream = m_UpstreamCache && ((Policy.GetRecordPolicy() & CachePolicy::QueryRemote) == CachePolicy::QueryRemote); std::vector ChunkRequests; std::vector Missing; - for (CbFieldView ChunkView : Params["chunkrequests"sv]) + for (CbFieldView ChunkView : Params["ChunkRequests"sv]) { CbObjectView Chunk = ChunkView.AsObjectView(); - CbObjectView CacheKeyObject = Chunk["key"sv].AsObjectView(); - const CacheKey Key = CacheKey::Create(CacheKeyObject["bucket"sv].AsString(), CacheKeyObject["hash"sv].AsHash()); + CbObjectView CacheKeyObject = Chunk["Key"sv].AsObjectView(); + const CacheKey Key = CacheKey::Create(CacheKeyObject["Bucket"sv].AsString(), CacheKeyObject["Hash"sv].AsHash()); const IoHash ChunkId = IoHash::Zero; - const Oid PayloadId = Chunk["payloadid"sv].AsObjectId(); - const uint64_t RawOffset = Chunk["rawoffset"sv].AsUInt64(); - const uint64_t RawSize = Chunk["rawsize"sv].AsUInt64(); - const uint32_t ChunkPolicy = Chunk["policy"sv].AsUInt32(); + const Oid PayloadId = Chunk["PayloadId"sv].AsObjectId(); + const uint64_t RawOffset = Chunk["RawoffSet"sv].AsUInt64(); + const uint64_t RawSize = Chunk["RawSize"sv].AsUInt64(); + const uint32_t ChunkPolicy = Chunk["Policy"sv].AsUInt32(); ChunkRequests.emplace_back(Key, ChunkId, PayloadId, RawOffset, RawSize, static_cast(ChunkPolicy)); } @@ -1061,7 +1061,7 @@ HttpStructuredCacheService::HandleBatchGetCachePayloads(zen::HttpServerRequest& std::vector Chunks; Chunks.resize(ChunkRequests.size()); - for (size_t Idx = 0; const CacheChunkRequest& ChunkRequest : ChunkRequests) + for (size_t RequestIndex = 0; const CacheChunkRequest& ChunkRequest : ChunkRequests) { if (IoBuffer Chunk = m_CidStore.FindChunkByCid(ChunkRequest.ChunkId)) { @@ -1073,14 +1073,14 @@ HttpStructuredCacheService::HandleBatchGetCachePayloads(zen::HttpServerRequest& ToString(Chunk.GetContentType()), "LOCAL"); - Chunks[Idx] = Chunk; + Chunks[RequestIndex] = Chunk; m_CacheStats.HitCount++; } else { - Missing.push_back(Idx); + Missing.push_back(RequestIndex); } - ++Idx; + ++RequestIndex; } if (!Missing.empty() && QueryUpstream) @@ -1116,9 +1116,9 @@ HttpStructuredCacheService::HandleBatchGetCachePayloads(zen::HttpServerRequest& Missing = std::move(UpstreamResult.Missing); } - for (size_t Idx : Missing) + for (size_t RequestIndex : Missing) { - const CacheChunkRequest& ChunkRequest = ChunkRequests[Idx]; + const CacheChunkRequest& ChunkRequest = ChunkRequests[RequestIndex]; ZEN_DEBUG("MISS - '{}/{}/{}'", ChunkRequest.Key.Bucket, ChunkRequest.Key.Hash, ChunkRequest.ChunkId); m_CacheStats.MissCount++; } @@ -1126,14 +1126,14 @@ HttpStructuredCacheService::HandleBatchGetCachePayloads(zen::HttpServerRequest& CbPackage Package; CbObjectWriter BatchResponse; - BatchResponse.BeginArray("result"sv); + BatchResponse.BeginArray("Result"sv); - for (size_t Idx = 0; Idx < Chunks.size(); ++Idx) + for (size_t ChunkIndex = 0; ChunkIndex < Chunks.size(); ++ChunkIndex) { - if (Chunks[Idx]) + if (Chunks[ChunkIndex]) { - BatchResponse << ChunkRequests[Idx].ChunkId; - Package.AddAttachment(CbAttachment(CompressedBuffer::FromCompressed(SharedBuffer(std::move(Chunks[Idx]))))); + BatchResponse << ChunkRequests[ChunkIndex].ChunkId; + Package.AddAttachment(CbAttachment(CompressedBuffer::FromCompressed(SharedBuffer(std::move(Chunks[ChunkIndex]))))); } else { -- cgit v1.2.3 From 39e064c35abd6bc8d2d32b9bb99b5805d1d1e78c Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Wed, 10 Nov 2021 14:58:04 +0100 Subject: Honor cache policy. --- zenserver/cache/structuredcache.cpp | 72 ++++++++++++++++++++++++------------- 1 file changed, 48 insertions(+), 24 deletions(-) (limited to 'zenserver') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 45f626ff5..de2fcb27c 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -833,11 +833,12 @@ HttpStructuredCacheService::HandleBatchGetCacheRecords(zen::HttpServerRequest& R CacheRecordPolicy::FromCompactBinary(Params["Policy"sv].AsObjectView(), Policy); const bool SkipAttachments = (Policy.GetRecordPolicy() & CachePolicy::SkipAttachments) == CachePolicy::SkipAttachments; - const bool QueryUpstream = m_UpstreamCache && ((Policy.GetRecordPolicy() & CachePolicy::QueryRemote) == CachePolicy::QueryRemote); + const bool QueryRemote = m_UpstreamCache && ((Policy.GetRecordPolicy() & CachePolicy::QueryRemote) == CachePolicy::QueryRemote); std::vector CacheKeys; std::vector CacheValues; std::vector Payloads; + std::vector UpstreamRequests; std::vector Missing; for (CbFieldView QueryView : Params["CacheKeys"sv]) @@ -884,6 +885,10 @@ HttpStructuredCacheService::HandleBatchGetCacheRecords(zen::HttpServerRequest& R CacheValues[KeyIndex] = CacheValue.Value; m_CacheStats.HitCount++; } + else if (QueryRemote) + { + UpstreamRequests.push_back(KeyIndex); + } else { Missing.push_back(KeyIndex); @@ -892,11 +897,11 @@ HttpStructuredCacheService::HandleBatchGetCacheRecords(zen::HttpServerRequest& R ++KeyIndex; } - if (!Missing.empty() && QueryUpstream) + if (!UpstreamRequests.empty() && m_UpstreamCache) { auto UpstreamResult = m_UpstreamCache->GetCacheRecords( CacheKeys, - Missing, + UpstreamRequests, [this, &CacheKeys, &CacheValues, &Payloads, SkipAttachments](size_t KeyIndex, IoBuffer UpstreamValue) { const CacheKey& Key = CacheKeys[KeyIndex]; CbPackage UpstreamPackage; @@ -930,7 +935,10 @@ HttpStructuredCacheService::HandleBatchGetCacheRecords(zen::HttpServerRequest& R } }); - Missing = std::move(UpstreamResult.Missing); + for (size_t MissingUpstream : UpstreamResult.Missing) + { + Missing.push_back(MissingUpstream); + } } for (size_t KeyIndex : Missing) @@ -1007,12 +1015,8 @@ HttpStructuredCacheService::HandleBatchGetCachePayloads(zen::HttpServerRequest& CbObjectView Params = BatchRequest["Params"sv].AsObjectView(); - CacheRecordPolicy Policy; - CacheRecordPolicy::FromCompactBinary(Params["Policy"sv].AsObjectView(), Policy); - - const bool QueryUpstream = m_UpstreamCache && ((Policy.GetRecordPolicy() & CachePolicy::QueryRemote) == CachePolicy::QueryRemote); - std::vector ChunkRequests; + std::vector UpstreamRequests; std::vector Missing; for (CbFieldView ChunkView : Params["ChunkRequests"sv]) @@ -1063,31 +1067,46 @@ HttpStructuredCacheService::HandleBatchGetCachePayloads(zen::HttpServerRequest& for (size_t RequestIndex = 0; const CacheChunkRequest& ChunkRequest : ChunkRequests) { - if (IoBuffer Chunk = m_CidStore.FindChunkByCid(ChunkRequest.ChunkId)) + const bool QueryLocal = (ChunkRequest.Policy & CachePolicy::QueryLocal) == CachePolicy::QueryLocal; + const bool QueryRemote = (ChunkRequest.Policy & CachePolicy::QueryRemote) == CachePolicy::QueryRemote; + + if (QueryLocal) { - ZEN_DEBUG("HIT - '{}/{}/{}' {} '{}' ({})", - ChunkRequest.Key.Bucket, - ChunkRequest.Key.Hash, - ChunkRequest.ChunkId, - NiceBytes(Chunk.Size()), - ToString(Chunk.GetContentType()), - "LOCAL"); - - Chunks[RequestIndex] = Chunk; - m_CacheStats.HitCount++; + if (IoBuffer Chunk = m_CidStore.FindChunkByCid(ChunkRequest.ChunkId)) + { + ZEN_DEBUG("HIT - '{}/{}/{}' {} '{}' ({})", + ChunkRequest.Key.Bucket, + ChunkRequest.Key.Hash, + ChunkRequest.ChunkId, + NiceBytes(Chunk.Size()), + ToString(Chunk.GetContentType()), + "LOCAL"); + + Chunks[RequestIndex] = Chunk; + m_CacheStats.HitCount++; + } + else if (QueryRemote) + { + UpstreamRequests.push_back(RequestIndex); + } + else + { + Missing.push_back(RequestIndex); + } } else { - Missing.push_back(RequestIndex); + ZEN_DEBUG("SKIP - '{}/{}/{}'", ChunkRequest.Key.Bucket, ChunkRequest.Key.Hash, ChunkRequest.ChunkId); } + ++RequestIndex; } - if (!Missing.empty() && QueryUpstream) + if (!UpstreamRequests.empty() && m_UpstreamCache) { auto UpstreamResult = m_UpstreamCache->GetCachePayloads( ChunkRequests, - Missing, + UpstreamRequests, [this, &ChunkRequests, &Chunks](size_t ChunkIndex, IoBuffer UpstreamValue) { const CacheChunkRequest& ChunkRequest = ChunkRequests[ChunkIndex]; @@ -1105,6 +1124,8 @@ HttpStructuredCacheService::HandleBatchGetCachePayloads(zen::HttpServerRequest& "UPSTREAM"); Chunks[ChunkIndex] = Chunk; + + m_CacheStats.HitCount++; m_CacheStats.UpstreamHitCount++; } else @@ -1113,7 +1134,10 @@ HttpStructuredCacheService::HandleBatchGetCachePayloads(zen::HttpServerRequest& } }); - Missing = std::move(UpstreamResult.Missing); + for (size_t MissingUpstream : UpstreamResult.Missing) + { + Missing.push_back(MissingUpstream); + } } for (size_t RequestIndex : Missing) -- cgit v1.2.3 From 2c0e2ab5de21b13dcd25758ca3b96af889db7137 Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Thu, 11 Nov 2021 11:19:17 +0100 Subject: Added batch API to upstream endpoints. --- zenserver/cache/cachekey.cpp | 21 ++- zenserver/cache/cachekey.h | 4 +- zenserver/cache/structuredcache.cpp | 93 ++++------ zenserver/upstream/upstreamcache.cpp | 346 ++++++++++++++++++++++++++++------- zenserver/upstream/upstreamcache.h | 65 ++++--- zenserver/upstream/zen.cpp | 29 +++ zenserver/upstream/zen.h | 3 + 7 files changed, 413 insertions(+), 148 deletions(-) (limited to 'zenserver') diff --git a/zenserver/cache/cachekey.cpp b/zenserver/cache/cachekey.cpp index eca2d95d5..2ead9ac58 100644 --- a/zenserver/cache/cachekey.cpp +++ b/zenserver/cache/cachekey.cpp @@ -3,6 +3,7 @@ #include "cachekey.h" #include +#include #include namespace zen { @@ -113,7 +114,7 @@ CacheRecordPolicy::GetPayloadPolicy(const Oid& PayloadId) const } bool -CacheRecordPolicy::FromCompactBinary(CbObjectView RecordPolicyObject, CacheRecordPolicy& OutRecordPolicy) +CacheRecordPolicy::Load(CbObjectView RecordPolicyObject, CacheRecordPolicy& OutRecordPolicy) { using namespace std::literals; @@ -139,6 +140,24 @@ CacheRecordPolicy::FromCompactBinary(CbObjectView RecordPolicyObject, CacheRecor return true; } +void +CacheRecordPolicy::Save(const CacheRecordPolicy& Policy, CbWriter& Writer) +{ + Writer << "RecordPolicy"sv << static_cast(Policy.GetRecordPolicy()); + Writer << "DefaultPayloadPolicy"sv << static_cast(Policy.GetDefaultPayloadPolicy()); + + if (!Policy.m_PayloadPolicies.empty()) + { + Writer.BeginArray("PayloadPolicies"sv); + for (const auto& Kv : Policy.m_PayloadPolicies) + { + Writer.AddObjectId("Id"sv, Kv.first); + Writer << "Policy"sv << static_cast(Kv.second); + } + Writer.EndArray(); + } +} + const CacheKey CacheKey::Empty = CacheKey{.Bucket = std::string(), .Hash = IoHash()}; } // namespace zen diff --git a/zenserver/cache/cachekey.h b/zenserver/cache/cachekey.h index 6ce5d3aab..c32f7ed87 100644 --- a/zenserver/cache/cachekey.h +++ b/zenserver/cache/cachekey.h @@ -12,6 +12,7 @@ namespace zen { class CbObjectView; +class CbWriter; enum class CachePolicy : uint8_t { @@ -50,7 +51,8 @@ public: CachePolicy GetPayloadPolicy(const Oid& PayloadId) const; CachePolicy GetDefaultPayloadPolicy() const { return m_DefaultPayloadPolicy; } - static bool FromCompactBinary(CbObjectView RecordPolicyObject, CacheRecordPolicy& OutRecordPolicy); + static bool Load(CbObjectView RecordPolicyObject, CacheRecordPolicy& OutRecordPolicy); + static void Save(const CacheRecordPolicy& Policy, CbWriter& Writer); private: using PayloadPolicyMap = std::unordered_map; diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index de2fcb27c..721942cc8 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -618,7 +618,7 @@ HttpStructuredCacheService::HandleGetCachePayload(zen::HttpServerRequest& Reques if (QueryUpstream) { - if (auto UpstreamResult = m_UpstreamCache->GetCachePayload({{Ref.BucketSegment, Ref.HashKey}, Ref.PayloadId}); + if (auto UpstreamResult = m_UpstreamCache->GetCachePayload({Ref.BucketSegment, Ref.HashKey}, Ref.PayloadId); UpstreamResult.Success) { if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(UpstreamResult.Value))) @@ -830,7 +830,7 @@ HttpStructuredCacheService::HandleBatchGetCacheRecords(zen::HttpServerRequest& R CbObjectView Params = BatchRequest["Params"sv].AsObjectView(); CacheRecordPolicy Policy; - CacheRecordPolicy::FromCompactBinary(Params["Policy"sv].AsObjectView(), Policy); + CacheRecordPolicy::Load(Params["Policy"sv].AsObjectView(), Policy); const bool SkipAttachments = (Policy.GetRecordPolicy() & CachePolicy::SkipAttachments) == CachePolicy::SkipAttachments; const bool QueryRemote = m_UpstreamCache && ((Policy.GetRecordPolicy() & CachePolicy::QueryRemote) == CachePolicy::QueryRemote); @@ -899,18 +899,12 @@ HttpStructuredCacheService::HandleBatchGetCacheRecords(zen::HttpServerRequest& R if (!UpstreamRequests.empty() && m_UpstreamCache) { - auto UpstreamResult = m_UpstreamCache->GetCacheRecords( - CacheKeys, - UpstreamRequests, - [this, &CacheKeys, &CacheValues, &Payloads, SkipAttachments](size_t KeyIndex, IoBuffer UpstreamValue) { - const CacheKey& Key = CacheKeys[KeyIndex]; - CbPackage UpstreamPackage; - if (UpstreamValue && UpstreamPackage.TryLoad(UpstreamValue)) + const auto OnCacheRecordGetComplete = + [this, &CacheKeys, &CacheValues, &Payloads, &Missing, SkipAttachments](CacheRecordGetCompleteParams&& Params) { + if (Params.Record) { - CbObjectView CacheRecord = UpstreamPackage.GetObject(); - - CacheRecord.IterateAttachments([&](CbFieldView AttachmentHash) { - if (const CbAttachment* Attachment = UpstreamPackage.FindAttachment(AttachmentHash.AsHash())) + Params.Record.IterateAttachments([&](CbFieldView AttachmentHash) { + if (const CbAttachment* Attachment = Params.Package.FindAttachment(AttachmentHash.AsHash())) { if (CompressedBuffer Chunk = Attachment->AsCompressedBinary()) { @@ -925,20 +919,21 @@ HttpStructuredCacheService::HandleBatchGetCacheRecords(zen::HttpServerRequest& R }); ZEN_DEBUG("HIT - '{}/{}' {} '{}' (UPSTREAM)", - Key.Bucket, - Key.Hash, - NiceBytes(UpstreamValue.Size()), - ToString(UpstreamValue.GetContentType())); + Params.CacheKey.Bucket, + Params.CacheKey.Hash, + NiceBytes(Params.Record.GetView().GetSize()), + ToString(HttpContentType::kCbObject)); - CacheValues[KeyIndex] = IoBufferBuilder::MakeCloneFromMemory(CacheRecord.GetView()); + CacheValues[Params.KeyIndex] = IoBufferBuilder::MakeCloneFromMemory(Params.Record.GetView()); m_CacheStats.UpstreamHitCount++; } - }); + else + { + Missing.push_back(Params.KeyIndex); + } + }; - for (size_t MissingUpstream : UpstreamResult.Missing) - { - Missing.push_back(MissingUpstream); - } + m_UpstreamCache->GetCacheRecords(CacheKeys, UpstreamRequests, Policy, std::move(OnCacheRecordGetComplete)); } for (size_t KeyIndex : Missing) @@ -1104,40 +1099,32 @@ HttpStructuredCacheService::HandleBatchGetCachePayloads(zen::HttpServerRequest& if (!UpstreamRequests.empty() && m_UpstreamCache) { - auto UpstreamResult = m_UpstreamCache->GetCachePayloads( - ChunkRequests, - UpstreamRequests, - [this, &ChunkRequests, &Chunks](size_t ChunkIndex, IoBuffer UpstreamValue) { - const CacheChunkRequest& ChunkRequest = ChunkRequests[ChunkIndex]; - - if (CompressedBuffer CompressedChunk = CompressedBuffer::FromCompressed(SharedBuffer(UpstreamValue))) - { - auto InsertResult = m_CidStore.AddChunk(CompressedChunk); - IoBuffer Chunk = CompressedChunk.GetCompressed().Flatten().AsIoBuffer(); + const auto OnCachePayloadGetComplete = [this, &ChunkRequests, &Chunks, &Missing](CachePayloadGetCompleteParams&& Params) { + if (Params.Payload) + { + CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Params.Payload)); + auto InsertResult = m_CidStore.AddChunk(Compressed); - ZEN_DEBUG("HIT - '{}/{}/{}' {} '{}' ({})", - ChunkRequest.Key.Bucket, - ChunkRequest.Key.Hash, - ChunkRequest.ChunkId, - NiceBytes(Chunk.GetSize()), - ToString(Chunk.GetContentType()), - "UPSTREAM"); + ZEN_DEBUG("HIT - '{}/{}/{}' {} '{}' ({})", + Params.Request.Key.Bucket, + Params.Request.Key.Hash, + Params.Request.ChunkId, + NiceBytes(Params.Payload.GetSize()), + ToString(Params.Payload.GetContentType()), + "UPSTREAM"); - Chunks[ChunkIndex] = Chunk; + Chunks[Params.RequestIndex] = std::move(Params.Payload); - m_CacheStats.HitCount++; - m_CacheStats.UpstreamHitCount++; - } - else - { - ZEN_WARN("got uncompressed upstream cache payload"); - } - }); + m_CacheStats.HitCount++; + m_CacheStats.UpstreamHitCount++; + } + else + { + Missing.push_back(Params.RequestIndex); + } + }; - for (size_t MissingUpstream : UpstreamResult.Missing) - { - Missing.push_back(MissingUpstream); - } + m_UpstreamCache->GetCachePayloads(ChunkRequests, UpstreamRequests, std::move(OnCachePayloadGetComplete)); } for (size_t RequestIndex : Missing) diff --git a/zenserver/upstream/upstreamcache.cpp b/zenserver/upstream/upstreamcache.cpp index 7ef0caf62..0a0706656 100644 --- a/zenserver/upstream/upstreamcache.cpp +++ b/zenserver/upstream/upstreamcache.cpp @@ -70,7 +70,7 @@ namespace detail { virtual std::string_view DisplayName() const override { return m_DisplayName; } - virtual GetUpstreamCacheResult GetCacheRecord(UpstreamCacheKey CacheKey, ZenContentType Type) override + virtual GetUpstreamCacheResult GetCacheRecord(CacheKey CacheKey, ZenContentType Type) override { try { @@ -144,12 +144,54 @@ namespace detail { } } - virtual GetUpstreamCacheResult GetCachePayload(UpstreamPayloadKey PayloadKey) override + virtual GetUpstreamCacheBatchResult GetCacheRecords(std::span CacheKeys, + std::span KeyIndex, + const CacheRecordPolicy& Policy, + OnCacheRecordGetComplete&& OnComplete) override + { + ZEN_UNUSED(Policy); + + CloudCacheSession Session(m_Client); + + for (size_t Index : KeyIndex) + { + const CacheKey& CacheKey = CacheKeys[Index]; + CloudCacheResult Result = Session.GetRef(CacheKey.Bucket, CacheKey.Hash, ZenContentType::kCbObject); + + CbPackage Package; + CbObjectView Record; + + if (Result.Success) + { + const CbValidateError ValidationResult = ValidateCompactBinary(Result.Response, CbValidateMode::All); + if (ValidationResult == CbValidateError::None) + { + Record = CbObjectView(Result.Response.GetData()); + Record.IterateAttachments([&Session, &Result, &Package](CbFieldView AttachmentHash) { + CloudCacheResult AttachmentResult = Session.GetCompressedBlob(AttachmentHash.AsHash()); + if (AttachmentResult.Success) + { + if (CompressedBuffer Chunk = CompressedBuffer::FromCompressed(SharedBuffer(AttachmentResult.Response))) + { + Package.AddAttachment(CbAttachment(Chunk)); + } + } + }); + } + } + + OnComplete({.CacheKey = CacheKey, .KeyIndex = Index, .Record = Record, .Package = Package}); + } + + return {}; + } + + virtual GetUpstreamCacheResult GetCachePayload(const CacheKey&, const IoHash& PayloadId) override { try { CloudCacheSession Session(m_Client); - const CloudCacheResult Result = Session.GetCompressedBlob(PayloadKey.PayloadId); + const CloudCacheResult Result = Session.GetCompressedBlob(PayloadId); if (Result.ErrorCode == 0) { @@ -171,6 +213,29 @@ namespace detail { } } + virtual GetUpstreamCacheBatchResult GetCachePayloads(std::span CacheChunkRequests, + std::span RequestIndex, + OnCachePayloadGetComplete&& OnComplete) override final + { + CloudCacheSession Session(m_Client); + + for (size_t Index : RequestIndex) + { + const CacheChunkRequest& Request = CacheChunkRequests[Index]; + const CloudCacheResult Result = Session.GetCompressedBlob(Request.ChunkId); + + OnComplete({.Request = Request, .RequestIndex = Index, .Payload = Result.Response}); + + if (Result.ErrorCode) + { + m_HealthOk = false; + break; + } + } + + return {}; + } + virtual PutUpstreamCacheResult PutCacheRecord(const UpstreamCacheRecord& CacheRecord, IoBuffer RecordValue, std::span Payloads) override @@ -419,7 +484,7 @@ namespace detail { virtual std::string_view DisplayName() const override { return m_DisplayName; } - virtual GetUpstreamCacheResult GetCacheRecord(UpstreamCacheKey CacheKey, ZenContentType Type) override + virtual GetUpstreamCacheResult GetCacheRecord(CacheKey CacheKey, ZenContentType Type) override { try { @@ -446,13 +511,81 @@ namespace detail { } } - virtual GetUpstreamCacheResult GetCachePayload(UpstreamPayloadKey PayloadKey) override + virtual GetUpstreamCacheBatchResult GetCacheRecords(std::span CacheKeys, + std::span KeyIndex, + const CacheRecordPolicy& Policy, + OnCacheRecordGetComplete&& OnComplete) override + { + std::vector IndexMap; + IndexMap.reserve(KeyIndex.size()); + + CbObjectWriter BatchRequest; + BatchRequest << "Method"sv + << "GetCacheRecords"; + + BatchRequest.BeginObject("Params"sv); + { + BatchRequest.BeginArray("CacheKeys"sv); + for (size_t Index : KeyIndex) + { + const CacheKey& Key = CacheKeys[Index]; + IndexMap.push_back(Index); + + BatchRequest.BeginObject(); + BatchRequest << "Bucket"sv << Key.Bucket; + BatchRequest << "Hash"sv << Key.Hash; + BatchRequest.EndObject(); + } + BatchRequest.EndArray(); + + BatchRequest.BeginObject("Policy"sv); + CacheRecordPolicy::Save(Policy, BatchRequest); + BatchRequest.EndObject(); + } + BatchRequest.EndObject(); + + CbPackage BatchResponse; + bool Success = false; + + { + ZenStructuredCacheSession Session(*m_Client); + ZenCacheResult Result = Session.InvokeRpc(BatchRequest.Save()); + if (Result.Success) + { + Success = BatchResponse.TryLoad(Result.Response); + } + else if (Result.ErrorCode) + { + Success = m_HealthOk = false; + } + } + + if (!Success) + { + for (size_t Index : KeyIndex) + { + OnComplete({.CacheKey = CacheKeys[Index], .KeyIndex = Index, .Record = CbObjectView(), .Package = CbPackage()}); + } + + return {}; + } + + for (size_t LocalIndex = 0; CbFieldView Record : BatchResponse.GetObject()["Result"sv]) + { + const size_t Index = IndexMap[LocalIndex++]; + OnComplete({.CacheKey = CacheKeys[Index], .KeyIndex = Index, .Record = Record.AsObjectView(), .Package = BatchResponse}); + } + + return {}; + } + + virtual GetUpstreamCacheResult GetCachePayload(const CacheKey& CacheKey, const IoHash& PayloadId) override { try { ZenStructuredCacheSession Session(*m_Client); const ZenCacheResult Result = - Session.GetCachePayload(PayloadKey.CacheKey.Bucket, PayloadKey.CacheKey.Hash, PayloadKey.PayloadId); + Session.GetCachePayload(CacheKey.Bucket, CacheKey.Hash, PayloadId); if (Result.ErrorCode == 0) { @@ -474,6 +607,91 @@ namespace detail { } } + virtual GetUpstreamCacheBatchResult GetCachePayloads(std::span CacheChunkRequests, + std::span RequestIndex, + OnCachePayloadGetComplete&& OnComplete) override final + { + std::vector IndexMap; + IndexMap.reserve(RequestIndex.size()); + + CbObjectWriter BatchRequest; + BatchRequest << "Method"sv + << "GetCachePayloads"; + + BatchRequest.BeginObject("Params"sv); + { + BatchRequest.BeginArray("ChunkRequests"sv); + { + for (size_t Index : RequestIndex) + { + const CacheChunkRequest& Request = CacheChunkRequests[Index]; + IndexMap.push_back(Index); + + BatchRequest.BeginObject(); + { + BatchRequest.BeginObject("Key"sv); + BatchRequest << "Bucket"sv << Request.Key.Bucket; + BatchRequest << "Hash"sv << Request.Key.Hash; + BatchRequest.EndObject(); + + BatchRequest.AddObjectId("PayloadId"sv, Request.PayloadId); + BatchRequest << "ChunkId"sv << Request.ChunkId; + BatchRequest << "RawOffset"sv << Request.RawOffset; + BatchRequest << "RawSize"sv << Request.RawSize; + BatchRequest << "Policy"sv << static_cast(Request.Policy); + } + BatchRequest.EndObject(); + } + } + BatchRequest.EndArray(); + } + BatchRequest.EndObject(); + + CbPackage BatchResponse; + bool Success = false; + + { + ZenStructuredCacheSession Session(*m_Client); + ZenCacheResult Result = Session.InvokeRpc(BatchRequest.Save()); + if (Result.Success) + { + Success = BatchResponse.TryLoad(Result.Response); + } + else if (Result.ErrorCode) + { + m_HealthOk = false; + } + } + + if (!Success) + { + for (size_t Index : RequestIndex) + { + OnComplete({.Request = CacheChunkRequests[Index], .RequestIndex = Index, .Payload = IoBuffer()}); + } + + return {}; + } + + for (int32_t LocalIndex = 0; CbFieldView AttachmentHash : BatchResponse.GetObject()["Result"sv]) + { + const size_t Index = IndexMap[LocalIndex++]; + IoBuffer Payload; + + if (const CbAttachment* Attachment = BatchResponse.FindAttachment(AttachmentHash.AsHash())) + { + if (const CompressedBuffer& Compressed = Attachment->AsCompressedBinary()) + { + Payload = Compressed.GetCompressed().Flatten().AsIoBuffer(); + } + } + + OnComplete({.Request = CacheChunkRequests[Index], .RequestIndex = Index, .Payload = std::move(Payload)}); + } + + return {}; + } + virtual PutUpstreamCacheResult PutCacheRecord(const UpstreamCacheRecord& CacheRecord, IoBuffer RecordValue, std::span Payloads) override @@ -758,7 +976,7 @@ public: virtual void RegisterEndpoint(std::unique_ptr Endpoint) override { m_Endpoints.emplace_back(std::move(Endpoint)); } - virtual GetUpstreamCacheResult GetCacheRecord(UpstreamCacheKey CacheKey, ZenContentType Type) override + virtual GetUpstreamCacheResult GetCacheRecord(CacheKey CacheKey, ZenContentType Type) override { if (m_Options.ReadUpstream) { @@ -780,94 +998,82 @@ public: return {}; } - virtual GetUpstreamCacheBatchResult GetCacheRecords(std::span CacheKeys, - std::span KeyIndex, - OnCacheGetComplete OnComplete) override final + virtual void GetCacheRecords(std::span CacheKeys, + std::span KeyIndex, + const CacheRecordPolicy& Policy, + OnCacheRecordGetComplete&& OnComplete) override final { - if (!m_Options.ReadUpstream) - { - return {.Missing = std::vector(KeyIndex.begin(), KeyIndex.end())}; - } + std::vector MissingKeys(KeyIndex.begin(), KeyIndex.end()); - GetUpstreamCacheBatchResult Result; - - for (size_t Idx : KeyIndex) + if (m_Options.ReadUpstream) { - const UpstreamCacheKey CacheKey = {CacheKeys[Idx].Bucket, CacheKeys[Idx].Hash}; - - GetUpstreamCacheResult CacheResult; for (auto& Endpoint : m_Endpoints) { - if (Endpoint->IsHealthy()) + if (Endpoint->IsHealthy() && !MissingKeys.empty()) { - CacheResult = Endpoint->GetCacheRecord(CacheKey, ZenContentType::kCbPackage); - m_Stats.Add(m_Log, *Endpoint, CacheResult, m_Endpoints); + std::vector Missing; - if (CacheResult.Success) - { - break; - } - } - } + auto EndpointResult = + Endpoint->GetCacheRecords(CacheKeys, MissingKeys, Policy, [&](CacheRecordGetCompleteParams&& Params) { + if (Params.Record) + { + OnComplete(std::forward(Params)); + } + else + { + Missing.push_back(Params.KeyIndex); + } + }); - if (CacheResult.Success) - { - OnComplete(Idx, CacheResult.Value); - } - else - { - Result.Missing.push_back(Idx); + MissingKeys = std::move(Missing); + } } } - return Result; - } - - virtual GetUpstreamCacheBatchResult GetCachePayloads(std::span CacheChunkRequests, - std::span ChunkIndex, - OnCacheGetComplete OnComplete) override final - { - if (!m_Options.ReadUpstream) + for (size_t Index : MissingKeys) { - return {.Missing = std::vector(ChunkIndex.begin(), ChunkIndex.end())}; + OnComplete({.CacheKey = CacheKeys[Index], .KeyIndex = Index, .Record = CbObjectView(), .Package = CbPackage()}); } + } - GetUpstreamCacheBatchResult Result; + virtual void GetCachePayloads(std::span CacheChunkRequests, + std::span RequestIndex, + OnCachePayloadGetComplete&& OnComplete) override final + { + std::vector MissingPayloads(RequestIndex.begin(), RequestIndex.end()); - for (size_t Idx : ChunkIndex) + if (m_Options.ReadUpstream) { - const CacheChunkRequest& Chunk = CacheChunkRequests[Idx]; - UpstreamPayloadKey PayloadKey{{Chunk.Key.Bucket, Chunk.Key.Hash}, Chunk.ChunkId}; - - GetUpstreamCacheResult CacheResult; for (auto& Endpoint : m_Endpoints) { - if (Endpoint->IsHealthy()) + if (Endpoint->IsHealthy() && !MissingPayloads.empty()) { - CacheResult = Endpoint->GetCachePayload(PayloadKey); - m_Stats.Add(m_Log, *Endpoint, CacheResult, m_Endpoints); + std::vector Missing; - if (CacheResult.Success) - { - break; - } - } - } + auto EndpointResult = + Endpoint->GetCachePayloads(CacheChunkRequests, MissingPayloads, [&](CachePayloadGetCompleteParams&& Params) { + if (Params.Payload) + { + OnComplete(std::forward(Params)); + } + else + { + Missing.push_back(Params.RequestIndex); + } + }); - if (CacheResult.Success) - { - OnComplete(Idx, CacheResult.Value); - } - else - { - Result.Missing.push_back(Idx); + MissingPayloads = std::move(Missing); + } } } - return Result; + for (size_t Index : MissingPayloads) + { + OnComplete({.Request = CacheChunkRequests[Index], .RequestIndex = Index, .Payload = IoBuffer()}); + } } - virtual GetUpstreamCacheResult GetCachePayload(UpstreamPayloadKey PayloadKey) override + virtual GetUpstreamCacheResult GetCachePayload(const CacheKey& CacheKey, const IoHash& PayloadId) override { if (m_Options.ReadUpstream) { @@ -875,7 +1081,7 @@ public: { if (Endpoint->IsHealthy()) { - const GetUpstreamCacheResult Result = Endpoint->GetCachePayload(PayloadKey); + const GetUpstreamCacheResult Result = Endpoint->GetCachePayload(CacheKey, PayloadId); m_Stats.Add(m_Log, *Endpoint, Result, m_Endpoints); if (Result.Success) diff --git a/zenserver/upstream/upstreamcache.h b/zenserver/upstream/upstreamcache.h index a7bae302d..67bb73b4d 100644 --- a/zenserver/upstream/upstreamcache.h +++ b/zenserver/upstream/upstreamcache.h @@ -15,27 +15,17 @@ namespace zen { +class CbObjectView; +class CbPackage; class CbObjectWriter; class CidStore; class ZenCacheStore; struct CloudCacheClientOptions; -struct UpstreamCacheKey -{ - std::string Bucket; - IoHash Hash; -}; - -struct UpstreamPayloadKey -{ - UpstreamCacheKey CacheKey; - IoHash PayloadId; -}; - struct UpstreamCacheRecord { ZenContentType Type = ZenContentType::kBinary; - UpstreamCacheKey CacheKey; + CacheKey CacheKey; std::vector PayloadIds; }; @@ -98,6 +88,25 @@ struct UpstreamEndpointStats using OnCacheGetComplete = std::function; +struct CacheRecordGetCompleteParams +{ + const CacheKey& CacheKey; + size_t KeyIndex = ~size_t(0); + const CbObjectView& Record; + const CbPackage& Package; +}; + +using OnCacheRecordGetComplete = std::function; + +struct CachePayloadGetCompleteParams +{ + const CacheChunkRequest& Request; + size_t RequestIndex{~size_t(0)}; + IoBuffer Payload; +}; + +using OnCachePayloadGetComplete = std::function; + /** * The upstream endpont is responsible for handling upload/downloading of cache records. */ @@ -114,9 +123,18 @@ public: virtual std::string_view DisplayName() const = 0; - virtual GetUpstreamCacheResult GetCacheRecord(UpstreamCacheKey CacheKey, ZenContentType Type) = 0; + virtual GetUpstreamCacheResult GetCacheRecord(CacheKey CacheKey, ZenContentType Type) = 0; + + virtual GetUpstreamCacheBatchResult GetCacheRecords(std::span CacheKeys, + std::span KeyIndex, + const CacheRecordPolicy& Policy, + OnCacheRecordGetComplete&& OnComplete) = 0; - virtual GetUpstreamCacheResult GetCachePayload(UpstreamPayloadKey PayloadKey) = 0; + virtual GetUpstreamCacheResult GetCachePayload(const CacheKey& CacheKey, const IoHash& PayloadId) = 0; + + virtual GetUpstreamCacheBatchResult GetCachePayloads(std::span CacheChunkRequests, + std::span RequestIndex, + OnCachePayloadGetComplete&& OnComplete) = 0; virtual PutUpstreamCacheResult PutCacheRecord(const UpstreamCacheRecord& CacheRecord, IoBuffer RecordValue, @@ -137,17 +155,18 @@ public: virtual void RegisterEndpoint(std::unique_ptr Endpoint) = 0; - virtual GetUpstreamCacheResult GetCacheRecord(UpstreamCacheKey CacheKey, ZenContentType Type) = 0; + virtual GetUpstreamCacheResult GetCacheRecord(CacheKey CacheKey, ZenContentType Type) = 0; - virtual GetUpstreamCacheBatchResult GetCacheRecords(std::span CacheKeys, - std::span KeyIndex, - OnCacheGetComplete OnComplete) = 0; + virtual void GetCacheRecords(std::span CacheKeys, + std::span KeyIndex, + const CacheRecordPolicy& RecordPolicy, + OnCacheRecordGetComplete&& OnComplete) = 0; - virtual GetUpstreamCacheResult GetCachePayload(UpstreamPayloadKey PayloadKey) = 0; + virtual GetUpstreamCacheResult GetCachePayload(const CacheKey& CacheKey, const IoHash& PayloadId) = 0; - virtual GetUpstreamCacheBatchResult GetCachePayloads(std::span CacheChunkRequests, - std::span RequestIndex, - OnCacheGetComplete OnComplete) = 0; + virtual void GetCachePayloads(std::span CacheChunkRequests, + std::span RequestIndex, + OnCachePayloadGetComplete&& OnComplete) = 0; struct EnqueueResult { diff --git a/zenserver/upstream/zen.cpp b/zenserver/upstream/zen.cpp index 14333f45a..d11058180 100644 --- a/zenserver/upstream/zen.cpp +++ b/zenserver/upstream/zen.cpp @@ -499,4 +499,33 @@ ZenStructuredCacheSession::PutCachePayload(std::string_view BucketId, const IoHa .Success = (Response.status_code == 200 || Response.status_code == 201)}; } +ZenCacheResult +ZenStructuredCacheSession::InvokeRpc(const CbObjectView& Request) +{ + ExtendableStringBuilder<256> Uri; + Uri << m_Client.ServiceUrl() << "/z$/$batch"; + + BinaryWriter Body; + Request.CopyTo(Body); + + cpr::Session& Session = m_SessionState->Session; + + Session.SetOption(cpr::Url{Uri.c_str()}); + Session.SetHeader(cpr::Header{{"Content-Type", "application/x-ue-cbobject"}, {"Accept", "application/x-ue-cbpkg"}}); + Session.SetBody(cpr::Body{reinterpret_cast(Body.GetData()), Body.GetSize()}); + + cpr::Response Response = Session.Post(); + ZEN_DEBUG("POST {}", Response); + + if (Response.error) + { + return {.ErrorCode = static_cast(Response.error.code), .Reason = std::move(Response.error.message)}; + } + + const bool Success = Response.status_code == 200; + const IoBuffer Buffer = Success ? IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()) : IoBuffer(); + + return {.Response = std::move(Buffer), .Bytes = Response.uploaded_bytes, .ElapsedSeconds = Response.elapsed, .Success = Success}; +} + } // namespace zen diff --git a/zenserver/upstream/zen.h b/zenserver/upstream/zen.h index 12e46bd8d..5efe19094 100644 --- a/zenserver/upstream/zen.h +++ b/zenserver/upstream/zen.h @@ -28,6 +28,8 @@ class logger; namespace zen { class CbObjectWriter; +class CbObjectView; +class CbPackage; class ZenStructuredCacheClient; /** Zen mesh tracker @@ -116,6 +118,7 @@ public: ZenCacheResult GetCachePayload(std::string_view BucketId, const IoHash& Key, const IoHash& PayloadId); ZenCacheResult PutCacheRecord(std::string_view BucketId, const IoHash& Key, IoBuffer Value, ZenContentType Type); ZenCacheResult PutCachePayload(std::string_view BucketId, const IoHash& Key, const IoHash& PayloadId, IoBuffer Payload); + ZenCacheResult InvokeRpc(const CbObjectView& Request); private: inline spdlog::logger& Log() { return m_Log; } -- cgit v1.2.3 From 31ba344167677f175ec79ce7e579552a9811245d Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Thu, 11 Nov 2021 11:21:31 +0100 Subject: Format and remove unused type. --- zenserver/cache/structuredcache.cpp | 3 +-- zenserver/upstream/upstreamcache.cpp | 3 +-- zenserver/upstream/upstreamcache.h | 2 -- 3 files changed, 2 insertions(+), 6 deletions(-) (limited to 'zenserver') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 721942cc8..632368062 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -618,8 +618,7 @@ HttpStructuredCacheService::HandleGetCachePayload(zen::HttpServerRequest& Reques if (QueryUpstream) { - if (auto UpstreamResult = m_UpstreamCache->GetCachePayload({Ref.BucketSegment, Ref.HashKey}, Ref.PayloadId); - UpstreamResult.Success) + if (auto UpstreamResult = m_UpstreamCache->GetCachePayload({Ref.BucketSegment, Ref.HashKey}, Ref.PayloadId); UpstreamResult.Success) { if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(UpstreamResult.Value))) { diff --git a/zenserver/upstream/upstreamcache.cpp b/zenserver/upstream/upstreamcache.cpp index 0a0706656..a221c7d8e 100644 --- a/zenserver/upstream/upstreamcache.cpp +++ b/zenserver/upstream/upstreamcache.cpp @@ -584,8 +584,7 @@ namespace detail { try { ZenStructuredCacheSession Session(*m_Client); - const ZenCacheResult Result = - Session.GetCachePayload(CacheKey.Bucket, CacheKey.Hash, PayloadId); + const ZenCacheResult Result = Session.GetCachePayload(CacheKey.Bucket, CacheKey.Hash, PayloadId); if (Result.ErrorCode == 0) { diff --git a/zenserver/upstream/upstreamcache.h b/zenserver/upstream/upstreamcache.h index 67bb73b4d..71e7aed8d 100644 --- a/zenserver/upstream/upstreamcache.h +++ b/zenserver/upstream/upstreamcache.h @@ -86,8 +86,6 @@ struct UpstreamEndpointStats std::atomic SecondsDown{}; }; -using OnCacheGetComplete = std::function; - struct CacheRecordGetCompleteParams { const CacheKey& CacheKey; -- cgit v1.2.3 From efd0c3c0896c97c6cd37e6a6ac2edf331e5b4a3f Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Thu, 11 Nov 2021 11:37:31 +0100 Subject: Removed batch result. --- zenserver/upstream/upstreamcache.cpp | 34 +++++++++++++++++++--------------- zenserver/upstream/upstreamcache.h | 19 +++++++------------ 2 files changed, 26 insertions(+), 27 deletions(-) (limited to 'zenserver') diff --git a/zenserver/upstream/upstreamcache.cpp b/zenserver/upstream/upstreamcache.cpp index a221c7d8e..d5414daf1 100644 --- a/zenserver/upstream/upstreamcache.cpp +++ b/zenserver/upstream/upstreamcache.cpp @@ -144,10 +144,10 @@ namespace detail { } } - virtual GetUpstreamCacheBatchResult GetCacheRecords(std::span CacheKeys, - std::span KeyIndex, - const CacheRecordPolicy& Policy, - OnCacheRecordGetComplete&& OnComplete) override + virtual GetUpstreamCacheResult GetCacheRecords(std::span CacheKeys, + std::span KeyIndex, + const CacheRecordPolicy& Policy, + OnCacheRecordGetComplete&& OnComplete) override { ZEN_UNUSED(Policy); @@ -161,7 +161,7 @@ namespace detail { CbPackage Package; CbObjectView Record; - if (Result.Success) + if (Result.ErrorCode == 0) { const CbValidateError ValidationResult = ValidateCompactBinary(Result.Response, CbValidateMode::All); if (ValidationResult == CbValidateError::None) @@ -179,6 +179,10 @@ namespace detail { }); } } + else + { + m_HealthOk = false; + } OnComplete({.CacheKey = CacheKey, .KeyIndex = Index, .Record = Record, .Package = Package}); } @@ -213,9 +217,9 @@ namespace detail { } } - virtual GetUpstreamCacheBatchResult GetCachePayloads(std::span CacheChunkRequests, - std::span RequestIndex, - OnCachePayloadGetComplete&& OnComplete) override final + virtual GetUpstreamCacheResult GetCachePayloads(std::span CacheChunkRequests, + std::span RequestIndex, + OnCachePayloadGetComplete&& OnComplete) override final { CloudCacheSession Session(m_Client); @@ -511,10 +515,10 @@ namespace detail { } } - virtual GetUpstreamCacheBatchResult GetCacheRecords(std::span CacheKeys, - std::span KeyIndex, - const CacheRecordPolicy& Policy, - OnCacheRecordGetComplete&& OnComplete) override + virtual GetUpstreamCacheResult GetCacheRecords(std::span CacheKeys, + std::span KeyIndex, + const CacheRecordPolicy& Policy, + OnCacheRecordGetComplete&& OnComplete) override { std::vector IndexMap; IndexMap.reserve(KeyIndex.size()); @@ -606,9 +610,9 @@ namespace detail { } } - virtual GetUpstreamCacheBatchResult GetCachePayloads(std::span CacheChunkRequests, - std::span RequestIndex, - OnCachePayloadGetComplete&& OnComplete) override final + virtual GetUpstreamCacheResult GetCachePayloads(std::span CacheChunkRequests, + std::span RequestIndex, + OnCachePayloadGetComplete&& OnComplete) override final { std::vector IndexMap; IndexMap.reserve(RequestIndex.size()); diff --git a/zenserver/upstream/upstreamcache.h b/zenserver/upstream/upstreamcache.h index 71e7aed8d..681d8e96f 100644 --- a/zenserver/upstream/upstreamcache.h +++ b/zenserver/upstream/upstreamcache.h @@ -55,11 +55,6 @@ struct GetUpstreamCacheResult bool Success = false; }; -struct GetUpstreamCacheBatchResult -{ - std::vector Missing; -}; - struct PutUpstreamCacheResult { std::string Reason; @@ -123,16 +118,16 @@ public: virtual GetUpstreamCacheResult GetCacheRecord(CacheKey CacheKey, ZenContentType Type) = 0; - virtual GetUpstreamCacheBatchResult GetCacheRecords(std::span CacheKeys, - std::span KeyIndex, - const CacheRecordPolicy& Policy, - OnCacheRecordGetComplete&& OnComplete) = 0; + virtual GetUpstreamCacheResult GetCacheRecords(std::span CacheKeys, + std::span KeyIndex, + const CacheRecordPolicy& Policy, + OnCacheRecordGetComplete&& OnComplete) = 0; virtual GetUpstreamCacheResult GetCachePayload(const CacheKey& CacheKey, const IoHash& PayloadId) = 0; - virtual GetUpstreamCacheBatchResult GetCachePayloads(std::span CacheChunkRequests, - std::span RequestIndex, - OnCachePayloadGetComplete&& OnComplete) = 0; + virtual GetUpstreamCacheResult GetCachePayloads(std::span CacheChunkRequests, + std::span RequestIndex, + OnCachePayloadGetComplete&& OnComplete) = 0; virtual PutUpstreamCacheResult PutCacheRecord(const UpstreamCacheRecord& CacheRecord, IoBuffer RecordValue, -- cgit v1.2.3 From 7940ae2ff24e1b708d1d6a0bccf266213ea7316b Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Thu, 11 Nov 2021 13:28:52 +0100 Subject: Fixed stats. --- zenserver/upstream/upstreamcache.cpp | 211 ++++++++++++++++++++--------------- 1 file changed, 119 insertions(+), 92 deletions(-) (limited to 'zenserver') diff --git a/zenserver/upstream/upstreamcache.cpp b/zenserver/upstream/upstreamcache.cpp index d5414daf1..b486b751c 100644 --- a/zenserver/upstream/upstreamcache.cpp +++ b/zenserver/upstream/upstreamcache.cpp @@ -151,43 +151,54 @@ namespace detail { { ZEN_UNUSED(Policy); - CloudCacheSession Session(m_Client); + CloudCacheSession Session(m_Client); + GetUpstreamCacheResult Result; for (size_t Index : KeyIndex) { - const CacheKey& CacheKey = CacheKeys[Index]; - CloudCacheResult Result = Session.GetRef(CacheKey.Bucket, CacheKey.Hash, ZenContentType::kCbObject); + const CacheKey& CacheKey = CacheKeys[Index]; + CbPackage Package; + CbObjectView Record; - CbPackage Package; - CbObjectView Record; - - if (Result.ErrorCode == 0) + if (!Result.Error) { - const CbValidateError ValidationResult = ValidateCompactBinary(Result.Response, CbValidateMode::All); - if (ValidationResult == CbValidateError::None) + CloudCacheResult RefResult = Session.GetRef(CacheKey.Bucket, CacheKey.Hash, ZenContentType::kCbObject); + AppendResult(RefResult, Result); + + if (RefResult.ErrorCode == 0) { - Record = CbObjectView(Result.Response.GetData()); - Record.IterateAttachments([&Session, &Result, &Package](CbFieldView AttachmentHash) { - CloudCacheResult AttachmentResult = Session.GetCompressedBlob(AttachmentHash.AsHash()); - if (AttachmentResult.Success) - { - if (CompressedBuffer Chunk = CompressedBuffer::FromCompressed(SharedBuffer(AttachmentResult.Response))) + const CbValidateError ValidationResult = ValidateCompactBinary(RefResult.Response, CbValidateMode::All); + if (ValidationResult == CbValidateError::None) + { + Record = CbObjectView(RefResult.Response.GetData()); + Record.IterateAttachments([this, &Session, &Result, &Package](CbFieldView AttachmentHash) { + CloudCacheResult BlobResult = Session.GetCompressedBlob(AttachmentHash.AsHash()); + AppendResult(BlobResult, Result); + + if (BlobResult.ErrorCode == 0) { - Package.AddAttachment(CbAttachment(Chunk)); + if (CompressedBuffer Chunk = CompressedBuffer::FromCompressed(SharedBuffer(BlobResult.Response))) + { + Package.AddAttachment(CbAttachment(Chunk)); + } } - } - }); + else + { + m_HealthOk = false; + } + }); + } + } + else + { + m_HealthOk = false; } - } - else - { - m_HealthOk = false; } OnComplete({.CacheKey = CacheKey, .KeyIndex = Index, .Record = Record, .Package = Package}); } - return {}; + return Result; } virtual GetUpstreamCacheResult GetCachePayload(const CacheKey&, const IoHash& PayloadId) override @@ -221,23 +232,27 @@ namespace detail { std::span RequestIndex, OnCachePayloadGetComplete&& OnComplete) override final { - CloudCacheSession Session(m_Client); + CloudCacheSession Session(m_Client); + GetUpstreamCacheResult Result; for (size_t Index : RequestIndex) { const CacheChunkRequest& Request = CacheChunkRequests[Index]; - const CloudCacheResult Result = Session.GetCompressedBlob(Request.ChunkId); - - OnComplete({.Request = Request, .RequestIndex = Index, .Payload = Result.Response}); + IoBuffer Payload; - if (Result.ErrorCode) + if (!Result.Error) { - m_HealthOk = false; - break; + const CloudCacheResult BlobResult = Session.GetCompressedBlob(Request.ChunkId); + Payload = BlobResult.Response; + + AppendResult(BlobResult, Result); + m_HealthOk = BlobResult.ErrorCode == 0; } + + OnComplete({.Request = Request, .RequestIndex = Index, .Payload = Payload}); } - return {}; + return Result; } virtual PutUpstreamCacheResult PutCacheRecord(const UpstreamCacheRecord& CacheRecord, @@ -392,6 +407,18 @@ namespace detail { virtual UpstreamEndpointStats& Stats() override { return m_Stats; } private: + static void AppendResult(const CloudCacheResult& Result, GetUpstreamCacheResult& Out) + { + Out.Success &= Result.Success; + Out.Bytes += Result.Bytes; + Out.ElapsedSeconds += Result.ElapsedSeconds; + + if (Result.ErrorCode) + { + Out.Error = {.ErrorCode = Result.ErrorCode, .Reason = std::move(Result.Reason)}; + } + }; + spdlog::logger& Log() { return m_Log; } spdlog::logger& m_Log; @@ -548,39 +575,39 @@ namespace detail { } BatchRequest.EndObject(); - CbPackage BatchResponse; - bool Success = false; + CbPackage BatchResponse; + ZenCacheResult Result; { ZenStructuredCacheSession Session(*m_Client); - ZenCacheResult Result = Session.InvokeRpc(BatchRequest.Save()); - if (Result.Success) - { - Success = BatchResponse.TryLoad(Result.Response); - } - else if (Result.ErrorCode) - { - Success = m_HealthOk = false; - } + Result = Session.InvokeRpc(BatchRequest.Save()); } - if (!Success) + if (Result.Success) { - for (size_t Index : KeyIndex) + if (BatchResponse.TryLoad(Result.Response)) { - OnComplete({.CacheKey = CacheKeys[Index], .KeyIndex = Index, .Record = CbObjectView(), .Package = CbPackage()}); - } + for (size_t LocalIndex = 0; CbFieldView Record : BatchResponse.GetObject()["Result"sv]) + { + const size_t Index = IndexMap[LocalIndex++]; + OnComplete( + {.CacheKey = CacheKeys[Index], .KeyIndex = Index, .Record = Record.AsObjectView(), .Package = BatchResponse}); + } - return {}; + return {.Bytes = Result.Bytes, .ElapsedSeconds = Result.ElapsedSeconds, .Success = true}; + } + } + else if (Result.ErrorCode) + { + m_HealthOk = false; } - for (size_t LocalIndex = 0; CbFieldView Record : BatchResponse.GetObject()["Result"sv]) + for (size_t Index : KeyIndex) { - const size_t Index = IndexMap[LocalIndex++]; - OnComplete({.CacheKey = CacheKeys[Index], .KeyIndex = Index, .Record = Record.AsObjectView(), .Package = BatchResponse}); + OnComplete({.CacheKey = CacheKeys[Index], .KeyIndex = Index, .Record = CbObjectView(), .Package = CbPackage()}); } - return {}; + return {.Error{.ErrorCode = Result.ErrorCode, .Reason = std::move(Result.Reason)}}; } virtual GetUpstreamCacheResult GetCachePayload(const CacheKey& CacheKey, const IoHash& PayloadId) override @@ -650,49 +677,48 @@ namespace detail { } BatchRequest.EndObject(); - CbPackage BatchResponse; - bool Success = false; + CbPackage BatchResponse; + ZenCacheResult Result; { ZenStructuredCacheSession Session(*m_Client); - ZenCacheResult Result = Session.InvokeRpc(BatchRequest.Save()); - if (Result.Success) - { - Success = BatchResponse.TryLoad(Result.Response); - } - else if (Result.ErrorCode) - { - m_HealthOk = false; - } + Result = Session.InvokeRpc(BatchRequest.Save()); } - if (!Success) + if (Result.Success) { - for (size_t Index : RequestIndex) + if (BatchResponse.TryLoad(Result.Response)) { - OnComplete({.Request = CacheChunkRequests[Index], .RequestIndex = Index, .Payload = IoBuffer()}); - } - - return {}; - } + for (size_t LocalIndex = 0; CbFieldView AttachmentHash : BatchResponse.GetObject()["Result"sv]) + { + const size_t Index = IndexMap[LocalIndex++]; + IoBuffer Payload; - for (int32_t LocalIndex = 0; CbFieldView AttachmentHash : BatchResponse.GetObject()["Result"sv]) - { - const size_t Index = IndexMap[LocalIndex++]; - IoBuffer Payload; + if (const CbAttachment* Attachment = BatchResponse.FindAttachment(AttachmentHash.AsHash())) + { + if (const CompressedBuffer& Compressed = Attachment->AsCompressedBinary()) + { + Payload = Compressed.GetCompressed().Flatten().AsIoBuffer(); + } + } - if (const CbAttachment* Attachment = BatchResponse.FindAttachment(AttachmentHash.AsHash())) - { - if (const CompressedBuffer& Compressed = Attachment->AsCompressedBinary()) - { - Payload = Compressed.GetCompressed().Flatten().AsIoBuffer(); + OnComplete({.Request = CacheChunkRequests[Index], .RequestIndex = Index, .Payload = std::move(Payload)}); } + + return {.Bytes = Result.Bytes, .ElapsedSeconds = Result.ElapsedSeconds, .Success = true}; } + } + else if (Result.ErrorCode) + { + m_HealthOk = false; + } - OnComplete({.Request = CacheChunkRequests[Index], .RequestIndex = Index, .Payload = std::move(Payload)}); + for (size_t Index : RequestIndex) + { + OnComplete({.Request = CacheChunkRequests[Index], .RequestIndex = Index, .Payload = IoBuffer()}); } - return {}; + return {.Error{.ErrorCode = Result.ErrorCode, .Reason = std::move(Result.Reason)}}; } virtual PutUpstreamCacheResult PutCacheRecord(const UpstreamCacheRecord& CacheRecord, @@ -1016,18 +1042,18 @@ public: { std::vector Missing; - auto EndpointResult = - Endpoint->GetCacheRecords(CacheKeys, MissingKeys, Policy, [&](CacheRecordGetCompleteParams&& Params) { - if (Params.Record) - { - OnComplete(std::forward(Params)); - } - else - { - Missing.push_back(Params.KeyIndex); - } - }); + auto Result = Endpoint->GetCacheRecords(CacheKeys, MissingKeys, Policy, [&](CacheRecordGetCompleteParams&& Params) { + if (Params.Record) + { + OnComplete(std::forward(Params)); + } + else + { + Missing.push_back(Params.KeyIndex); + } + }); + m_Stats.Add(m_Log, *Endpoint, Result, m_Endpoints); MissingKeys = std::move(Missing); } } @@ -1053,7 +1079,7 @@ public: { std::vector Missing; - auto EndpointResult = + auto Result = Endpoint->GetCachePayloads(CacheChunkRequests, MissingPayloads, [&](CachePayloadGetCompleteParams&& Params) { if (Params.Payload) { @@ -1065,6 +1091,7 @@ public: } }); + m_Stats.Add(m_Log, *Endpoint, Result, m_Endpoints); MissingPayloads = std::move(Missing); } } -- cgit v1.2.3 From 64a475ae0f8c0ed3294798b86f9c918a270b7d21 Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Thu, 11 Nov 2021 13:52:08 +0100 Subject: Correct content type when invoking RPC. --- zenserver/upstream/zen.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'zenserver') diff --git a/zenserver/upstream/zen.cpp b/zenserver/upstream/zen.cpp index d11058180..dc23b573b 100644 --- a/zenserver/upstream/zen.cpp +++ b/zenserver/upstream/zen.cpp @@ -511,7 +511,7 @@ ZenStructuredCacheSession::InvokeRpc(const CbObjectView& Request) cpr::Session& Session = m_SessionState->Session; Session.SetOption(cpr::Url{Uri.c_str()}); - Session.SetHeader(cpr::Header{{"Content-Type", "application/x-ue-cbobject"}, {"Accept", "application/x-ue-cbpkg"}}); + Session.SetHeader(cpr::Header{{"Content-Type", "application/x-ue-cb"}, {"Accept", "application/x-ue-cbpkg"}}); Session.SetBody(cpr::Body{reinterpret_cast(Body.GetData()), Body.GetSize()}); cpr::Response Response = Session.Post(); -- cgit v1.2.3 From 3efc2ddb02511300cd6dfe59cd89ca4338f6ec4c Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Thu, 11 Nov 2021 15:05:11 +0100 Subject: Handle batch requests asynchronously. --- zenserver/cache/structuredcache.cpp | 264 ++++++++++++++++-------------------- 1 file changed, 120 insertions(+), 144 deletions(-) (limited to 'zenserver') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 632368062..10fbb3709 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -795,21 +795,22 @@ HttpStructuredCacheService::HandleBatchRequest(zen::HttpServerRequest& Request) return Request.WriteResponse(HttpResponseCode::BadRequest); } - CbObject BatchRequest = zen::LoadCompactBinaryObject(Request.ReadPayload()); - const std::string_view Method = BatchRequest["Method"sv].AsString(); - - if (Method == "GetCacheRecords"sv) - { - HandleBatchGetCacheRecords(Request, BatchRequest); - } - else if (Method == "GetCachePayloads"sv) - { - HandleBatchGetCachePayloads(Request, BatchRequest); - } - else - { - Request.WriteResponse(HttpResponseCode::BadRequest); - } + Request.WriteResponseAsync( + [this, BatchRequest = zen::LoadCompactBinaryObject(Request.ReadPayload())](HttpServerRequest& AsyncRequest) { + const std::string_view Method = BatchRequest["Method"sv].AsString(); + if (Method == "GetCacheRecords"sv) + { + HandleBatchGetCacheRecords(AsyncRequest, BatchRequest); + } + else if (Method == "GetCachePayloads"sv) + { + HandleBatchGetCachePayloads(AsyncRequest, BatchRequest); + } + else + { + AsyncRequest.WriteResponse(HttpResponseCode::BadRequest); + } + }); } break; default: @@ -823,27 +824,24 @@ HttpStructuredCacheService::HandleBatchGetCacheRecords(zen::HttpServerRequest& R { using namespace fmt::literals; - const std::string_view Method = BatchRequest["Method"sv].AsString(); - ZEN_ASSERT(Method == "GetCacheRecords"sv); + CbPackage BatchResponse; + CacheRecordPolicy Policy; + CbObjectView Params = BatchRequest["Params"sv].AsObjectView(); + std::vector CacheKeys; + std::vector CacheValues; + std::vector UpstreamRequests; - CbObjectView Params = BatchRequest["Params"sv].AsObjectView(); + ZEN_ASSERT(BatchRequest["Method"sv].AsString() == "GetCacheRecords"sv); - CacheRecordPolicy Policy; CacheRecordPolicy::Load(Params["Policy"sv].AsObjectView(), Policy); const bool SkipAttachments = (Policy.GetRecordPolicy() & CachePolicy::SkipAttachments) == CachePolicy::SkipAttachments; const bool QueryRemote = m_UpstreamCache && ((Policy.GetRecordPolicy() & CachePolicy::QueryRemote) == CachePolicy::QueryRemote); - std::vector CacheKeys; - std::vector CacheValues; - std::vector Payloads; - std::vector UpstreamRequests; - std::vector Missing; - - for (CbFieldView QueryView : Params["CacheKeys"sv]) + for (CbFieldView KeyView : Params["CacheKeys"sv]) { - CbObjectView Query = QueryView.AsObjectView(); - CacheKeys.push_back(CacheKey::Create(Query["Bucket"sv].AsString(), Query["Hash"sv].AsHash())); + CbObjectView KeyObject = KeyView.AsObjectView(); + CacheKeys.push_back(CacheKey::Create(KeyObject["Bucket"sv].AsString(), KeyObject["Hash"sv].AsHash())); } if (CacheKeys.empty()) @@ -853,11 +851,6 @@ HttpStructuredCacheService::HandleBatchGetCacheRecords(zen::HttpServerRequest& R CacheValues.resize(CacheKeys.size()); - if (!SkipAttachments) - { - Payloads.reserve(CacheKeys.size()); - } - for (size_t KeyIndex = 0; const CacheKey& Key : CacheKeys) { ZenCacheValue CacheValue; @@ -867,10 +860,10 @@ HttpStructuredCacheService::HandleBatchGetCacheRecords(zen::HttpServerRequest& R if (!SkipAttachments) { - CacheRecord.IterateAttachments([this, &Payloads](CbFieldView AttachmentHash) { + CacheRecord.IterateAttachments([this, &BatchResponse](CbFieldView AttachmentHash) { if (IoBuffer Chunk = m_CidStore.FindChunkByCid(AttachmentHash.AsHash())) { - Payloads.push_back(Chunk); + BatchResponse.AddAttachment(CbAttachment(CompressedBuffer::FromCompressed(SharedBuffer(Chunk)))); } }); } @@ -890,7 +883,8 @@ HttpStructuredCacheService::HandleBatchGetCacheRecords(zen::HttpServerRequest& R } else { - Missing.push_back(KeyIndex); + ZEN_DEBUG("MISS - '{}/{}'", Key.Bucket, Key.Hash); + m_CacheStats.MissCount++; } ++KeyIndex; @@ -899,19 +893,19 @@ HttpStructuredCacheService::HandleBatchGetCacheRecords(zen::HttpServerRequest& R if (!UpstreamRequests.empty() && m_UpstreamCache) { const auto OnCacheRecordGetComplete = - [this, &CacheKeys, &CacheValues, &Payloads, &Missing, SkipAttachments](CacheRecordGetCompleteParams&& Params) { + [this, &CacheKeys, &CacheValues, &BatchResponse, SkipAttachments](CacheRecordGetCompleteParams&& Params) { if (Params.Record) { Params.Record.IterateAttachments([&](CbFieldView AttachmentHash) { if (const CbAttachment* Attachment = Params.Package.FindAttachment(AttachmentHash.AsHash())) { - if (CompressedBuffer Chunk = Attachment->AsCompressedBinary()) + if (CompressedBuffer Compressed = Attachment->AsCompressedBinary()) { - m_CidStore.AddChunk(Chunk); + m_CidStore.AddChunk(Compressed); if (!SkipAttachments) { - Payloads.push_back(Chunk.GetCompressed().Flatten().AsIoBuffer()); + BatchResponse.AddAttachment(CbAttachment(Compressed)); } } } @@ -924,51 +918,40 @@ HttpStructuredCacheService::HandleBatchGetCacheRecords(zen::HttpServerRequest& R ToString(HttpContentType::kCbObject)); CacheValues[Params.KeyIndex] = IoBufferBuilder::MakeCloneFromMemory(Params.Record.GetView()); + m_CacheStats.HitCount++; m_CacheStats.UpstreamHitCount++; } else { - Missing.push_back(Params.KeyIndex); + ZEN_DEBUG("MISS - '{}/{}'", Params.CacheKey.Bucket, Params.CacheKey.Hash); + m_CacheStats.MissCount++; } }; m_UpstreamCache->GetCacheRecords(CacheKeys, UpstreamRequests, Policy, std::move(OnCacheRecordGetComplete)); } - for (size_t KeyIndex : Missing) - { - const CacheKey& Key = CacheKeys[KeyIndex]; - ZEN_DEBUG("MISS - '{}/{}'", Key.Bucket, Key.Hash); - m_CacheStats.MissCount++; - } + CbObjectWriter ResponseObject; - CbObjectWriter BatchResponse; - - BatchResponse.BeginArray("Result"sv); + ResponseObject.BeginArray("Result"sv); for (const IoBuffer& Value : CacheValues) { if (Value) { CbObjectView Record(Value.Data()); - BatchResponse << Record; + ResponseObject << Record; } else { - BatchResponse.AddNull(); + ResponseObject.AddNull(); } } - BatchResponse.EndArray(); + ResponseObject.EndArray(); - CbPackage Package; - Package.SetObject(BatchResponse.Save()); - - for (const IoBuffer& Payload : Payloads) - { - Package.AddAttachment(CbAttachment(CompressedBuffer::FromCompressed(SharedBuffer(Payload)))); - } + BatchResponse.SetObject(ResponseObject.Save()); BinaryWriter MemStream; - Package.Save(MemStream); + BatchResponse.Save(MemStream); Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, @@ -980,84 +963,84 @@ HttpStructuredCacheService::HandleBatchGetCachePayloads(zen::HttpServerRequest& { using namespace fmt::literals; - const auto GetChunkIdFromPayloadId = [](CbObjectView Record, const Oid& PayloadId) -> IoHash { - if (CbObjectView ValueObject = Record["Value"].AsObjectView()) - { - const Oid Id = ValueObject["Id"].AsObjectId(); - if (Id == PayloadId) - { - return ValueObject["RawHash"sv].AsHash(); - } - } - - for (CbFieldView AttachmentView : Record["Attachments"sv]) - { - CbObjectView AttachmentObject = AttachmentView.AsObjectView(); - const Oid Id = AttachmentObject["Id"].AsObjectId(); - - if (Id == PayloadId) - { - return AttachmentObject["RawHash"sv].AsHash(); - } - } - - return IoHash::Zero; - }; - - const std::string_view Method = BatchRequest["Method"sv].AsString(); - ZEN_ASSERT(Method == "GetCachePayloads"sv); - - CbObjectView Params = BatchRequest["Params"sv].AsObjectView(); + ZEN_ASSERT(BatchRequest["Method"sv].AsString() == "GetCachePayloads"sv); std::vector ChunkRequests; std::vector UpstreamRequests; - std::vector Missing; + std::vector Chunks; + CbObjectView Params = BatchRequest["Params"sv].AsObjectView(); - for (CbFieldView ChunkView : Params["ChunkRequests"sv]) + for (CbFieldView RequestView : Params["ChunkRequests"sv]) { - CbObjectView Chunk = ChunkView.AsObjectView(); - CbObjectView CacheKeyObject = Chunk["Key"sv].AsObjectView(); - const CacheKey Key = CacheKey::Create(CacheKeyObject["Bucket"sv].AsString(), CacheKeyObject["Hash"sv].AsHash()); - const IoHash ChunkId = IoHash::Zero; - const Oid PayloadId = Chunk["PayloadId"sv].AsObjectId(); - const uint64_t RawOffset = Chunk["RawoffSet"sv].AsUInt64(); - const uint64_t RawSize = Chunk["RawSize"sv].AsUInt64(); - const uint32_t ChunkPolicy = Chunk["Policy"sv].AsUInt32(); + CbObjectView RequestObject = RequestView.AsObjectView(); + CbObjectView KeyObject = RequestObject["Key"sv].AsObjectView(); + const CacheKey Key = CacheKey::Create(KeyObject["Bucket"sv].AsString(), KeyObject["Hash"sv].AsHash()); + const IoHash ChunkId = IoHash::Zero; + const Oid PayloadId = RequestObject["PayloadId"sv].AsObjectId(); + const uint64_t RawOffset = RequestObject["RawoffSet"sv].AsUInt64(); + const uint64_t RawSize = RequestObject["RawSize"sv].AsUInt64(); + const uint32_t ChunkPolicy = RequestObject["Policy"sv].AsUInt32(); ChunkRequests.emplace_back(Key, ChunkId, PayloadId, RawOffset, RawSize, static_cast(ChunkPolicy)); } - std::stable_sort(ChunkRequests.begin(), ChunkRequests.end()); + if (ChunkRequests.empty()) + { + return Request.WriteResponse(HttpResponseCode::BadRequest); + } - CacheKey CurrentKey = CacheKey::Empty; - IoBuffer CurrentRecordBuffer; + Chunks.resize(ChunkRequests.size()); - for (CacheChunkRequest& ChunkRequest : ChunkRequests) + // Try to find the uncompressed raw hash from the payload ID. { - if (ChunkRequest.Key != CurrentKey) - { - CurrentKey = ChunkRequest.Key; + const auto GetChunkIdFromPayloadId = [](CbObjectView Record, const Oid& PayloadId) -> IoHash { + if (CbObjectView ValueObject = Record["Value"].AsObjectView()) + { + const Oid Id = ValueObject["Id"].AsObjectId(); + if (Id == PayloadId) + { + return ValueObject["RawHash"sv].AsHash(); + } + } - ZenCacheValue CacheValue; - if (m_CacheStore.Get(ChunkRequest.Key.Bucket, ChunkRequest.Key.Hash, CacheValue)) + for (CbFieldView AttachmentView : Record["Attachments"sv]) { - CurrentRecordBuffer = CacheValue.Value; + CbObjectView AttachmentObject = AttachmentView.AsObjectView(); + const Oid Id = AttachmentObject["Id"].AsObjectId(); + + if (Id == PayloadId) + { + return AttachmentObject["RawHash"sv].AsHash(); + } } - } - if (CurrentRecordBuffer) + return IoHash::Zero; + }; + + CacheKey CurrentKey = CacheKey::Empty; + IoBuffer CurrentRecordBuffer; + + std::stable_sort(ChunkRequests.begin(), ChunkRequests.end()); + + for (CacheChunkRequest& ChunkRequest : ChunkRequests) { - ChunkRequest.ChunkId = GetChunkIdFromPayloadId(CbObjectView(CurrentRecordBuffer.GetData()), ChunkRequest.PayloadId); - } - } + if (ChunkRequest.Key != CurrentKey) + { + CurrentKey = ChunkRequest.Key; - if (ChunkRequests.empty()) - { - return Request.WriteResponse(HttpResponseCode::BadRequest); - } + ZenCacheValue CacheValue; + if (m_CacheStore.Get(ChunkRequest.Key.Bucket, ChunkRequest.Key.Hash, CacheValue)) + { + CurrentRecordBuffer = CacheValue.Value; + } + } - std::vector Chunks; - Chunks.resize(ChunkRequests.size()); + if (CurrentRecordBuffer) + { + ChunkRequest.ChunkId = GetChunkIdFromPayloadId(CbObjectView(CurrentRecordBuffer.GetData()), ChunkRequest.PayloadId); + } + } + } for (size_t RequestIndex = 0; const CacheChunkRequest& ChunkRequest : ChunkRequests) { @@ -1085,7 +1068,8 @@ HttpStructuredCacheService::HandleBatchGetCachePayloads(zen::HttpServerRequest& } else { - Missing.push_back(RequestIndex); + ZEN_DEBUG("MISS - '{}/{}/{}'", ChunkRequest.Key.Bucket, ChunkRequest.Key.Hash, ChunkRequest.ChunkId); + m_CacheStats.MissCount++; } } else @@ -1098,18 +1082,16 @@ HttpStructuredCacheService::HandleBatchGetCachePayloads(zen::HttpServerRequest& if (!UpstreamRequests.empty() && m_UpstreamCache) { - const auto OnCachePayloadGetComplete = [this, &ChunkRequests, &Chunks, &Missing](CachePayloadGetCompleteParams&& Params) { - if (Params.Payload) + const auto OnCachePayloadGetComplete = [this, &ChunkRequests, &Chunks](CachePayloadGetCompleteParams&& Params) { + if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Params.Payload))) { - CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Params.Payload)); - auto InsertResult = m_CidStore.AddChunk(Compressed); + auto InsertResult = m_CidStore.AddChunk(Compressed); - ZEN_DEBUG("HIT - '{}/{}/{}' {} '{}' ({})", + ZEN_DEBUG("HIT - '{}/{}/{}' {} ({})", Params.Request.Key.Bucket, Params.Request.Key.Hash, Params.Request.ChunkId, NiceBytes(Params.Payload.GetSize()), - ToString(Params.Payload.GetContentType()), "UPSTREAM"); Chunks[Params.RequestIndex] = std::move(Params.Payload); @@ -1119,43 +1101,37 @@ HttpStructuredCacheService::HandleBatchGetCachePayloads(zen::HttpServerRequest& } else { - Missing.push_back(Params.RequestIndex); + ZEN_DEBUG("MISS - '{}/{}/{}'", Params.Request.Key.Bucket, Params.Request.Key.Hash, Params.Request.ChunkId); + m_CacheStats.MissCount++; } }; m_UpstreamCache->GetCachePayloads(ChunkRequests, UpstreamRequests, std::move(OnCachePayloadGetComplete)); } - for (size_t RequestIndex : Missing) - { - const CacheChunkRequest& ChunkRequest = ChunkRequests[RequestIndex]; - ZEN_DEBUG("MISS - '{}/{}/{}'", ChunkRequest.Key.Bucket, ChunkRequest.Key.Hash, ChunkRequest.ChunkId); - m_CacheStats.MissCount++; - } - - CbPackage Package; - CbObjectWriter BatchResponse; + CbPackage BatchResponse; + CbObjectWriter ResponseObject; - BatchResponse.BeginArray("Result"sv); + ResponseObject.BeginArray("Result"sv); for (size_t ChunkIndex = 0; ChunkIndex < Chunks.size(); ++ChunkIndex) { if (Chunks[ChunkIndex]) { - BatchResponse << ChunkRequests[ChunkIndex].ChunkId; - Package.AddAttachment(CbAttachment(CompressedBuffer::FromCompressed(SharedBuffer(std::move(Chunks[ChunkIndex]))))); + ResponseObject << ChunkRequests[ChunkIndex].ChunkId; + BatchResponse.AddAttachment(CbAttachment(CompressedBuffer::FromCompressed(SharedBuffer(std::move(Chunks[ChunkIndex]))))); } else { - BatchResponse << IoHash::Zero; + ResponseObject << IoHash::Zero; } } - BatchResponse.EndArray(); + ResponseObject.EndArray(); - Package.SetObject(BatchResponse.Save()); + BatchResponse.SetObject(ResponseObject.Save()); BinaryWriter MemStream; - Package.Save(MemStream); + BatchResponse.Save(MemStream); Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, -- cgit v1.2.3 From ca91bd3945fa9e93b48cad72a1bca910badfd940 Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Thu, 11 Nov 2021 15:08:02 +0100 Subject: Format fix. --- zenserver/cache/structuredcachestore.cpp | 15 ++++++++++----- zenserver/cache/structuredcachestore.h | 6 +++--- 2 files changed, 13 insertions(+), 8 deletions(-) (limited to 'zenserver') diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index 119062833..8b9ce8ff9 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -325,27 +325,32 @@ inline DiskLocation::DiskLocation(uint64_t Offset, uint64_t ValueSize, uint32_t { } -inline uint64_t DiskLocation::CombineOffsetAndFlags(uint64_t Offset, uint64_t Flags) +inline uint64_t +DiskLocation::CombineOffsetAndFlags(uint64_t Offset, uint64_t Flags) { return Offset | Flags; } -inline uint64_t DiskLocation::Offset() const +inline uint64_t +DiskLocation::Offset() const { return OffsetAndFlags & kOffsetMask; } -inline uint64_t DiskLocation::Size() const +inline uint64_t +DiskLocation::Size() const { return LowerSize; } -inline uint64_t DiskLocation::IsFlagSet(uint64_t Flag) const +inline uint64_t +DiskLocation::IsFlagSet(uint64_t Flag) const { return OffsetAndFlags & Flag; } -inline ZenContentType DiskLocation::GetContentType() const +inline ZenContentType +DiskLocation::GetContentType() const { ZenContentType ContentType = ZenContentType::kBinary; diff --git a/zenserver/cache/structuredcachestore.h b/zenserver/cache/structuredcachestore.h index 6521d8393..0dfcbc5ca 100644 --- a/zenserver/cache/structuredcachestore.h +++ b/zenserver/cache/structuredcachestore.h @@ -112,9 +112,9 @@ struct DiskLocation static const uint64_t kStructured = 0x4000'0000'0000'0000ull; static const uint64_t kTombStone = 0x2000'0000'0000'0000ull; - DiskLocation(); - DiskLocation(uint64_t Offset, uint64_t ValueSize, uint32_t IndexSize, uint64_t Flags); - static uint64_t CombineOffsetAndFlags(uint64_t Offset, uint64_t Flags); + DiskLocation(); + DiskLocation(uint64_t Offset, uint64_t ValueSize, uint32_t IndexSize, uint64_t Flags); + static uint64_t CombineOffsetAndFlags(uint64_t Offset, uint64_t Flags); uint64_t Offset() const; uint64_t Size() const; uint64_t IsFlagSet(uint64_t Flag) const; -- cgit v1.2.3 From 6c6b615e82444fbdfb4a2b8cc2ed173a1cf772b5 Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Thu, 11 Nov 2021 17:12:14 +0100 Subject: Changed from batch to RPC. --- zenserver/cache/structuredcache.cpp | 14 +++++++------- zenserver/cache/structuredcache.h | 6 +++--- zenserver/upstream/zen.cpp | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) (limited to 'zenserver') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 10fbb3709..e78e583aa 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -112,9 +112,9 @@ HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request) { std::string_view Key = Request.RelativeUri(); - if (Key == "$batch") + if (Key == "$rpc") { - return HandleBatchRequest(Request); + return HandleRpcRequest(Request); } if (std::all_of(begin(Key), end(Key), [](const char c) { return std::isalnum(c); })) @@ -779,7 +779,7 @@ HttpStructuredCacheService::ValidateKeyUri(HttpServerRequest& Request, CacheRef& } void -HttpStructuredCacheService::HandleBatchRequest(zen::HttpServerRequest& Request) +HttpStructuredCacheService::HandleRpcRequest(zen::HttpServerRequest& Request) { switch (auto Verb = Request.RequestVerb()) { @@ -800,11 +800,11 @@ HttpStructuredCacheService::HandleBatchRequest(zen::HttpServerRequest& Request) const std::string_view Method = BatchRequest["Method"sv].AsString(); if (Method == "GetCacheRecords"sv) { - HandleBatchGetCacheRecords(AsyncRequest, BatchRequest); + HandleRpcGetCacheRecords(AsyncRequest, BatchRequest); } else if (Method == "GetCachePayloads"sv) { - HandleBatchGetCachePayloads(AsyncRequest, BatchRequest); + HandleRpcGetCachePayloads(AsyncRequest, BatchRequest); } else { @@ -820,7 +820,7 @@ HttpStructuredCacheService::HandleBatchRequest(zen::HttpServerRequest& Request) } void -HttpStructuredCacheService::HandleBatchGetCacheRecords(zen::HttpServerRequest& Request, CbObjectView BatchRequest) +HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Request, CbObjectView BatchRequest) { using namespace fmt::literals; @@ -959,7 +959,7 @@ HttpStructuredCacheService::HandleBatchGetCacheRecords(zen::HttpServerRequest& R } void -HttpStructuredCacheService::HandleBatchGetCachePayloads(zen::HttpServerRequest& Request, CbObjectView BatchRequest) +HttpStructuredCacheService::HandleRpcGetCachePayloads(zen::HttpServerRequest& Request, CbObjectView BatchRequest) { using namespace fmt::literals; diff --git a/zenserver/cache/structuredcache.h b/zenserver/cache/structuredcache.h index 9efcc05fa..3d9914606 100644 --- a/zenserver/cache/structuredcache.h +++ b/zenserver/cache/structuredcache.h @@ -89,9 +89,9 @@ private: void HandleCachePayloadRequest(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy Policy); void HandleGetCachePayload(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy Policy); void HandlePutCachePayload(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy Policy); - void HandleBatchRequest(zen::HttpServerRequest& Request); - void HandleBatchGetCacheRecords(zen::HttpServerRequest& Request, CbObjectView BatchRequest); - void HandleBatchGetCachePayloads(zen::HttpServerRequest& Request, CbObjectView BatchRequest); + void HandleRpcRequest(zen::HttpServerRequest& Request); + void HandleRpcGetCacheRecords(zen::HttpServerRequest& Request, CbObjectView BatchRequest); + void HandleRpcGetCachePayloads(zen::HttpServerRequest& Request, CbObjectView BatchRequest); void HandleCacheBucketRequest(zen::HttpServerRequest& Request, std::string_view Bucket); virtual void HandleStatsRequest(zen::HttpServerRequest& Request) override; virtual void HandleStatusRequest(zen::HttpServerRequest& Request) override; diff --git a/zenserver/upstream/zen.cpp b/zenserver/upstream/zen.cpp index dc23b573b..9ba767098 100644 --- a/zenserver/upstream/zen.cpp +++ b/zenserver/upstream/zen.cpp @@ -503,7 +503,7 @@ ZenCacheResult ZenStructuredCacheSession::InvokeRpc(const CbObjectView& Request) { ExtendableStringBuilder<256> Uri; - Uri << m_Client.ServiceUrl() << "/z$/$batch"; + Uri << m_Client.ServiceUrl() << "/z$/$rpc"; BinaryWriter Body; Request.CopyTo(Body); -- cgit v1.2.3 From 5ad6c346f71054288f3022d93cd123a646a75873 Mon Sep 17 00:00:00 2001 From: Joe Kirchoff Date: Thu, 11 Nov 2021 14:18:47 -0800 Subject: Horde remote execute (#25) --- zenserver/upstream/jupiter.cpp | 215 +++-- zenserver/upstream/jupiter.h | 18 + zenserver/upstream/upstreamapply.cpp | 1491 ++++++++++++++++++++++++++++++++++ zenserver/upstream/upstreamapply.h | 173 ++++ zenserver/zenserver.vcxproj | 2 + zenserver/zenserver.vcxproj.filters | 6 + 6 files changed, 1841 insertions(+), 64 deletions(-) create mode 100644 zenserver/upstream/upstreamapply.cpp create mode 100644 zenserver/upstream/upstreamapply.h (limited to 'zenserver') diff --git a/zenserver/upstream/jupiter.cpp b/zenserver/upstream/jupiter.cpp index 7a36b5841..4caa5c8df 100644 --- a/zenserver/upstream/jupiter.cpp +++ b/zenserver/upstream/jupiter.cpp @@ -182,9 +182,14 @@ CloudCacheSession::GetBlob(const IoHash& Key) { return {.ErrorCode = static_cast(Response.error.code), .Reason = Response.error.message}; } + else if (!VerifyAccessToken(Response.status_code)) + { + return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; + } const bool Success = Response.status_code == 200; - const IoBuffer Buffer = Success ? IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()) : IoBuffer(); + const IoBuffer Buffer = + Success && Response.text.size() > 0 ? IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()) : IoBuffer(); return {.Response = Buffer, .Bytes = Response.downloaded_bytes, .ElapsedSeconds = Response.elapsed, .Success = Success}; } @@ -214,6 +219,10 @@ CloudCacheSession::GetCompressedBlob(const IoHash& Key) { return {.ErrorCode = static_cast(Response.error.code), .Reason = Response.error.message}; } + else if (!VerifyAccessToken(Response.status_code)) + { + return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; + } const bool Success = Response.status_code == 200; const IoBuffer Buffer = Success ? IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()) : IoBuffer(); @@ -246,6 +255,10 @@ CloudCacheSession::GetObject(const IoHash& Key) { return {.ErrorCode = static_cast(Response.error.code), .Reason = Response.error.message}; } + else if (!VerifyAccessToken(Response.status_code)) + { + return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; + } const bool Success = Response.status_code == 200; const IoBuffer Buffer = Success ? IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()) : IoBuffer(); @@ -571,67 +584,41 @@ CloudCacheSession::RefExists(std::string_view BucketId, const IoHash& Key) CloudCacheResult CloudCacheSession::BlobExists(const IoHash& Key) { - const CloudCacheAccessToken& AccessToken = GetAccessToken(); - if (!AccessToken.IsValid()) - { - return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; - } - - ExtendableStringBuilder<256> Uri; - Uri << m_CacheClient->ServiceUrl() << "/api/v1/blobs/" << m_CacheClient->BlobStoreNamespace() << "/" << Key.ToHexString(); - - cpr::Session& Session = m_SessionState->Session; - - Session.SetOption(cpr::Url{Uri.c_str()}); - - cpr::Response Response = Session.Head(); - ZEN_DEBUG("HEAD {}", Response); - - if (Response.error) - { - return {.ErrorCode = static_cast(Response.error.code), .Reason = Response.error.message}; - } - else if (!VerifyAccessToken(Response.status_code)) - { - return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; - } - - return {.ElapsedSeconds = Response.elapsed, .Success = Response.status_code == 200}; + return CacheTypeExists("blobs"sv, Key); } CloudCacheResult CloudCacheSession::CompressedBlobExists(const IoHash& Key) { - const CloudCacheAccessToken& AccessToken = GetAccessToken(); - if (!AccessToken.IsValid()) - { - return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; - } - - ExtendableStringBuilder<256> Uri; - Uri << m_CacheClient->ServiceUrl() << "/api/v1/compressed-blobs/" << m_CacheClient->BlobStoreNamespace() << "/" << Key.ToHexString(); - - cpr::Session& Session = m_SessionState->Session; + return CacheTypeExists("compressed-blobs"sv, Key); +} - Session.SetOption(cpr::Url{Uri.c_str()}); +CloudCacheResult +CloudCacheSession::ObjectExists(const IoHash& Key) +{ + return CacheTypeExists("objects"sv, Key); +} - cpr::Response Response = Session.Head(); - ZEN_DEBUG("HEAD {}", Response); +CloudCacheExistsResult +CloudCacheSession::BlobExists(const std::set& Keys) +{ + return CacheTypeExists("blobs"sv, Keys); +} - if (Response.error) - { - return {.ErrorCode = static_cast(Response.error.code), .Reason = Response.error.message}; - } - else if (!VerifyAccessToken(Response.status_code)) - { - return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; - } +CloudCacheExistsResult +CloudCacheSession::CompressedBlobExists(const std::set& Keys) +{ + return CacheTypeExists("compressed-blobs"sv, Keys); +} - return {.ElapsedSeconds = Response.elapsed, .Success = Response.status_code == 200}; +CloudCacheExistsResult +CloudCacheSession::ObjectExists(const std::set& Keys) +{ + return CacheTypeExists("objects"sv, Keys); } CloudCacheResult -CloudCacheSession::ObjectExists(const IoHash& Key) +CloudCacheSession::PostComputeTasks(std::string_view ChannelId, IoBuffer TasksData) { const CloudCacheAccessToken& AccessToken = GetAccessToken(); if (!AccessToken.IsValid()) @@ -640,14 +627,16 @@ CloudCacheSession::ObjectExists(const IoHash& Key) } ExtendableStringBuilder<256> Uri; - Uri << m_CacheClient->ServiceUrl() << "/api/v1/objects/" << m_CacheClient->BlobStoreNamespace() << "/" << Key.ToHexString(); + Uri << m_CacheClient->ServiceUrl() << "/api/v1/compute/" << ChannelId; - cpr::Session& Session = m_SessionState->Session; + auto& Session = m_SessionState->Session; Session.SetOption(cpr::Url{Uri.c_str()}); + Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}, {"Content-Type", "application/x-ue-cb"}}); + Session.SetBody(cpr::Body{(const char*)TasksData.Data(), TasksData.Size()}); - cpr::Response Response = Session.Head(); - ZEN_DEBUG("HEAD {}", Response); + cpr::Response Response = Session.Post(); + ZEN_DEBUG("POST {}", Response); if (Response.error) { @@ -662,7 +651,7 @@ CloudCacheSession::ObjectExists(const IoHash& Key) } CloudCacheResult -CloudCacheSession::PostComputeTasks(std::string_view ChannelId, IoBuffer TasksData) +CloudCacheSession::GetComputeUpdates(std::string_view ChannelId, const uint32_t WaitSeconds) { const CloudCacheAccessToken& AccessToken = GetAccessToken(); if (!AccessToken.IsValid()) @@ -671,13 +660,12 @@ CloudCacheSession::PostComputeTasks(std::string_view ChannelId, IoBuffer TasksDa } ExtendableStringBuilder<256> Uri; - Uri << m_CacheClient->ServiceUrl() << "/api/v1/compute/" << ChannelId; + Uri << m_CacheClient->ServiceUrl() << "/api/v1/compute/" << ChannelId << "/updates?wait=" << WaitSeconds; auto& Session = m_SessionState->Session; Session.SetOption(cpr::Url{Uri.c_str()}); - Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}, {"Content-Type", "application/x-ue-cb"}}); - Session.SetBody(cpr::Body{(const char*)TasksData.Data(), TasksData.Size()}); + Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}, {"Accept", "application/x-ue-cb"}}); cpr::Response Response = Session.Post(); ZEN_DEBUG("POST {}", Response); @@ -698,7 +686,7 @@ CloudCacheSession::PostComputeTasks(std::string_view ChannelId, IoBuffer TasksDa } CloudCacheResult -CloudCacheSession::GetComputeUpdates(std::string_view ChannelId, const uint32_t WaitSeconds) +CloudCacheSession::GetObjectTree(const IoHash& Key) { const CloudCacheAccessToken& AccessToken = GetAccessToken(); if (!AccessToken.IsValid()) @@ -707,15 +695,15 @@ CloudCacheSession::GetComputeUpdates(std::string_view ChannelId, const uint32_t } ExtendableStringBuilder<256> Uri; - Uri << m_CacheClient->ServiceUrl() << "/api/v1/compute/" << ChannelId << "/updates?wait=" << WaitSeconds; + Uri << m_CacheClient->ServiceUrl() << "/api/v1/objects/" << m_CacheClient->BlobStoreNamespace() << "/" << Key.ToHexString() << "/tree"; - auto& Session = m_SessionState->Session; + cpr::Session& Session = m_SessionState->Session; Session.SetOption(cpr::Url{Uri.c_str()}); - Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}, {"Content-Type", "application/x-ue-cb"}}); + Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}, {"Accept", "application/octet-stream"}}); - cpr::Response Response = Session.Post(); - ZEN_DEBUG("POST {}", Response); + cpr::Response Response = Session.Get(); + ZEN_DEBUG("GET {}", Response); if (Response.error) { @@ -761,6 +749,92 @@ CloudCacheSession::VerifyAccessToken(long StatusCode) return true; } +CloudCacheResult +CloudCacheSession::CacheTypeExists(std::string_view TypeId, const IoHash& Key) +{ + const CloudCacheAccessToken& AccessToken = GetAccessToken(); + if (!AccessToken.IsValid()) + { + return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; + } + + ExtendableStringBuilder<256> Uri; + Uri << m_CacheClient->ServiceUrl() << "/api/v1/" << TypeId << "/" << m_CacheClient->BlobStoreNamespace() << "/" << Key.ToHexString(); + + cpr::Session& Session = m_SessionState->Session; + + Session.SetOption(cpr::Url{Uri.c_str()}); + Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}}); + + cpr::Response Response = Session.Head(); + ZEN_DEBUG("HEAD {}", Response); + + if (Response.error) + { + return {.ErrorCode = static_cast(Response.error.code), .Reason = Response.error.message}; + } + else if (!VerifyAccessToken(Response.status_code)) + { + return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; + } + + return {.ElapsedSeconds = Response.elapsed, .Success = Response.status_code == 200}; +} + +CloudCacheExistsResult +CloudCacheSession::CacheTypeExists(std::string_view TypeId, const std::set& Keys) +{ + const CloudCacheAccessToken& AccessToken = GetAccessToken(); + if (!AccessToken.IsValid()) + { + return {CloudCacheResult{.ErrorCode = 401, .Reason = std::string("Invalid access token")}}; + } + + ExtendableStringBuilder<256> Query; + for (const auto& Key : Keys) + { + Query << (Query.Size() != 0 ? "&id=" : "id=") << Key.ToHexString(); + } + + ExtendableStringBuilder<256> Uri; + Uri << m_CacheClient->ServiceUrl() << "/api/v1/" << TypeId << "/" << m_CacheClient->BlobStoreNamespace() << "/exists?" << Query; + + cpr::Session& Session = m_SessionState->Session; + + Session.SetOption(cpr::Url{Uri.c_str()}); + Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}, {"Accept", "application/x-ue-cb"}}); + + cpr::Response Response = Session.Post(); + ZEN_DEBUG("POST {}", Response); + + if (Response.error) + { + return {CloudCacheResult{.ErrorCode = static_cast(Response.error.code), .Reason = Response.error.message}}; + } + else if (!VerifyAccessToken(Response.status_code)) + { + return {CloudCacheResult{.ErrorCode = 401, .Reason = std::string("Invalid access token")}}; + } + + CloudCacheExistsResult Result{ + CloudCacheResult{.Bytes = Response.downloaded_bytes, .ElapsedSeconds = Response.elapsed, .Success = Response.status_code == 200}}; + + if (Result.Success) + { + IoBuffer Buffer = IoBuffer(zen::IoBuffer::Wrap, Response.text.data(), Response.text.size()); + const CbObject ExistsResponse = LoadCompactBinaryObject(Buffer); + for (auto& Item : ExistsResponse["id"sv]) + { + if (Item.IsHash()) + { + Result.Have.insert(Item.AsHash()); + } + } + } + + return Result; +} + ////////////////////////////////////////////////////////////////////////// // // ServiceUrl: https://jupiter.devtools.epicgames.com @@ -778,7 +852,14 @@ CloudCacheClient::CloudCacheClient(const CloudCacheClientOptions& Options) , m_BlobStoreNamespace(Options.BlobStoreNamespace) , m_OAuthClientId(Options.OAuthClientId) , m_OAuthSecret(Options.OAuthSecret) +, m_AccessToken(Options.AccessToken) { + if (!Options.AccessToken.empty()) + { + // If an access token was provided, OAuth settings are not used. + return; + } + if (!Options.OAuthProvider.starts_with("http://"sv) && !Options.OAuthProvider.starts_with("https://"sv)) { ZEN_WARN("bad provider specification: '{}' - must be fully qualified", Options.OAuthProvider); @@ -828,6 +909,12 @@ CloudCacheClient::AcquireAccessToken() { using namespace std::chrono; + // If an access token was provided, return it instead of querying OAuth + if (!m_AccessToken.empty()) + { + return {m_AccessToken, steady_clock::time_point::max()}; + } + ExtendableStringBuilder<128> OAuthFormData; OAuthFormData << "client_id=" << m_OAuthClientId << "&scope=cache_access&grant_type=client_credentials&client_secret=" << m_OAuthSecret; diff --git a/zenserver/upstream/jupiter.h b/zenserver/upstream/jupiter.h index 9471ef64f..13d65587e 100644 --- a/zenserver/upstream/jupiter.h +++ b/zenserver/upstream/jupiter.h @@ -12,6 +12,7 @@ #include #include #include +#include #include struct ZenCacheValue; @@ -64,6 +65,11 @@ struct FinalizeRefResult : CloudCacheResult std::vector Needs; }; +struct CloudCacheExistsResult : CloudCacheResult +{ + std::set Have; +}; + /** * Context for performing Jupiter operations * @@ -95,12 +101,18 @@ public: FinalizeRefResult FinalizeRef(std::string_view BucketId, const IoHash& Key, const IoHash& RefHah); CloudCacheResult RefExists(std::string_view BucketId, const IoHash& Key); + CloudCacheResult BlobExists(const IoHash& Key); CloudCacheResult CompressedBlobExists(const IoHash& Key); CloudCacheResult ObjectExists(const IoHash& Key); + CloudCacheExistsResult BlobExists(const std::set& Keys); + CloudCacheExistsResult CompressedBlobExists(const std::set& Keys); + CloudCacheExistsResult ObjectExists(const std::set& Keys); + CloudCacheResult PostComputeTasks(std::string_view ChannelId, IoBuffer TasksData); CloudCacheResult GetComputeUpdates(std::string_view ChannelId, const uint32_t WaitSeconds = 0); + CloudCacheResult GetObjectTree(const IoHash& Key); std::vector Filter(std::string_view BucketId, const std::vector& ChunkHashes); @@ -109,6 +121,10 @@ private: const CloudCacheAccessToken& GetAccessToken(); bool VerifyAccessToken(long StatusCode); + CloudCacheResult CacheTypeExists(std::string_view TypeId, const IoHash& Key); + + CloudCacheExistsResult CacheTypeExists(std::string_view TypeId, const std::set& Keys); + spdlog::logger& m_Log; RefPtr m_CacheClient; detail::CloudCacheSessionState* m_SessionState; @@ -122,6 +138,7 @@ struct CloudCacheClientOptions std::string_view OAuthProvider; std::string_view OAuthClientId; std::string_view OAuthSecret; + std::string_view AccessToken; bool UseLegacyDdc = false; }; @@ -152,6 +169,7 @@ private: std::string m_BlobStoreNamespace; std::string m_OAuthClientId; std::string m_OAuthSecret; + std::string m_AccessToken; bool m_IsValid = false; RwLock m_SessionStateLock; diff --git a/zenserver/upstream/upstreamapply.cpp b/zenserver/upstream/upstreamapply.cpp new file mode 100644 index 000000000..fd24d241c --- /dev/null +++ b/zenserver/upstream/upstreamapply.cpp @@ -0,0 +1,1491 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "upstreamapply.h" +#include "jupiter.h" +#include "zen.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "cache/structuredcachestore.h" +#include "diag/logging.h" + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace zen { + +using namespace std::literals; + +namespace detail { + + class HordeUpstreamApplyEndpoint final : public UpstreamApplyEndpoint + { + public: + HordeUpstreamApplyEndpoint(const CloudCacheClientOptions& Options, CasStore& CasStore, CidStore& CidStore) + : m_Log(logging::Get("upstream-apply")) + , m_CasStore(CasStore) + , m_CidStore(CidStore) + { + using namespace fmt::literals; + m_DisplayName = "Horde - '{}'"_format(Options.ServiceUrl); + m_Client = new CloudCacheClient(Options); + m_ChannelId = "zen-{}"_format(zen::GetSessionIdString()); + } + + virtual ~HordeUpstreamApplyEndpoint() = default; + + virtual UpstreamEndpointHealth Initialize() override { return CheckHealth(); } + + virtual bool IsHealthy() const override { return m_HealthOk.load(); } + + virtual UpstreamEndpointHealth CheckHealth() override + { + try + { + CloudCacheSession Session(m_Client); + CloudCacheResult Result = Session.Authenticate(); + + m_HealthOk = Result.ErrorCode == 0; + + return {.Reason = std::move(Result.Reason), .Ok = Result.Success}; + } + catch (std::exception& Err) + { + return {.Reason = Err.what(), .Ok = false}; + } + } + + virtual std::string_view DisplayName() const override { return m_DisplayName; } + + virtual PostUpstreamApplyResult PostApply(const UpstreamApplyRecord& ApplyRecord) override + { + int64_t Bytes{}; + double ElapsedSeconds{}; + + try + { + UpstreamData UpstreamData; + if (!ProcessApplyKey(ApplyRecord, UpstreamData)) + { + return {.Error{.ErrorCode = -1, .Reason = "Failed to generate task data"}}; + } + + { + std::scoped_lock Lock(m_TaskMutex); + if (m_PendingTasks.contains(UpstreamData.TaskId)) + { + // Pending task is already queued, return success + return {.Bytes = Bytes, .ElapsedSeconds = ElapsedSeconds, .Success = true}; + } + m_PendingTasks[UpstreamData.TaskId] = ApplyRecord; + } + + CloudCacheSession Session(m_Client); + + { + CloudCacheResult Result = BatchPutBlobsIfMissing(Session, UpstreamData.Blobs); + Bytes += Result.Bytes; + ElapsedSeconds += Result.ElapsedSeconds; + if (!Result.Success) + { + return {.Error{.ErrorCode = Result.ErrorCode, .Reason = std::move(Result.Reason)}, + .Bytes = Bytes, + .ElapsedSeconds = ElapsedSeconds}; + } + UpstreamData.Blobs.clear(); + } + + { + CloudCacheResult Result = BatchPutObjectsIfMissing(Session, UpstreamData.Objects); + Bytes += Result.Bytes; + ElapsedSeconds += Result.ElapsedSeconds; + if (!Result.Success) + { + return {.Error{.ErrorCode = Result.ErrorCode, .Reason = std::move(Result.Reason)}, + .Bytes = Bytes, + .ElapsedSeconds = ElapsedSeconds}; + } + UpstreamData.Objects.clear(); + } + + CbObjectWriter Writer; + Writer.AddObjectAttachment("r"sv, UpstreamData.RequirementsId); + Writer.BeginArray("t"sv); + Writer.AddObjectAttachment(UpstreamData.TaskId); + Writer.EndArray(); + IoBuffer TasksData = Writer.Save().GetBuffer().AsIoBuffer(); + + CloudCacheResult Result = Session.PostComputeTasks(m_ChannelId, TasksData); + Bytes += Result.Bytes; + ElapsedSeconds += Result.ElapsedSeconds; + if (!Result.Success) + { + { + std::scoped_lock Lock(m_TaskMutex); + m_PendingTasks.erase(UpstreamData.TaskId); + } + + return {.Error{.ErrorCode = Result.ErrorCode, .Reason = std::move(Result.Reason)}, + .Bytes = Bytes, + .ElapsedSeconds = ElapsedSeconds}; + } + + return {.Bytes = Bytes, .ElapsedSeconds = ElapsedSeconds, .Success = true}; + } + catch (std::exception& Err) + { + m_HealthOk = false; + return {.Error{.ErrorCode = -1, .Reason = Err.what()}, .Bytes = Bytes, .ElapsedSeconds = ElapsedSeconds}; + } + } + + [[nodiscard]] CloudCacheResult BatchPutBlobsIfMissing(CloudCacheSession& Session, const std::map& Blobs) + { + if (Blobs.size() == 0) + { + return {.Success = true}; + } + + int64_t Bytes{}; + double ElapsedSeconds{}; + + // Batch check for missing blobs + std::set Keys; + for (const auto& It : Blobs) + { + Keys.insert(It.first); + } + + CloudCacheExistsResult ExistsResult = Session.BlobExists(Keys); + ElapsedSeconds += ExistsResult.ElapsedSeconds; + if (ExistsResult.ErrorCode != 0) + { + return {.Bytes = Bytes, + .ElapsedSeconds = ElapsedSeconds, + .ErrorCode = ExistsResult.ErrorCode, + .Reason = std::move(ExistsResult.Reason)}; + } + + // TODO: Batch upload missing blobs + + for (const auto& It : Blobs) + { + if (ExistsResult.Have.contains(It.first)) + { + continue; + } + + CloudCacheResult Result = Session.PutBlob(It.first, It.second); + Bytes += Result.Bytes; + ElapsedSeconds += Result.ElapsedSeconds; + if (!Result.Success) + { + return {.Bytes = Bytes, + .ElapsedSeconds = ElapsedSeconds, + .ErrorCode = Result.ErrorCode, + .Reason = std::move(Result.Reason)}; + } + } + + return {.Bytes = Bytes, .ElapsedSeconds = ElapsedSeconds, .Success = true}; + } + + [[nodiscard]] CloudCacheResult BatchPutObjectsIfMissing(CloudCacheSession& Session, const std::map& Objects) + { + if (Objects.size() == 0) + { + return {.Success = true}; + } + + int64_t Bytes{}; + double ElapsedSeconds{}; + + // Batch check for missing objects + std::set Keys; + for (const auto& It : Objects) + { + Keys.insert(It.first); + } + + CloudCacheExistsResult ExistsResult = Session.ObjectExists(Keys); + ElapsedSeconds += ExistsResult.ElapsedSeconds; + if (ExistsResult.ErrorCode != 0) + { + return {.Bytes = Bytes, + .ElapsedSeconds = ElapsedSeconds, + .ErrorCode = ExistsResult.ErrorCode, + .Reason = std::move(ExistsResult.Reason)}; + } + + // TODO: Batch upload missing objects + + for (const auto& It : Objects) + { + if (ExistsResult.Have.contains(It.first)) + { + continue; + } + + CloudCacheResult Result = Session.PutObject(It.first, It.second.GetBuffer().AsIoBuffer()); + Bytes += Result.Bytes; + ElapsedSeconds += Result.ElapsedSeconds; + if (!Result.Success) + { + return {.Bytes = Bytes, + .ElapsedSeconds = ElapsedSeconds, + .ErrorCode = Result.ErrorCode, + .Reason = std::move(Result.Reason)}; + } + } + + return {.Bytes = Bytes, .ElapsedSeconds = ElapsedSeconds, .Success = true}; + } + + enum class ComputeTaskState : int32_t + { + Queued = 0, + Executing = 1, + Complete = 2, + }; + + enum class ComputeTaskOutcome : int32_t + { + Success = 0, + Failed = 1, + Cancelled = 2, + NoResult = 3, + Exipred = 4, + BlobNotFound = 5, + Exception = 6, + }; + + virtual GetUpstreamApplyUpdatesResult GetUpdates() override + { + int64_t Bytes{}; + double ElapsedSeconds{}; + UpstreamApplyCompleted CompletedTasks; + + { + std::scoped_lock Lock(m_TaskMutex); + if (m_PendingTasks.empty()) + { + // Nothing to do. + return {.Success = true}; + } + } + + try + { + CloudCacheSession Session(m_Client); + + CloudCacheResult UpdatesResult = Session.GetComputeUpdates(m_ChannelId); + Bytes += UpdatesResult.Bytes; + ElapsedSeconds += UpdatesResult.ElapsedSeconds; + if (UpdatesResult.ErrorCode != 0) + { + return {.Error{.ErrorCode = UpdatesResult.ErrorCode, .Reason = std::move(UpdatesResult.Reason)}, + .Bytes = Bytes, + .ElapsedSeconds = ElapsedSeconds}; + } + + if (!UpdatesResult.Success) + { + return {.Error{.ErrorCode = -1, .Reason = "Failed get task updates"}, .Bytes = Bytes, .ElapsedSeconds = ElapsedSeconds}; + } + + CbObject TaskStatus = LoadCompactBinaryObject(UpdatesResult.Response); + + // zen::StringBuilder<4096> ObjStr; + // zen::CompactBinaryToJson(TaskStatus, ObjStr); + + for (auto& It : TaskStatus["u"sv]) + { + CbObjectView Status = It.AsObjectView(); + const ComputeTaskState State = (ComputeTaskState)Status["s"sv].AsInt32(); + + const std::string_view AgentId = TaskStatus["a"sv].AsString(); + const std::string_view LeaseId = TaskStatus["l"sv].AsString(); + + // Only care about completed tasks + if (State != ComputeTaskState::Complete) + { + continue; + } + + const IoHash TaskId = Status["h"sv].AsObjectAttachment(); + + IoHash WorkerId; + IoHash ActionId; + + { + std::scoped_lock Lock(m_TaskMutex); + auto TaskIt = m_PendingTasks.find(TaskId); + if (TaskIt == m_PendingTasks.end()) + { + continue; + } + WorkerId = TaskIt->second.WorkerDescriptor.GetHash(); + ActionId = TaskIt->second.Action.GetHash(); + m_PendingTasks.erase(TaskIt); + } + + GetUpstreamApplyResult Result = ProcessTaskStatus(Status, Session); + Bytes += Result.Bytes; + ElapsedSeconds += Result.ElapsedSeconds; + + CompletedTasks[WorkerId][ActionId] = std::move(Result); + } + + return {.Bytes = Bytes, .ElapsedSeconds = ElapsedSeconds, .Completed = std::move(CompletedTasks), .Success = true}; + } + catch (std::exception& Err) + { + m_HealthOk = false; + return { + .Error{.ErrorCode = -1, .Reason = Err.what()}, + .Bytes = Bytes, + .ElapsedSeconds = ElapsedSeconds, + .Completed = std::move(CompletedTasks), + }; + } + } + + virtual UpstreamApplyEndpointStats& Stats() override { return m_Stats; } + + private: + spdlog::logger& Log() { return m_Log; } + + CasStore& m_CasStore; + CidStore& m_CidStore; + spdlog::logger& m_Log; + std::string m_DisplayName; + RefPtr m_Client; + UpstreamApplyEndpointStats m_Stats; + std::atomic_bool m_HealthOk{false}; + std::string m_ChannelId; + + std::mutex m_TaskMutex; + std::unordered_map m_PendingTasks; + + struct UpstreamData + { + std::map Blobs; + std::map Objects; + IoHash TaskId; + IoHash RequirementsId; + }; + + struct UpstreamDirectory + { + std::filesystem::path Path; + std::map Directories; + std::set Files; + }; + + [[nodiscard]] GetUpstreamApplyResult ProcessTaskStatus(const CbObjectView& TaskStatus, CloudCacheSession& Session) + { + try + { + const ComputeTaskOutcome Outcome = (ComputeTaskOutcome)TaskStatus["o"sv].AsInt32(); + + if (Outcome != ComputeTaskOutcome::Success) + { + const std::string_view Detail = TaskStatus["d"sv].AsString(); + return {.Error{.ErrorCode = -1, .Reason = std::string(Detail)}}; + } + + const IoHash TaskId = TaskStatus["h"sv].AsObjectAttachment(); + const DateTime Time = TaskStatus["t"sv].AsDateTime(); + const IoHash ResultHash = TaskStatus["r"sv].AsObjectAttachment(); + const std::string_view AgentId = TaskStatus["a"sv].AsString(); + const std::string_view LeaseId = TaskStatus["l"sv].AsString(); + + int64_t Bytes{}; + double ElapsedSeconds{}; + + // Get Result object and all Object Attachments + Binary Attachment IDs + CloudCacheResult ObjectTreeResult = Session.GetObjectTree(ResultHash); + Bytes += ObjectTreeResult.Bytes; + ElapsedSeconds += ObjectTreeResult.ElapsedSeconds; + + if (ObjectTreeResult.ErrorCode != 0) + { + return {.Error{.ErrorCode = ObjectTreeResult.ErrorCode, .Reason = std::move(ObjectTreeResult.Reason)}, + .Bytes = Bytes, + .ElapsedSeconds = ElapsedSeconds}; + } + + if (!ObjectTreeResult.Success) + { + return {.Error{.ErrorCode = -1, .Reason = "Failed to get result object data"}, + .Bytes = Bytes, + .ElapsedSeconds = ElapsedSeconds}; + } + + std::map TreeObjectData; + std::map TreeBinaryData; + + MemoryView ResponseView = ObjectTreeResult.Response; + while (ResponseView.GetSize() > 0) + { + CbFieldView Field = CbFieldView(ResponseView.GetData()); + ResponseView += Field.GetSize(); + if (Field.IsObjectAttachment()) + { + const IoHash Hash = Field.AsObjectAttachment(); + Field = CbFieldView(ResponseView.GetData()); + ResponseView += Field.GetSize(); + if (!Field.IsObject()) // No data + { + TreeObjectData[Hash] = {}; + continue; + } + MemoryView FieldView = Field.AsObjectView().GetView(); + + TreeObjectData[Hash] = IoBuffer(IoBuffer::Wrap, FieldView.GetData(), FieldView.GetSize()); + } + else if (Field.IsBinaryAttachment()) + { + const IoHash Hash = Field.AsBinaryAttachment(); + TreeBinaryData[Hash] = {}; + } + else // Unknown type + { + } + } + + for (auto& It : TreeObjectData) + { + if (It.second.GetSize() == 0) + { + CloudCacheResult ObjectResult = Session.GetObject(It.first); + Bytes += ObjectTreeResult.Bytes; + ElapsedSeconds += ObjectTreeResult.ElapsedSeconds; + if (ObjectTreeResult.ErrorCode != 0) + { + return {.Error{.ErrorCode = ObjectResult.ErrorCode, .Reason = std::move(ObjectResult.Reason)}, + .Bytes = Bytes, + .ElapsedSeconds = ElapsedSeconds}; + } + + if (!ObjectResult.Success) + { + return {.Error{.ErrorCode = -1, .Reason = "Failed to get result object attachment data"}, + .Bytes = Bytes, + .ElapsedSeconds = ElapsedSeconds}; + } + It.second = std::move(ObjectResult.Response); + } + } + + for (auto& It : TreeBinaryData) + { + if (It.second.GetSize() == 0) + { + CloudCacheResult BlobResult = Session.GetBlob(It.first); + Bytes += ObjectTreeResult.Bytes; + ElapsedSeconds += ObjectTreeResult.ElapsedSeconds; + if (BlobResult.ErrorCode != 0) + { + return {.Error{.ErrorCode = BlobResult.ErrorCode, .Reason = std::move(BlobResult.Reason)}, + .Bytes = Bytes, + .ElapsedSeconds = ElapsedSeconds}; + } + + if (!BlobResult.Success) + { + return {.Error{.ErrorCode = -1, .Reason = "Failed to get result binary attachment data"}, + .Bytes = Bytes, + .ElapsedSeconds = ElapsedSeconds}; + } + It.second = std::move(BlobResult.Response); + } + } + + CbObject ResultObject = LoadCompactBinaryObject(TreeObjectData[ResultHash]); + int32_t ExitCode = ResultObject["e"sv].AsInt32(); + IoHash StdOutHash = ResultObject["so"sv].AsBinaryAttachment(); + IoHash StdErrHash = ResultObject["se"sv].AsBinaryAttachment(); + IoHash OutputHash = ResultObject["o"sv].AsObjectAttachment(); + + std::string StdOut = std::string((const char*)TreeBinaryData[StdOutHash].GetData(), TreeBinaryData[StdOutHash].GetSize()); + std::string StdErr = std::string((const char*)TreeBinaryData[StdErrHash].GetData(), TreeBinaryData[StdErrHash].GetSize()); + + if (ExitCode != 0) + { + return {.Error{.ErrorCode = ExitCode, .Reason = "Task completed with errors"}, + .Bytes = Bytes, + .ElapsedSeconds = ElapsedSeconds, + .StdOut = std::move(StdOut), + .StdErr = std::move(StdErr)}; + } + + CbObject OutputObject = LoadCompactBinaryObject(TreeObjectData[OutputHash]); + + // Get build.output + IoHash BuildOutputId; + IoBuffer BuildOutput; + for (auto& It : OutputObject["f"sv]) + { + const CbObjectView FileObject = It.AsObjectView(); + if (FileObject["n"sv].AsString() == "Build.output"sv) + { + BuildOutputId = FileObject["h"sv].AsBinaryAttachment(); + BuildOutput = TreeBinaryData[BuildOutputId]; + break; + } + } + + if (BuildOutput.GetSize() == 0) + { + return {.Error{.ErrorCode = ExitCode, .Reason = "Build.output file not found in task results"}, + .Bytes = Bytes, + .ElapsedSeconds = ElapsedSeconds}; + } + + // Get Output directory node + IoBuffer OutputDirectoryTree; + for (auto& It : OutputObject["d"sv]) + { + const CbObjectView DirectoryObject = It.AsObjectView(); + if (DirectoryObject["n"sv].AsString() == "Outputs"sv) + { + OutputDirectoryTree = TreeObjectData[DirectoryObject["h"sv].AsObjectAttachment()]; + break; + } + } + + if (OutputDirectoryTree.GetSize() == 0) + { + return {.Error{.ErrorCode = ExitCode, .Reason = "Outputs directory not found in task results"}, + .Bytes = Bytes, + .ElapsedSeconds = ElapsedSeconds}; + } + + // load build.output as CbObject + + // Move Outputs from Horde to CbPackage + + std::unordered_map CidToCompressedId; + CbPackage OutputPackage; + CbObject OutputDirectoryTreeObject = LoadCompactBinaryObject(OutputDirectoryTree); + int64_t TotalAttachmentBytes = 0; + int64_t TotalRawAttachmentBytes = 0; + + for (auto& It : OutputDirectoryTreeObject["f"sv]) + { + CbObjectView FileObject = It.AsObjectView(); + // Name is the uncompressed hash + IoHash DecompressedId = IoHash::FromHexString(FileObject["n"sv].AsString()); + // Hash is the compressed data hash, and how it is stored in Horde + IoHash CompressedId = FileObject["h"sv].AsBinaryAttachment(); + + if (!TreeBinaryData.contains(CompressedId)) + { + Log().warn("Object attachment chunk not retrieved from Horde {}", CompressedId.ToHexString()); + return {.Error{.ErrorCode = -1, .Reason = "Object attachment chunk not retrieved from Horde"}, + .Bytes = Bytes, + .ElapsedSeconds = ElapsedSeconds}; + } + CidToCompressedId[DecompressedId] = CompressedId; + } + + // Iterate attachments, verify all chunks exist, and add to CbPackage + bool AnyErrors = false; + CbObject BuildOutputObject = LoadCompactBinaryObject(BuildOutput); + BuildOutputObject.IterateAttachments([&](CbFieldView Field) { + const IoHash DecompressedId = Field.AsHash(); + if (!CidToCompressedId.contains(DecompressedId)) + { + Log().warn("Attachment not found {}", DecompressedId.ToHexString()); + AnyErrors = true; + return; + } + const IoHash& CompressedId = CidToCompressedId.at(DecompressedId); + + if (!TreeBinaryData.contains(CompressedId)) + { + Log().warn("Missing output {} compressed {} uncompressed", + CompressedId.ToHexString(), + DecompressedId.ToHexString()); + AnyErrors = true; + return; + } + + CompressedBuffer AttachmentBuffer = CompressedBuffer::FromCompressed(SharedBuffer(TreeBinaryData[CompressedId])); + + if (!AttachmentBuffer) + { + Log().warn("Invalid output encountered (not valid CompressedBuffer format) {} compressed {} uncompressed", + CompressedId.ToHexString(), + DecompressedId.ToHexString()); + AnyErrors = true; + return; + } + + TotalAttachmentBytes += AttachmentBuffer.GetCompressedSize(); + TotalRawAttachmentBytes += AttachmentBuffer.GetRawSize(); + + CbAttachment Attachment(AttachmentBuffer); + OutputPackage.AddAttachment(Attachment); + }); + + if (AnyErrors) + { + return {.Error{.ErrorCode = -1, .Reason = "Failed to get result object attachment data"}, + .Bytes = Bytes, + .ElapsedSeconds = ElapsedSeconds}; + } + + OutputPackage.SetObject(BuildOutputObject); + + return {.OutputPackage = std::move(OutputPackage), + .TotalAttachmentBytes = TotalAttachmentBytes, + .TotalRawAttachmentBytes = TotalRawAttachmentBytes, + .Bytes = Bytes, + .ElapsedSeconds = ElapsedSeconds, + .StdOut = std::move(StdOut), + .StdErr = std::move(StdErr), + .Success = true}; + } + catch (std::exception& Err) + { + return {.Error{.ErrorCode = -1, .Reason = Err.what()}}; + } + } + + [[nodiscard]] bool ProcessApplyKey(const UpstreamApplyRecord& ApplyRecord, UpstreamData& Data) + { + std::string ExecutablePath; + std::map Environment; + std::set InputFiles; + std::map InputFileHashes; + + ExecutablePath = ApplyRecord.WorkerDescriptor["path"sv].AsString(); + if (ExecutablePath.empty()) + { + Log().warn("process apply upstream FAILED, '{}', path missing from worker descriptor", + ApplyRecord.WorkerDescriptor.GetHash()); + return false; + } + + for (auto& It : ApplyRecord.WorkerDescriptor["executables"sv]) + { + CbObjectView FileEntry = It.AsObjectView(); + if (!ProcessFileEntry(FileEntry, InputFiles, InputFileHashes, Data.Blobs)) + { + return false; + } + } + + for (auto& It : ApplyRecord.WorkerDescriptor["files"sv]) + { + CbObjectView FileEntry = It.AsObjectView(); + if (!ProcessFileEntry(FileEntry, InputFiles, InputFileHashes, Data.Blobs)) + { + return false; + } + } + + for (auto& It : ApplyRecord.WorkerDescriptor["environment"sv]) + { + std::string_view Env = It.AsString(); + auto Index = Env.find('='); + if (Index < 0) + { + Log().warn("process apply upstream FAILED, environment '{}' malformed", Env); + return false; + } + + Environment[std::string(Env.substr(0, Index))] = Env.substr(Index + 1); + } + + { + static const std::filesystem::path BuildActionPath = "Build.action"sv; + static const std::filesystem::path InputPath = "Inputs"sv; + const IoHash ActionId = ApplyRecord.Action.GetHash(); + + InputFiles.insert(BuildActionPath); + InputFileHashes[BuildActionPath] = ActionId; + Data.Blobs[ActionId] = IoBufferBuilder::MakeCloneFromMemory(ApplyRecord.Action.GetBuffer().GetData(), + ApplyRecord.Action.GetBuffer().GetSize()); + + bool AnyErrors = false; + ApplyRecord.Action.IterateAttachments([&](CbFieldView Field) { + const IoHash Cid = Field.AsHash(); + const std::filesystem::path FilePath = {InputPath / Cid.ToHexString()}; + IoBuffer DataBuffer = m_CidStore.FindChunkByCid(Cid); + + if (!DataBuffer) + { + Log().warn("process apply upstream FAILED, input CID chunk '{}' missing", Cid); + AnyErrors = true; + return; + } + + if (InputFiles.contains(FilePath)) + { + return; + } + + const IoHash CompressedId = IoHash::HashBuffer(DataBuffer.GetData(), DataBuffer.GetSize()); + + InputFiles.insert(FilePath); + InputFileHashes[FilePath] = CompressedId; + Data.Blobs[CompressedId] = std::move(DataBuffer); + }); + + if (AnyErrors) + { + return false; + } + } + + const UpstreamDirectory RootDirectory = BuildDirectoryTree(InputFiles); + + CbObject Sandbox = BuildMerkleTreeDirectory(RootDirectory, InputFileHashes, Data.Blobs, Data.Objects); + const IoHash SandboxHash = Sandbox.GetHash(); + Data.Objects[SandboxHash] = std::move(Sandbox); + + CbObject Requirements = BuildRequirements("OSFamily == 'Windows'"sv, {}, false); + const IoHash RequirementsId = Requirements.GetHash(); + Data.Objects[RequirementsId] = std::move(Requirements); + Data.RequirementsId = RequirementsId; + + CbObject Task = BuildTask(ExecutablePath, + {"-Build=build.action"}, + Environment, + {}, + SandboxHash, + RequirementsId, + {"Build.output", "Outputs"}); + + const IoHash TaskId = Task.GetHash(); + Data.Objects[TaskId] = std::move(Task); + Data.TaskId = TaskId; + + return true; + } + + [[nodiscard]] bool ProcessFileEntry(const CbObjectView& FileEntry, + std::set& InputFiles, + std::map& InputFileHashes, + std::map& Blobs) + { + const std::filesystem::path FilePath = FileEntry["name"sv].AsString(); + const IoHash ChunkId = FileEntry["hash"sv].AsHash(); + const uint64_t Size = FileEntry["size"sv].AsUInt64(); + IoBuffer DataBuffer = m_CasStore.FindChunk(ChunkId); + + if (!DataBuffer) + { + Log().warn("process apply upstream FAILED, worker CAS chunk '{}' missing", ChunkId); + return false; + } + + if (DataBuffer.Size() != Size) + { + Log().warn("process apply upstream FAILED, worker CAS chunk '{}' size: {}, action spec expected {}", + ChunkId, + DataBuffer.Size(), + Size); + return false; + } + + if (InputFiles.contains(FilePath)) + { + Log().warn("process apply upstream FAILED, worker CAS chunk '{}' size: {} duplicate filename {}", ChunkId, Size, FilePath); + return false; + } + + InputFiles.insert(FilePath); + InputFileHashes[FilePath] = ChunkId; + Blobs[ChunkId] = std::move(DataBuffer); + return true; + } + + [[nodiscard]] UpstreamDirectory BuildDirectoryTree(const std::set& InputFiles) + { + static const std::filesystem::path RootPath; + std::map AllDirectories; + UpstreamDirectory RootDirectory = {.Path = RootPath}; + + AllDirectories[RootPath] = &RootDirectory; + + // Build tree from flat list + for (const auto& Path : InputFiles) + { + if (Path.has_parent_path()) + { + if (!AllDirectories.contains(Path.parent_path())) + { + std::stack PathSplit; + { + std::filesystem::path ParentPath = Path.parent_path(); + PathSplit.push(ParentPath.filename().string()); + while (ParentPath.has_parent_path()) + { + ParentPath = ParentPath.parent_path(); + PathSplit.push(ParentPath.filename().string()); + } + } + UpstreamDirectory* ParentPtr = &RootDirectory; + while (!PathSplit.empty()) + { + if (!ParentPtr->Directories.contains(PathSplit.top())) + { + std::filesystem::path NewParentPath = {ParentPtr->Path / PathSplit.top()}; + ParentPtr->Directories[PathSplit.top()] = {.Path = NewParentPath}; + AllDirectories[NewParentPath] = &ParentPtr->Directories[PathSplit.top()]; + } + ParentPtr = &ParentPtr->Directories[PathSplit.top()]; + PathSplit.pop(); + } + } + + AllDirectories[Path.parent_path()]->Files.insert(Path.filename().string()); + } + else + { + RootDirectory.Files.insert(Path.filename().string()); + } + } + + return RootDirectory; + } + + [[nodiscard]] CbObject BuildMerkleTreeDirectory(const UpstreamDirectory& RootDirectory, + const std::map& InputFileHashes, + const std::map& Blobs, + std::map& Objects) + { + CbObjectWriter DirectoryTreeWriter; + + if (!RootDirectory.Files.empty()) + { + DirectoryTreeWriter.BeginArray("f"sv); + for (const auto& File : RootDirectory.Files) + { + const std::filesystem::path FilePath = {RootDirectory.Path / File}; + const IoHash& FileHash = InputFileHashes.at(FilePath); + const uint64_t FileSize = Blobs.at(FileHash).Size(); + DirectoryTreeWriter.BeginObject(); + DirectoryTreeWriter.AddString("n"sv, File); + DirectoryTreeWriter.AddBinaryAttachment("h"sv, FileHash); + DirectoryTreeWriter.AddInteger("s"sv, FileSize); // Size + // DirectoryTreeWriter.AddInteger("a"sv, 0); // Attributes Currently unneeded + DirectoryTreeWriter.EndObject(); + } + DirectoryTreeWriter.EndArray(); + } + + if (!RootDirectory.Directories.empty()) + { + DirectoryTreeWriter.BeginArray("d"sv); + for (const auto& Item : RootDirectory.Directories) + { + CbObject Directory = BuildMerkleTreeDirectory(Item.second, InputFileHashes, Blobs, Objects); + const IoHash DirectoryHash = Directory.GetHash(); + Objects[DirectoryHash] = std::move(Directory); + + DirectoryTreeWriter.BeginObject(); + DirectoryTreeWriter.AddString("n"sv, Item.first); + DirectoryTreeWriter.AddObjectAttachment("h"sv, DirectoryHash); + DirectoryTreeWriter.EndObject(); + } + DirectoryTreeWriter.EndArray(); + } + + return std::move(DirectoryTreeWriter.Save()); + } + + [[nodiscard]] CbObject BuildRequirements(const std::string_view Condition, + const std::map& Resources, + const bool Exclusive) + { + CbObjectWriter Writer; + Writer.AddString("c", Condition); + if (!Resources.empty()) + { + Writer.BeginArray("r"); + for (const auto& Resource : Resources) + { + Writer.BeginArray(); + Writer.AddString(Resource.first); + Writer.AddInteger(Resource.second); + Writer.EndArray(); + } + Writer.EndArray(); + } + Writer.AddBool("e", Exclusive); + return std::move(Writer.Save()); + } + + [[nodiscard]] CbObject BuildTask(const std::string_view Executable, + const std::vector& Arguments, + const std::map& Environment, + const std::string_view WorkingDirectory, + const IoHash& SandboxHash, + const IoHash& RequirementsId, + const std::set& Outputs) + { + CbObjectWriter TaskWriter; + TaskWriter.AddString("e"sv, Executable); + + if (!Arguments.empty()) + { + TaskWriter.BeginArray("a"sv); + for (const auto& Argument : Arguments) + { + TaskWriter.AddString(Argument); + } + TaskWriter.EndArray(); + } + + if (!Environment.empty()) + { + TaskWriter.BeginArray("v"sv); + for (const auto& Env : Environment) + { + TaskWriter.BeginArray(); + TaskWriter.AddString(Env.first); + TaskWriter.AddString(Env.second); + TaskWriter.EndArray(); + } + TaskWriter.EndArray(); + } + + if (!WorkingDirectory.empty()) + { + TaskWriter.AddString("s"sv, WorkingDirectory); + } + + TaskWriter.AddObjectAttachment("s"sv, SandboxHash); + TaskWriter.AddObjectAttachment("r"sv, RequirementsId); + + // Outputs + if (!Outputs.empty()) + { + TaskWriter.BeginArray("o"sv); + for (const auto& Output : Outputs) + { + TaskWriter.AddString(Output); + } + TaskWriter.EndArray(); + } + + return std::move(TaskWriter.Save()); + } + }; +} // namespace detail + +////////////////////////////////////////////////////////////////////////// + +struct UpstreamApplyStats +{ + static constexpr uint64_t MaxSampleCount = 1000ull; + + UpstreamApplyStats(bool Enabled) : m_Enabled(Enabled) {} + + void Add(spdlog::logger& Logger, + UpstreamApplyEndpoint& Endpoint, + const PostUpstreamApplyResult& Result, + const std::vector>& Endpoints) + { + UpstreamApplyEndpointStats& Stats = Endpoint.Stats(); + + if (Result.Error) + { + Stats.ErrorCount++; + } + else if (Result.Success) + { + Stats.PostCount++; + Stats.UpBytes.fetch_add(double(Result.Bytes) / 1024.0 / 1024.0); + Stats.SecondsUp.fetch_add(Result.ElapsedSeconds); + } + + if (m_Enabled && m_SampleCount++ % MaxSampleCount) + { + Dump(Logger, Endpoints); + } + } + + void Add(spdlog::logger& Logger, + UpstreamApplyEndpoint& Endpoint, + const GetUpstreamApplyUpdatesResult& Result, + const std::vector>& Endpoints) + { + UpstreamApplyEndpointStats& Stats = Endpoint.Stats(); + + if (Result.Error) + { + Stats.ErrorCount++; + } + else if (Result.Success) + { + Stats.UpdateCount++; + Stats.DownBytes.fetch_add(double(Result.Bytes) / 1024.0 / 1024.0); + Stats.SecondsDown.fetch_add(Result.ElapsedSeconds); + if (!Result.Completed.empty()) + { + uint64_t Completed = 0; + for (auto& It : Result.Completed) + { + Completed += It.second.size(); + } + Stats.CompleteCount.fetch_add(Completed); + } + } + + if (m_Enabled && m_SampleCount++ % MaxSampleCount) + { + Dump(Logger, Endpoints); + } + } + + void Dump(spdlog::logger& Logger, const std::vector>& Endpoints) + { + for (auto& Ep : Endpoints) + { + // These stats will not be totally correct as the numbers are not captured atomically + + UpstreamApplyEndpointStats& Stats = Ep->Stats(); + const uint64_t PostCount = Stats.PostCount; + const uint64_t CompleteCount = Stats.CompleteCount; + const uint64_t UpdateCount = Stats.UpdateCount; + const double DownBytes = Stats.DownBytes; + const double SecondsDown = Stats.SecondsDown; + const double UpBytes = Stats.UpBytes; + const double SecondsUp = Stats.SecondsUp; + + const double UpSpeed = UpBytes > 0 ? UpBytes / SecondsUp : 0.0; + const double DownSpeed = DownBytes > 0 ? DownBytes / SecondsDown : 0.0; + const double CompleteRate = CompleteCount > 0 ? (double(PostCount) / double(CompleteCount)) : 0.0; + + Logger.debug("STATS - '{}', Complete rate: {:.2f}%, DOWN: '{:.2f} MiB {:.2f} MiB/s', UP: '{:.2f} MiB {:.2f} MiB/s'", + Ep->DisplayName(), + CompleteRate, + DownBytes, + DownSpeed, + UpBytes, + UpSpeed); + } + } + + bool m_Enabled; + std::atomic_uint64_t m_SampleCount = {}; +}; + +////////////////////////////////////////////////////////////////////////// + +class DefaultUpstreamApply final : public UpstreamApply +{ +public: + DefaultUpstreamApply(const UpstreamApplyOptions& Options, CasStore& CasStore, CidStore& CidStore) + : m_Log(logging::Get("upstream-apply")) + , m_Options(Options) + , m_CasStore(CasStore) + , m_CidStore(CidStore) + , m_Stats(Options.StatsEnabled) + { + } + + virtual ~DefaultUpstreamApply() { Shutdown(); } + + virtual bool Initialize() override + { + for (auto& Endpoint : m_Endpoints) + { + const UpstreamEndpointHealth Health = Endpoint->Initialize(); + if (Health.Ok) + { + Log().info("initialize endpoint '{}' OK", Endpoint->DisplayName()); + } + else + { + Log().warn("initialize endpoint '{}' FAILED, reason '{}'", Endpoint->DisplayName(), Health.Reason); + } + } + + m_RunState.IsRunning = !m_Endpoints.empty(); + + if (m_RunState.IsRunning) + { + for (uint32_t Idx = 0; Idx < m_Options.ThreadCount; Idx++) + { + m_UpstreamThreads.emplace_back(&DefaultUpstreamApply::ProcessUpstreamQueue, this); + } + + m_UpstreamUpdatesThread = std::thread(&DefaultUpstreamApply::ProcessUpstreamUpdates, this); + + m_EndpointMonitorThread = std::thread(&DefaultUpstreamApply::MonitorEndpoints, this); + } + + return m_RunState.IsRunning; + } + + virtual void RegisterEndpoint(std::unique_ptr Endpoint) override + { + m_Endpoints.emplace_back(std::move(Endpoint)); + } + + virtual EnqueueResult EnqueueUpstream(UpstreamApplyRecord ApplyRecord) override + { + if (m_RunState.IsRunning) + { + const IoHash WorkerId = ApplyRecord.WorkerDescriptor.GetHash(); + const IoHash ActionId = ApplyRecord.Action.GetHash(); + + { + std::scoped_lock Lock(m_ApplyTasksMutex); + if (auto Status = FindStatus(WorkerId, ActionId); Status != nullptr) + { + // Already in progress + return {.ApplyId = ActionId, .Success = true}; + } + + std::chrono::steady_clock::time_point ExpireTime = + ApplyRecord.ExpireSeconds > 0 ? std::chrono::steady_clock::now() + std::chrono::seconds(ApplyRecord.ExpireSeconds) + : std::chrono::steady_clock::time_point::max(); + + m_ApplyTasks[WorkerId][ActionId] = {.State = UpstreamApplyState::Queued, .Result{}, .ExpireTime = std::move(ExpireTime)}; + } + + if (!m_UpstreamThreads.empty()) + { + m_UpstreamQueue.Enqueue(std::move(ApplyRecord)); + } + else + { + ProcessApplyRecord(std::move(ApplyRecord)); + } + + return {.ApplyId = ActionId, .Success = true}; + } + + return {}; + } + + virtual StatusResult GetStatus(const IoHash& WorkerId, const IoHash& ActionId) override + { + if (m_RunState.IsRunning) + { + std::scoped_lock Lock(m_ApplyTasksMutex); + if (auto Status = FindStatus(WorkerId, ActionId); Status != nullptr) + { + return {.Status = *Status, .Success = true}; + } + } + + return {}; + } + + virtual void GetStatus(CbObjectWriter& Status) override + { + Status << "worker_threads" << m_Options.ThreadCount; + Status << "queue_count" << m_UpstreamQueue.Size(); + + Status.BeginArray("endpoints"); + for (const auto& Ep : m_Endpoints) + { + Status.BeginObject(); + Status << "name" << Ep->DisplayName(); + Status << "health" << (Ep->IsHealthy() ? "ok"sv : "inactive"sv); + + UpstreamApplyEndpointStats& Stats = Ep->Stats(); + const uint64_t PostCount = Stats.PostCount; + const uint64_t CompleteCount = Stats.CompleteCount; + const uint64_t UpdateCount = Stats.UpdateCount; + const double CompleteRate = CompleteCount > 0 ? (double(PostCount) / double(CompleteCount)) : 0.0; + + Status << "post_count" << PostCount; + Status << "complete_count" << PostCount; + Status << "update_count" << Stats.UpdateCount; + + Status << "complete_ratio" << CompleteRate; + Status << "downloaded_mb" << Stats.DownBytes; + Status << "uploaded_mb" << Stats.UpBytes; + Status << "error_count" << Stats.ErrorCount; + + Status.EndObject(); + } + Status.EndArray(); + } + +private: + // The caller is responsible for locking if required + UpstreamApplyStatus* FindStatus(const IoHash& WorkerId, const IoHash& ActionId) + { + if (auto It = m_ApplyTasks.find(WorkerId); It != m_ApplyTasks.end()) + { + if (auto It2 = It->second.find(ActionId); It2 != It->second.end()) + { + return &It2->second; + } + } + return nullptr; + } + + void ProcessApplyRecord(UpstreamApplyRecord ApplyRecord) + { + const IoHash WorkerId = ApplyRecord.WorkerDescriptor.GetHash(); + const IoHash ActionId = ApplyRecord.Action.GetHash(); + try + { + for (auto& Endpoint : m_Endpoints) + { + if (Endpoint->IsHealthy()) + { + PostUpstreamApplyResult Result = Endpoint->PostApply(std::move(ApplyRecord)); + { + std::scoped_lock Lock(m_ApplyTasksMutex); + if (auto Status = FindStatus(WorkerId, ActionId); Status != nullptr) + { + if (Result.Success) + { + Status->State = UpstreamApplyState::Executing; + } + else + { + Status->State = UpstreamApplyState::Complete; + Status->Result = {.Error = std::move(Result.Error), + .Bytes = Result.Bytes, + .ElapsedSeconds = Result.ElapsedSeconds}; + } + } + } + m_Stats.Add(m_Log, *Endpoint, Result, m_Endpoints); + return; + } + } + + { + std::scoped_lock Lock(m_ApplyTasksMutex); + if (auto Status = FindStatus(WorkerId, ActionId); Status != nullptr) + { + Status->State = UpstreamApplyState::Complete; + Status->Result = {.Error{.ErrorCode = -1, .Reason = "No available endpoint"}}; + } + Log().warn("process upstream apply ({}/{}) FAILED 'No available endpoint'", WorkerId, ActionId); + } + } + catch (std::exception& e) + { + std::scoped_lock Lock(m_ApplyTasksMutex); + if (auto Status = FindStatus(WorkerId, ActionId); Status != nullptr) + { + Status->State = UpstreamApplyState::Complete; + Status->Result = {.Error{.ErrorCode = -1, .Reason = e.what()}}; + } + Log().warn("process upstream apply ({}/{}) FAILED '{}'", WorkerId, ActionId, e.what()); + } + } + + void ProcessUpstreamQueue() + { + for (;;) + { + UpstreamApplyRecord ApplyRecord; + if (m_UpstreamQueue.WaitAndDequeue(ApplyRecord)) + { + ProcessApplyRecord(std::move(ApplyRecord)); + } + + if (!m_RunState.IsRunning) + { + break; + } + } + } + + void ProcessApplyUpdates() + { + for (auto& Endpoint : m_Endpoints) + { + if (Endpoint->IsHealthy()) + { + GetUpstreamApplyUpdatesResult Result = Endpoint->GetUpdates(); + m_Stats.Add(m_Log, *Endpoint, Result, m_Endpoints); + + if (!Result.Success) + { + Log().warn("process upstream apply updates FAILED '{}'", Result.Error.Reason); + } + + if (!Result.Completed.empty()) + { + for (auto& It : Result.Completed) + { + for (auto& It2 : It.second) + { + std::scoped_lock Lock(m_ApplyTasksMutex); + if (auto Status = FindStatus(It.first, It2.first); Status != nullptr) + { + Status->State = UpstreamApplyState::Complete; + Status->Result = std::move(It2.second); + } + } + } + } + } + } + } + + void ProcessUpstreamUpdates() + { + const auto& UpdateSleep = std::chrono::seconds(m_Options.UpdatesInterval); + for (;;) + { + std::this_thread::sleep_for(UpdateSleep); + + if (!m_RunState.IsRunning) + { + break; + } + + ProcessApplyUpdates(); + + // Remove any expired tasks, regardless of state + { + std::scoped_lock Lock(m_ApplyTasksMutex); + for (auto& WorkerIt : m_ApplyTasks) + { + const auto Count = std::erase_if(WorkerIt.second, [](const auto& Item) { + return Item.second.ExpireTime < std::chrono::steady_clock::now(); + }); + if (Count > 0) + { + Log().debug("Removed '{}' expired tasks", Count); + } + } + const auto Count = std::erase_if(m_ApplyTasks, [](const auto& Item) { return Item.second.empty(); }); + if (Count > 0) + { + Log().debug("Removed '{}' empty task lists", Count); + } + } + } + } + + void MonitorEndpoints() + { + for (;;) + { + { + std::unique_lock Lock(m_RunState.Mutex); + if (m_RunState.ExitSignal.wait_for(Lock, m_Options.HealthCheckInterval, [this]() { return !m_RunState.IsRunning.load(); })) + { + break; + } + } + + for (auto& Endpoint : m_Endpoints) + { + if (!Endpoint->IsHealthy()) + { + if (const UpstreamEndpointHealth Health = Endpoint->CheckHealth(); Health.Ok) + { + Log().warn("health check endpoint '{}' OK", Endpoint->DisplayName(), Health.Reason); + } + else + { + Log().warn("health check endpoint '{}' FAILED, reason '{}'", Endpoint->DisplayName(), Health.Reason); + } + } + } + } + } + + void Shutdown() + { + if (m_RunState.Stop()) + { + m_UpstreamQueue.CompleteAdding(); + for (std::thread& Thread : m_UpstreamThreads) + { + Thread.join(); + } + + m_EndpointMonitorThread.join(); + m_UpstreamUpdatesThread.join(); + m_UpstreamThreads.clear(); + m_Endpoints.clear(); + } + } + + spdlog::logger& Log() { return m_Log; } + + using UpstreamApplyQueue = BlockingQueue; + + struct RunState + { + std::mutex Mutex; + std::condition_variable ExitSignal; + std::atomic_bool IsRunning{false}; + + bool Stop() + { + bool Stopped = false; + { + std::scoped_lock Lock(Mutex); + Stopped = IsRunning.exchange(false); + } + if (Stopped) + { + ExitSignal.notify_all(); + } + return Stopped; + } + }; + + spdlog::logger& m_Log; + UpstreamApplyOptions m_Options; + CasStore& m_CasStore; + CidStore& m_CidStore; + UpstreamApplyQueue m_UpstreamQueue; + UpstreamApplyStats m_Stats; + UpstreamApplyTasks m_ApplyTasks; + std::mutex m_ApplyTasksMutex; + std::vector> m_Endpoints; + std::vector m_UpstreamThreads; + std::thread m_UpstreamUpdatesThread; + std::thread m_EndpointMonitorThread; + RunState m_RunState; +}; + +////////////////////////////////////////////////////////////////////////// + +std::unique_ptr +MakeUpstreamApply(const UpstreamApplyOptions& Options, CasStore& CasStore, CidStore& CidStore) +{ + return std::make_unique(Options, CasStore, CidStore); +} + +std::unique_ptr +MakeHordeUpstreamEndpoint(const CloudCacheClientOptions& Options, CasStore& CasStore, CidStore& CidStore) +{ + return std::make_unique(Options, CasStore, CidStore); +} + +} // namespace zen diff --git a/zenserver/upstream/upstreamapply.h b/zenserver/upstream/upstreamapply.h new file mode 100644 index 000000000..0d26e9d06 --- /dev/null +++ b/zenserver/upstream/upstreamapply.h @@ -0,0 +1,173 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace zen { + +class CbObjectWriter; +class CasStore; +class CidStore; +class ZenCacheStore; +struct CloudCacheClientOptions; + +enum class UpstreamApplyState : int32_t +{ + Queued = 0, + Executing = 1, + Complete = 2, +}; + +struct UpstreamApplyRecord +{ + CbObject WorkerDescriptor; + CbObject Action; + uint32_t ExpireSeconds{}; +}; + +struct UpstreamApplyOptions +{ + std::chrono::seconds HealthCheckInterval{5}; + std::chrono::seconds UpdatesInterval{5}; + uint32_t ThreadCount = 4; + bool StatsEnabled = false; +}; + +struct UpstreamApplyError +{ + int32_t ErrorCode{}; + std::string Reason{}; + + explicit operator bool() const { return ErrorCode != 0; } +}; + +struct PostUpstreamApplyResult +{ + UpstreamApplyError Error{}; + int64_t Bytes{}; + double ElapsedSeconds{}; + bool Success = false; +}; + +struct GetUpstreamApplyResult +{ + CbPackage OutputPackage{}; + int64_t TotalAttachmentBytes{}; + int64_t TotalRawAttachmentBytes{}; + UpstreamApplyError Error{}; + int64_t Bytes{}; + double ElapsedSeconds{}; + std::string StdOut{}; + std::string StdErr{}; + bool Success = false; +}; + +using UpstreamApplyCompleted = std::unordered_map>; + +struct GetUpstreamApplyUpdatesResult +{ + UpstreamApplyError Error{}; + int64_t Bytes{}; + double ElapsedSeconds{}; + UpstreamApplyCompleted Completed{}; + bool Success = false; +}; + +struct UpstreamApplyStatus +{ + UpstreamApplyState State{}; + GetUpstreamApplyResult Result{}; + std::chrono::steady_clock::time_point ExpireTime{}; +}; + +using UpstreamApplyTasks = std::unordered_map>; + +struct UpstreamEndpointHealth +{ + std::string Reason; + bool Ok = false; +}; + +struct UpstreamApplyEndpointStats +{ + std::atomic_uint64_t PostCount{}; + std::atomic_uint64_t CompleteCount{}; + std::atomic_uint64_t UpdateCount{}; + std::atomic_uint64_t ErrorCount{}; + std::atomic UpBytes{}; + std::atomic DownBytes{}; + std::atomic SecondsUp{}; + std::atomic SecondsDown{}; +}; + +/** + * The upstream apply endpont is responsible for handling remote execution. + */ +class UpstreamApplyEndpoint +{ +public: + virtual ~UpstreamApplyEndpoint() = default; + + virtual UpstreamEndpointHealth Initialize() = 0; + + virtual bool IsHealthy() const = 0; + + virtual UpstreamEndpointHealth CheckHealth() = 0; + + virtual std::string_view DisplayName() const = 0; + + virtual PostUpstreamApplyResult PostApply(const UpstreamApplyRecord& ApplyRecord) = 0; + + virtual GetUpstreamApplyUpdatesResult GetUpdates() = 0; + + virtual UpstreamApplyEndpointStats& Stats() = 0; +}; + +/** + * Manages one or more upstream cache endpoints. + */ +class UpstreamApply +{ +public: + virtual ~UpstreamApply() = default; + + virtual bool Initialize() = 0; + + virtual void RegisterEndpoint(std::unique_ptr Endpoint) = 0; + + struct EnqueueResult + { + IoHash ApplyId{}; + bool Success = false; + }; + + struct StatusResult + { + UpstreamApplyStatus Status{}; + bool Success = false; + }; + + virtual EnqueueResult EnqueueUpstream(UpstreamApplyRecord ApplyRecord) = 0; + + virtual StatusResult GetStatus(const IoHash& WorkerId, const IoHash& ActionId) = 0; + + virtual void GetStatus(CbObjectWriter& CbO) = 0; +}; + +std::unique_ptr MakeUpstreamApply(const UpstreamApplyOptions& Options, CasStore& CasStore, CidStore& CidStore); + +std::unique_ptr MakeHordeUpstreamEndpoint(const CloudCacheClientOptions& Options, + CasStore& CasStore, + CidStore& CidStore); + +} // namespace zen diff --git a/zenserver/zenserver.vcxproj b/zenserver/zenserver.vcxproj index d954d3f8d..480d5dd15 100644 --- a/zenserver/zenserver.vcxproj +++ b/zenserver/zenserver.vcxproj @@ -125,6 +125,7 @@ + @@ -147,6 +148,7 @@ + diff --git a/zenserver/zenserver.vcxproj.filters b/zenserver/zenserver.vcxproj.filters index 04c6267ba..6de9230d3 100644 --- a/zenserver/zenserver.vcxproj.filters +++ b/zenserver/zenserver.vcxproj.filters @@ -41,6 +41,9 @@ + + upstream + @@ -76,6 +79,9 @@ + + upstream + -- cgit v1.2.3 From 3faf0b57c625152a8facfca1c4995bd9edc95707 Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Fri, 12 Nov 2021 11:08:53 +0100 Subject: Movec cache utility types to zenutil and fixed unit tests. --- zenserver/cache/cachekey.cpp | 163 ------------------------------------ zenserver/cache/cachekey.h | 135 ----------------------------- zenserver/cache/structuredcache.cpp | 49 ++++++----- zenserver/upstream/upstreamcache.h | 3 +- zenserver/zenserver.vcxproj | 2 - zenserver/zenserver.vcxproj.filters | 6 -- 6 files changed, 27 insertions(+), 331 deletions(-) delete mode 100644 zenserver/cache/cachekey.cpp delete mode 100644 zenserver/cache/cachekey.h (limited to 'zenserver') diff --git a/zenserver/cache/cachekey.cpp b/zenserver/cache/cachekey.cpp deleted file mode 100644 index 2ead9ac58..000000000 --- a/zenserver/cache/cachekey.cpp +++ /dev/null @@ -1,163 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "cachekey.h" - -#include -#include -#include - -namespace zen { - -using namespace std::literals; - -namespace detail { namespace cacheopt { - constexpr std::string_view Local = "local"sv; - constexpr std::string_view Remote = "remote"sv; - constexpr std::string_view Data = "data"sv; - constexpr std::string_view Meta = "meta"sv; - constexpr std::string_view Value = "value"sv; - constexpr std::string_view Attachments = "attachments"sv; -}} // namespace detail::cacheopt - -CachePolicy -ParseQueryCachePolicy(std::string_view QueryPolicy, CachePolicy Default) -{ - if (QueryPolicy.empty()) - { - return Default; - } - - CachePolicy Result = CachePolicy::None; - - ForEachStrTok(QueryPolicy, ',', [&Result](const std::string_view& Token) { - if (Token == detail::cacheopt::Local) - { - Result |= CachePolicy::QueryLocal; - } - if (Token == detail::cacheopt::Remote) - { - Result |= CachePolicy::QueryRemote; - } - return true; - }); - - return Result; -} - -CachePolicy -ParseStoreCachePolicy(std::string_view StorePolicy, CachePolicy Default) -{ - if (StorePolicy.empty()) - { - return Default; - } - - CachePolicy Result = CachePolicy::None; - - ForEachStrTok(StorePolicy, ',', [&Result](const std::string_view& Token) { - if (Token == detail::cacheopt::Local) - { - Result |= CachePolicy::StoreLocal; - } - if (Token == detail::cacheopt::Remote) - { - Result |= CachePolicy::StoreRemote; - } - return true; - }); - - return Result; -} - -CachePolicy -ParseSkipCachePolicy(std::string_view SkipPolicy, CachePolicy Default) -{ - if (SkipPolicy.empty()) - { - return Default; - } - - CachePolicy Result = CachePolicy::None; - - ForEachStrTok(SkipPolicy, ',', [&Result](const std::string_view& Token) { - if (Token == detail::cacheopt::Meta) - { - Result |= CachePolicy::SkipMeta; - } - if (Token == detail::cacheopt::Value) - { - Result |= CachePolicy::SkipValue; - } - if (Token == detail::cacheopt::Attachments) - { - Result |= CachePolicy::SkipAttachments; - } - if (Token == detail::cacheopt::Data) - { - Result |= CachePolicy::SkipData; - } - return true; - }); - - return Result; -} - -CachePolicy -CacheRecordPolicy::GetPayloadPolicy(const Oid& PayloadId) const -{ - if (const auto It = m_PayloadPolicies.find(PayloadId); It != m_PayloadPolicies.end()) - { - return It->second; - } - - return m_DefaultPayloadPolicy; -} - -bool -CacheRecordPolicy::Load(CbObjectView RecordPolicyObject, CacheRecordPolicy& OutRecordPolicy) -{ - using namespace std::literals; - - const uint32_t RecordPolicy = RecordPolicyObject["RecordPolicy"sv].AsUInt32(static_cast(CachePolicy::Default)); - const uint32_t DefaultPayloadPolicy = - RecordPolicyObject["DefaultPayloadPolicy"sv].AsUInt32(static_cast(CachePolicy::Default)); - - OutRecordPolicy.m_RecordPolicy = static_cast(RecordPolicy); - OutRecordPolicy.m_DefaultPayloadPolicy = static_cast(DefaultPayloadPolicy); - - for (CbFieldView PayloadPolicyView : RecordPolicyObject["PayloadPolicies"sv]) - { - CbObjectView PayloadPolicyObject = PayloadPolicyView.AsObjectView(); - const Oid PayloadId = PayloadPolicyObject["Id"sv].AsObjectId(); - const uint32_t PayloadPolicy = PayloadPolicyObject["Policy"sv].AsUInt32(); - - if (PayloadId != Oid::Zero && PayloadPolicy != 0) - { - OutRecordPolicy.m_PayloadPolicies.emplace(PayloadId, static_cast(PayloadPolicy)); - } - } - - return true; -} - -void -CacheRecordPolicy::Save(const CacheRecordPolicy& Policy, CbWriter& Writer) -{ - Writer << "RecordPolicy"sv << static_cast(Policy.GetRecordPolicy()); - Writer << "DefaultPayloadPolicy"sv << static_cast(Policy.GetDefaultPayloadPolicy()); - - if (!Policy.m_PayloadPolicies.empty()) - { - Writer.BeginArray("PayloadPolicies"sv); - for (const auto& Kv : Policy.m_PayloadPolicies) - { - Writer.AddObjectId("Id"sv, Kv.first); - Writer << "Policy"sv << static_cast(Kv.second); - } - Writer.EndArray(); - } -} - -const CacheKey CacheKey::Empty = CacheKey{.Bucket = std::string(), .Hash = IoHash()}; - -} // namespace zen diff --git a/zenserver/cache/cachekey.h b/zenserver/cache/cachekey.h deleted file mode 100644 index c32f7ed87..000000000 --- a/zenserver/cache/cachekey.h +++ /dev/null @@ -1,135 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include -#include -#include -#include - -#include - -namespace zen { - -class CbObjectView; -class CbWriter; - -enum class CachePolicy : uint8_t -{ - None = 0, - QueryLocal = 1 << 0, - QueryRemote = 1 << 1, - Query = QueryLocal | QueryRemote, - StoreLocal = 1 << 2, - StoreRemote = 1 << 3, - Store = StoreLocal | StoreRemote, - SkipMeta = 1 << 4, - SkipValue = 1 << 5, - SkipAttachments = 1 << 6, - SkipData = SkipMeta | SkipValue | SkipAttachments, - SkipLocalCopy = 1 << 7, - Local = QueryLocal | StoreLocal, - Remote = QueryRemote | StoreRemote, - Default = Query | Store, - Disable = None, -}; - -gsl_DEFINE_ENUM_BITMASK_OPERATORS(CachePolicy); - -CachePolicy ParseQueryCachePolicy(std::string_view QueryPolicy, CachePolicy Default = CachePolicy::Query); - -CachePolicy ParseStoreCachePolicy(std::string_view StorePolicy, CachePolicy Default = CachePolicy::Store); - -CachePolicy ParseSkipCachePolicy(std::string_view SkipPolicy, CachePolicy Default = CachePolicy::None); - -class CacheRecordPolicy -{ -public: - CacheRecordPolicy() = default; - - CachePolicy GetRecordPolicy() const { return m_RecordPolicy; } - CachePolicy GetPayloadPolicy(const Oid& PayloadId) const; - CachePolicy GetDefaultPayloadPolicy() const { return m_DefaultPayloadPolicy; } - - static bool Load(CbObjectView RecordPolicyObject, CacheRecordPolicy& OutRecordPolicy); - static void Save(const CacheRecordPolicy& Policy, CbWriter& Writer); - -private: - using PayloadPolicyMap = std::unordered_map; - - CachePolicy m_RecordPolicy = CachePolicy::Default; - CachePolicy m_DefaultPayloadPolicy = CachePolicy::Default; - PayloadPolicyMap m_PayloadPolicies; -}; - -struct CacheKey -{ - std::string Bucket; - IoHash Hash; - - static CacheKey Create(std::string_view Bucket, const IoHash& Hash) { return {.Bucket = ToLower(Bucket), .Hash = Hash}; } - - static const CacheKey Empty; -}; - -inline bool -operator==(const CacheKey& A, const CacheKey& B) -{ - return A.Bucket == B.Bucket && A.Hash == B.Hash; -} - -inline bool -operator!=(const CacheKey& A, const CacheKey& B) -{ - return A.Bucket != B.Bucket || A.Hash != B.Hash; -} - -inline bool -operator<(const CacheKey& A, const CacheKey& B) -{ - const std::string& BucketA = A.Bucket; - const std::string& BucketB = B.Bucket; - return BucketA == BucketB ? A.Hash < B.Hash : BucketA < BucketB; -} - -struct CacheChunkRequest -{ - CacheKey Key; - IoHash ChunkId; - Oid PayloadId; - uint64_t RawOffset = 0ull; - uint64_t RawSize = ~uint64_t(0); - CachePolicy Policy = CachePolicy::Default; -}; - -inline bool -operator<(const CacheChunkRequest& A, const CacheChunkRequest& B) -{ - if (A.Key < B.Key) - { - return true; - } - if (B.Key < A.Key) - { - return false; - } - if (A.ChunkId < B.ChunkId) - { - return true; - } - if (B.ChunkId < A.ChunkId) - { - return false; - } - if (A.PayloadId < B.PayloadId) - { - return true; - } - if (B.PayloadId < A.PayloadId) - { - return false; - } - return A.RawOffset < B.RawOffset; -} - -} // namespace zen diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index e78e583aa..1b6406562 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -11,8 +11,9 @@ #include #include #include +#include -#include "cachekey.h" +//#include "cachekey.h" #include "monitoring/httpstats.h" #include "structuredcache.h" #include "structuredcachestore.h" @@ -796,15 +797,15 @@ HttpStructuredCacheService::HandleRpcRequest(zen::HttpServerRequest& Request) } Request.WriteResponseAsync( - [this, BatchRequest = zen::LoadCompactBinaryObject(Request.ReadPayload())](HttpServerRequest& AsyncRequest) { - const std::string_view Method = BatchRequest["Method"sv].AsString(); + [this, RpcRequest = zen::LoadCompactBinaryObject(Request.ReadPayload())](HttpServerRequest& AsyncRequest) { + const std::string_view Method = RpcRequest["Method"sv].AsString(); if (Method == "GetCacheRecords"sv) { - HandleRpcGetCacheRecords(AsyncRequest, BatchRequest); + HandleRpcGetCacheRecords(AsyncRequest, RpcRequest); } else if (Method == "GetCachePayloads"sv) { - HandleRpcGetCachePayloads(AsyncRequest, BatchRequest); + HandleRpcGetCachePayloads(AsyncRequest, RpcRequest); } else { @@ -820,18 +821,18 @@ HttpStructuredCacheService::HandleRpcRequest(zen::HttpServerRequest& Request) } void -HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Request, CbObjectView BatchRequest) +HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Request, CbObjectView RpcRequest) { using namespace fmt::literals; - CbPackage BatchResponse; + CbPackage RpcResponse; CacheRecordPolicy Policy; - CbObjectView Params = BatchRequest["Params"sv].AsObjectView(); + CbObjectView Params = RpcRequest["Params"sv].AsObjectView(); std::vector CacheKeys; std::vector CacheValues; std::vector UpstreamRequests; - ZEN_ASSERT(BatchRequest["Method"sv].AsString() == "GetCacheRecords"sv); + ZEN_ASSERT(RpcRequest["Method"sv].AsString() == "GetCacheRecords"sv); CacheRecordPolicy::Load(Params["Policy"sv].AsObjectView(), Policy); @@ -860,10 +861,10 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Req if (!SkipAttachments) { - CacheRecord.IterateAttachments([this, &BatchResponse](CbFieldView AttachmentHash) { + CacheRecord.IterateAttachments([this, &RpcResponse](CbFieldView AttachmentHash) { if (IoBuffer Chunk = m_CidStore.FindChunkByCid(AttachmentHash.AsHash())) { - BatchResponse.AddAttachment(CbAttachment(CompressedBuffer::FromCompressed(SharedBuffer(Chunk)))); + RpcResponse.AddAttachment(CbAttachment(CompressedBuffer::FromCompressed(SharedBuffer(Chunk)))); } }); } @@ -893,7 +894,7 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Req if (!UpstreamRequests.empty() && m_UpstreamCache) { const auto OnCacheRecordGetComplete = - [this, &CacheKeys, &CacheValues, &BatchResponse, SkipAttachments](CacheRecordGetCompleteParams&& Params) { + [this, &CacheKeys, &CacheValues, &RpcResponse, SkipAttachments](CacheRecordGetCompleteParams&& Params) { if (Params.Record) { Params.Record.IterateAttachments([&](CbFieldView AttachmentHash) { @@ -905,7 +906,7 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Req if (!SkipAttachments) { - BatchResponse.AddAttachment(CbAttachment(Compressed)); + RpcResponse.AddAttachment(CbAttachment(Compressed)); } } } @@ -917,6 +918,7 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Req NiceBytes(Params.Record.GetView().GetSize()), ToString(HttpContentType::kCbObject)); + ZEN_ASSERT(Params.KeyIndex < CacheValues.size()); CacheValues[Params.KeyIndex] = IoBufferBuilder::MakeCloneFromMemory(Params.Record.GetView()); m_CacheStats.HitCount++; m_CacheStats.UpstreamHitCount++; @@ -948,10 +950,10 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Req } ResponseObject.EndArray(); - BatchResponse.SetObject(ResponseObject.Save()); + RpcResponse.SetObject(ResponseObject.Save()); BinaryWriter MemStream; - BatchResponse.Save(MemStream); + RpcResponse.Save(MemStream); Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, @@ -959,16 +961,16 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Req } void -HttpStructuredCacheService::HandleRpcGetCachePayloads(zen::HttpServerRequest& Request, CbObjectView BatchRequest) +HttpStructuredCacheService::HandleRpcGetCachePayloads(zen::HttpServerRequest& Request, CbObjectView RpcRequest) { using namespace fmt::literals; - ZEN_ASSERT(BatchRequest["Method"sv].AsString() == "GetCachePayloads"sv); + ZEN_ASSERT(RpcRequest["Method"sv].AsString() == "GetCachePayloads"sv); std::vector ChunkRequests; std::vector UpstreamRequests; std::vector Chunks; - CbObjectView Params = BatchRequest["Params"sv].AsObjectView(); + CbObjectView Params = RpcRequest["Params"sv].AsObjectView(); for (CbFieldView RequestView : Params["ChunkRequests"sv]) { @@ -1029,7 +1031,7 @@ HttpStructuredCacheService::HandleRpcGetCachePayloads(zen::HttpServerRequest& Re CurrentKey = ChunkRequest.Key; ZenCacheValue CacheValue; - if (m_CacheStore.Get(ChunkRequest.Key.Bucket, ChunkRequest.Key.Hash, CacheValue)) + if (m_CacheStore.Get(CurrentKey.Bucket, CurrentKey.Hash, CacheValue)) { CurrentRecordBuffer = CacheValue.Value; } @@ -1094,6 +1096,7 @@ HttpStructuredCacheService::HandleRpcGetCachePayloads(zen::HttpServerRequest& Re NiceBytes(Params.Payload.GetSize()), "UPSTREAM"); + ZEN_ASSERT(Params.RequestIndex < Chunks.size()); Chunks[Params.RequestIndex] = std::move(Params.Payload); m_CacheStats.HitCount++; @@ -1109,7 +1112,7 @@ HttpStructuredCacheService::HandleRpcGetCachePayloads(zen::HttpServerRequest& Re m_UpstreamCache->GetCachePayloads(ChunkRequests, UpstreamRequests, std::move(OnCachePayloadGetComplete)); } - CbPackage BatchResponse; + CbPackage RpcResponse; CbObjectWriter ResponseObject; ResponseObject.BeginArray("Result"sv); @@ -1119,7 +1122,7 @@ HttpStructuredCacheService::HandleRpcGetCachePayloads(zen::HttpServerRequest& Re if (Chunks[ChunkIndex]) { ResponseObject << ChunkRequests[ChunkIndex].ChunkId; - BatchResponse.AddAttachment(CbAttachment(CompressedBuffer::FromCompressed(SharedBuffer(std::move(Chunks[ChunkIndex]))))); + RpcResponse.AddAttachment(CbAttachment(CompressedBuffer::FromCompressed(SharedBuffer(std::move(Chunks[ChunkIndex]))))); } else { @@ -1128,10 +1131,10 @@ HttpStructuredCacheService::HandleRpcGetCachePayloads(zen::HttpServerRequest& Re } ResponseObject.EndArray(); - BatchResponse.SetObject(ResponseObject.Save()); + RpcResponse.SetObject(ResponseObject.Save()); BinaryWriter MemStream; - BatchResponse.Save(MemStream); + RpcResponse.Save(MemStream); Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, diff --git a/zenserver/upstream/upstreamcache.h b/zenserver/upstream/upstreamcache.h index 681d8e96f..e5c3521b9 100644 --- a/zenserver/upstream/upstreamcache.h +++ b/zenserver/upstream/upstreamcache.h @@ -2,11 +2,10 @@ #pragma once -#include "cache/cachekey.h" - #include #include #include +#include #include #include diff --git a/zenserver/zenserver.vcxproj b/zenserver/zenserver.vcxproj index d90dd009a..d954d3f8d 100644 --- a/zenserver/zenserver.vcxproj +++ b/zenserver/zenserver.vcxproj @@ -105,7 +105,6 @@ - @@ -132,7 +131,6 @@ - diff --git a/zenserver/zenserver.vcxproj.filters b/zenserver/zenserver.vcxproj.filters index ae5411afb..04c6267ba 100644 --- a/zenserver/zenserver.vcxproj.filters +++ b/zenserver/zenserver.vcxproj.filters @@ -41,9 +41,6 @@ - - cache - @@ -79,9 +76,6 @@ - - cache - -- cgit v1.2.3 From 58bec60cd39b697d5cd3ab6c757a92e528a638fc Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Fri, 12 Nov 2021 14:36:34 +0100 Subject: Fixed bug when cloning CbObject. --- zenserver/cache/structuredcache.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'zenserver') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 1b6406562..d3da98620 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -919,7 +919,8 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Req ToString(HttpContentType::kCbObject)); ZEN_ASSERT(Params.KeyIndex < CacheValues.size()); - CacheValues[Params.KeyIndex] = IoBufferBuilder::MakeCloneFromMemory(Params.Record.GetView()); + + CacheValues[Params.KeyIndex] = CbObject::Clone(Params.Record).GetBuffer().AsIoBuffer(); m_CacheStats.HitCount++; m_CacheStats.UpstreamHitCount++; } -- cgit v1.2.3 From 22dcab1d6abeaf1f39581b1f45619aadf81d14cf Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Fri, 12 Nov 2021 16:42:46 +0100 Subject: Relax constraint on partial cache records. --- zenserver/cache/structuredcache.cpp | 132 ++++++++++++++++-------------------- 1 file changed, 60 insertions(+), 72 deletions(-) (limited to 'zenserver') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index d3da98620..ec27265a7 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -454,52 +454,39 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request CbObjectView CacheRecord(Body.Data()); std::vector ValidAttachments; - uint32_t AttachmentCount = 0; + int32_t TotalCount = 0; - CacheRecord.IterateAttachments([this, &AttachmentCount, &ValidAttachments](CbFieldView AttachmentHash) { + CacheRecord.IterateAttachments([this, &TotalCount, &ValidAttachments](CbFieldView AttachmentHash) { const IoHash Hash = AttachmentHash.AsHash(); if (m_CidStore.ContainsChunk(Hash)) { ValidAttachments.emplace_back(Hash); } - AttachmentCount++; + TotalCount++; }); - const uint32_t ValidCount = static_cast(ValidAttachments.size()); - const bool ValidCacheRecord = ValidCount == AttachmentCount; + const bool IsPartialCacheRecord = TotalCount != static_cast(ValidAttachments.size()); - if (ValidCacheRecord) - { - ZEN_DEBUG("PUT - '{}/{}' {} '{}', {} attachments", - Ref.BucketSegment, - Ref.HashKey, - NiceBytes(Body.Size()), - ToString(ContentType), - ValidCount); - - m_CacheStore.Put(Ref.BucketSegment, Ref.HashKey, {.Value = Body}); + ZEN_DEBUG("PUT - '{}/{}' {} '{}', {} attachments {}", + Ref.BucketSegment, + Ref.HashKey, + NiceBytes(Body.Size()), + ToString(ContentType), + ValidAttachments.size(), + IsPartialCacheRecord ? "(PARTIAL)"sv : ""sv); - if (StoreUpstream) - { - ZEN_ASSERT(m_UpstreamCache); - auto Result = m_UpstreamCache->EnqueueUpstream({.Type = ZenContentType::kCbObject, - .CacheKey = {Ref.BucketSegment, Ref.HashKey}, - .PayloadIds = std::move(ValidAttachments)}); - } + Body.SetContentType(ZenContentType::kCbObject); + m_CacheStore.Put(Ref.BucketSegment, Ref.HashKey, {.Value = Body}); - Request.WriteResponse(HttpResponseCode::Created); - } - else + if (StoreUpstream && !IsPartialCacheRecord) { - ZEN_WARN("PUT - '{}/{}' '{}' FAILED, found {}/{} attachments", - Ref.BucketSegment, - Ref.HashKey, - ToString(ContentType), - ValidCount, - AttachmentCount); - - Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Missing attachments"sv); + ZEN_ASSERT(m_UpstreamCache); + auto Result = m_UpstreamCache->EnqueueUpstream({.Type = ZenContentType::kCbObject, + .CacheKey = {Ref.BucketSegment, Ref.HashKey}, + .PayloadIds = std::move(ValidAttachments)}); } + + Request.WriteResponse(HttpResponseCode::Created); } else if (ContentType == HttpContentType::kCbPackage) { @@ -511,62 +498,63 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid package"sv); } - CbObject CacheRecord = Package.GetObject(); - - std::span Attachments = Package.GetAttachments(); - std::vector ValidAttachments; - int32_t NewAttachmentCount = 0; + CbObject CacheRecord = Package.GetObject(); + std::vector ValidAttachments; + int32_t TotalCount = 0; + int32_t NewCount = 0; + int32_t InvalidCount = 0; - ValidAttachments.reserve(Attachments.size()); + ValidAttachments.reserve(Package.GetAttachments().size()); - CacheRecord.IterateAttachments([this, &Ref, &Package, &ValidAttachments, &NewAttachmentCount](CbFieldView AttachmentHash) { - if (const CbAttachment* Attachment = Package.FindAttachment(AttachmentHash.AsHash())) - { - if (Attachment->IsCompressedBinary()) + CacheRecord.IterateAttachments( + [this, &Ref, &Package, &ValidAttachments, &TotalCount, &NewCount, &InvalidCount](CbFieldView HashView) { + const IoHash Hash = HashView.AsHash(); + if (const CbAttachment* Attachment = Package.FindAttachment(Hash)) { - CompressedBuffer Chunk = Attachment->AsCompressedBinary(); - CidStore::InsertResult InsertResult = m_CidStore.AddChunk(Chunk); + if (Attachment->IsCompressedBinary()) + { + CompressedBuffer Chunk = Attachment->AsCompressedBinary(); + CidStore::InsertResult InsertResult = m_CidStore.AddChunk(Chunk); - ValidAttachments.emplace_back(InsertResult.DecompressedId); + ValidAttachments.emplace_back(InsertResult.DecompressedId); - if (InsertResult.New) + if (InsertResult.New) + { + NewCount++; + } + } + else { - NewAttachmentCount++; + ZEN_WARN("PUT - '{}/{}' '{}' FAILED, attachment '{}' is not compressed", + Ref.BucketSegment, + Ref.HashKey, + ToString(HttpContentType::kCbPackage), + Hash); + InvalidCount++; } } - else + else if (m_CidStore.ContainsChunk(Hash)) { - ZEN_WARN("PUT - '{}/{}' '{}' FAILED, attachment '{}' is not compressed", - Ref.BucketSegment, - Ref.HashKey, - ToString(HttpContentType::kCbPackage), - AttachmentHash.AsHash()); + ValidAttachments.emplace_back(Hash); } - } - else - { - ZEN_WARN("PUT - '{}/{}' '{}' FAILED, missing attachment '{}'", - Ref.BucketSegment, - Ref.HashKey, - ToString(HttpContentType::kCbPackage), - AttachmentHash.AsHash()); - } - }); + TotalCount++; + }); - const bool IsPartialCacheRecord = ValidAttachments.size() != Attachments.size(); + if (InvalidCount > 0) + { + return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid attachment(s)"sv); + } - // if (!AttachmentsValid) - //{ - // return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid attachments"sv); - //} + const bool IsPartialCacheRecord = TotalCount != static_cast(ValidAttachments.size()); - ZEN_DEBUG("PUT - '{}/{}' {} '{}', {}/{} new attachments", + ZEN_DEBUG("PUT - '{}/{}' {} '{}', {}/{} new attachments {}", Ref.BucketSegment, Ref.HashKey, NiceBytes(Body.GetSize()), ToString(ContentType), - NewAttachmentCount, - Attachments.size()); + NewCount, + TotalCount, + IsPartialCacheRecord ? "(PARTIAL)"sv : ""sv); IoBuffer CacheRecordValue = CacheRecord.GetBuffer().AsIoBuffer(); CacheRecordValue.SetContentType(ZenContentType::kCbObject); -- cgit v1.2.3 From 7e86593aedaba74ba626d197fdffa308aa0116cf Mon Sep 17 00:00:00 2001 From: Joe Kirchoff Date: Fri, 12 Nov 2021 16:48:44 -0800 Subject: Remote Apply: Get Expire timeout from worker --- zenserver/upstream/upstreamapply.cpp | 9 +++++---- zenserver/upstream/upstreamapply.h | 1 - 2 files changed, 5 insertions(+), 5 deletions(-) (limited to 'zenserver') diff --git a/zenserver/upstream/upstreamapply.cpp b/zenserver/upstream/upstreamapply.cpp index fd24d241c..3f1b0d8f9 100644 --- a/zenserver/upstream/upstreamapply.cpp +++ b/zenserver/upstream/upstreamapply.cpp @@ -1154,8 +1154,9 @@ public: { if (m_RunState.IsRunning) { - const IoHash WorkerId = ApplyRecord.WorkerDescriptor.GetHash(); - const IoHash ActionId = ApplyRecord.Action.GetHash(); + const IoHash WorkerId = ApplyRecord.WorkerDescriptor.GetHash(); + const IoHash ActionId = ApplyRecord.Action.GetHash(); + const uint32_t TimeoutSeconds = ApplyRecord.WorkerDescriptor["timeout"sv].AsInt32(300); { std::scoped_lock Lock(m_ApplyTasksMutex); @@ -1166,8 +1167,8 @@ public: } std::chrono::steady_clock::time_point ExpireTime = - ApplyRecord.ExpireSeconds > 0 ? std::chrono::steady_clock::now() + std::chrono::seconds(ApplyRecord.ExpireSeconds) - : std::chrono::steady_clock::time_point::max(); + TimeoutSeconds > 0 ? std::chrono::steady_clock::now() + std::chrono::seconds(TimeoutSeconds) + : std::chrono::steady_clock::time_point::max(); m_ApplyTasks[WorkerId][ActionId] = {.State = UpstreamApplyState::Queued, .Result{}, .ExpireTime = std::move(ExpireTime)}; } diff --git a/zenserver/upstream/upstreamapply.h b/zenserver/upstream/upstreamapply.h index 0d26e9d06..8f72660c7 100644 --- a/zenserver/upstream/upstreamapply.h +++ b/zenserver/upstream/upstreamapply.h @@ -32,7 +32,6 @@ struct UpstreamApplyRecord { CbObject WorkerDescriptor; CbObject Action; - uint32_t ExpireSeconds{}; }; struct UpstreamApplyOptions -- cgit v1.2.3 From c5cb8971186f376b8296c26d1412d6f44757ac30 Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Sat, 13 Nov 2021 14:33:29 +0100 Subject: Cleanup attachment validation. --- zenserver/cache/structuredcache.cpp | 130 +++++++++++++++++++++++------------- 1 file changed, 82 insertions(+), 48 deletions(-) (limited to 'zenserver') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index ec27265a7..feb8efb2e 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -48,6 +48,14 @@ ParseCachePolicy(const HttpServerRequest::QueryParams& QueryParams) return QueryPolicy | StorePolicy | SkipPolicy; } +struct AttachmentCount +{ + uint32_t New = 0; + uint32_t Valid = 0; + uint32_t Invalid = 0; + uint32_t Total = 0; +}; + ////////////////////////////////////////////////////////////////////////// HttpStructuredCacheService::HttpStructuredCacheService(ZenCacheStore& InCacheStore, @@ -465,20 +473,20 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request TotalCount++; }); - const bool IsPartialCacheRecord = TotalCount != static_cast(ValidAttachments.size()); - - ZEN_DEBUG("PUT - '{}/{}' {} '{}', {} attachments {}", + ZEN_DEBUG("PUT - '{}/{}' {} '{}' attachments '{}/{}' (Valid/Total)", Ref.BucketSegment, Ref.HashKey, NiceBytes(Body.Size()), ToString(ContentType), - ValidAttachments.size(), - IsPartialCacheRecord ? "(PARTIAL)"sv : ""sv); + TotalCount, + ValidAttachments.size()); Body.SetContentType(ZenContentType::kCbObject); m_CacheStore.Put(Ref.BucketSegment, Ref.HashKey, {.Value = Body}); - if (StoreUpstream && !IsPartialCacheRecord) + const bool IsPartialRecord = TotalCount != static_cast(ValidAttachments.size()); + + if (StoreUpstream && !IsPartialRecord) { ZEN_ASSERT(m_UpstreamCache); auto Result = m_UpstreamCache->EnqueueUpstream({.Type = ZenContentType::kCbObject, @@ -499,69 +507,68 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request } CbObject CacheRecord = Package.GetObject(); + AttachmentCount Count; std::vector ValidAttachments; - int32_t TotalCount = 0; - int32_t NewCount = 0; - int32_t InvalidCount = 0; ValidAttachments.reserve(Package.GetAttachments().size()); - CacheRecord.IterateAttachments( - [this, &Ref, &Package, &ValidAttachments, &TotalCount, &NewCount, &InvalidCount](CbFieldView HashView) { - const IoHash Hash = HashView.AsHash(); - if (const CbAttachment* Attachment = Package.FindAttachment(Hash)) + CacheRecord.IterateAttachments([this, &Ref, &Package, &ValidAttachments, &Count](CbFieldView HashView) { + const IoHash Hash = HashView.AsHash(); + if (const CbAttachment* Attachment = Package.FindAttachment(Hash)) + { + if (Attachment->IsCompressedBinary()) { - if (Attachment->IsCompressedBinary()) - { - CompressedBuffer Chunk = Attachment->AsCompressedBinary(); - CidStore::InsertResult InsertResult = m_CidStore.AddChunk(Chunk); + CompressedBuffer Chunk = Attachment->AsCompressedBinary(); + CidStore::InsertResult InsertResult = m_CidStore.AddChunk(Chunk); - ValidAttachments.emplace_back(InsertResult.DecompressedId); + ValidAttachments.emplace_back(InsertResult.DecompressedId); - if (InsertResult.New) - { - NewCount++; - } - } - else + if (InsertResult.New) { - ZEN_WARN("PUT - '{}/{}' '{}' FAILED, attachment '{}' is not compressed", - Ref.BucketSegment, - Ref.HashKey, - ToString(HttpContentType::kCbPackage), - Hash); - InvalidCount++; + Count.New++; } + Count.Valid++; } - else if (m_CidStore.ContainsChunk(Hash)) + else { - ValidAttachments.emplace_back(Hash); + ZEN_WARN("PUT - '{}/{}' '{}' FAILED, attachment '{}' is not compressed", + Ref.BucketSegment, + Ref.HashKey, + ToString(HttpContentType::kCbPackage), + Hash); + Count.Invalid++; } - TotalCount++; - }); + } + else if (m_CidStore.ContainsChunk(Hash)) + { + ValidAttachments.emplace_back(Hash); + Count.Valid++; + } + Count.Total++; + }); - if (InvalidCount > 0) + if (Count.Invalid > 0) { return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid attachment(s)"sv); } - const bool IsPartialCacheRecord = TotalCount != static_cast(ValidAttachments.size()); - - ZEN_DEBUG("PUT - '{}/{}' {} '{}', {}/{} new attachments {}", + ZEN_DEBUG("PUT - '{}/{}' {} '{}', attachments '{}/{}/{}' (New/Valid/Total)", Ref.BucketSegment, Ref.HashKey, NiceBytes(Body.GetSize()), ToString(ContentType), - NewCount, - TotalCount, - IsPartialCacheRecord ? "(PARTIAL)"sv : ""sv); + Count.New, + Count.Valid, + Count.Total); IoBuffer CacheRecordValue = CacheRecord.GetBuffer().AsIoBuffer(); CacheRecordValue.SetContentType(ZenContentType::kCbObject); m_CacheStore.Put(Ref.BucketSegment, Ref.HashKey, {.Value = CacheRecord.GetBuffer().AsIoBuffer()}); - if (StoreUpstream && !IsPartialCacheRecord) + const bool IsPartialRecord = Count.Valid != Count.Total; + + if (StoreUpstream && !IsPartialRecord) { ZEN_ASSERT(m_UpstreamCache); auto Result = m_UpstreamCache->EnqueueUpstream({.Type = ZenContentType::kCbPackage, @@ -885,30 +892,57 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Req [this, &CacheKeys, &CacheValues, &RpcResponse, SkipAttachments](CacheRecordGetCompleteParams&& Params) { if (Params.Record) { - Params.Record.IterateAttachments([&](CbFieldView AttachmentHash) { - if (const CbAttachment* Attachment = Params.Package.FindAttachment(AttachmentHash.AsHash())) + AttachmentCount Count; + Params.Record.IterateAttachments([this, &RpcResponse, SkipAttachments, &Params, &Count](CbFieldView HashView) { + if (const CbAttachment* Attachment = Params.Package.FindAttachment(HashView.AsHash())) { if (CompressedBuffer Compressed = Attachment->AsCompressedBinary()) { - m_CidStore.AddChunk(Compressed); + auto InsertResult = m_CidStore.AddChunk(Compressed); + if (InsertResult.New) + { + Count.New++; + } + Count.Valid++; if (!SkipAttachments) { RpcResponse.AddAttachment(CbAttachment(Compressed)); } } + else + { + ZEN_DEBUG("Uncompressed payload '{}' from upstream cache record '{}/{}'", + HashView.AsHash(), + Params.CacheKey.Bucket, + Params.CacheKey.Hash); + Count.Invalid++; + } } + else if (m_CidStore.ContainsChunk(HashView.AsHash())) + { + Count.Valid++; + } + Count.Total++; }); - ZEN_DEBUG("HIT - '{}/{}' {} '{}' (UPSTREAM)", + ZEN_DEBUG("HIT - '{}/{}' {} '{}' attachments '{}/{}/{}' (New/Valid/Total) (UPSTREAM)", Params.CacheKey.Bucket, Params.CacheKey.Hash, NiceBytes(Params.Record.GetView().GetSize()), - ToString(HttpContentType::kCbObject)); + ToString(HttpContentType::kCbPackage), + Count.New, + Count.Valid, + Count.Total); ZEN_ASSERT(Params.KeyIndex < CacheValues.size()); - CacheValues[Params.KeyIndex] = CbObject::Clone(Params.Record).GetBuffer().AsIoBuffer(); + IoBuffer CacheValue = CbObject::Clone(Params.Record).GetBuffer().AsIoBuffer(); + CacheValue.SetContentType(ZenContentType::kCbObject); + + CacheValues[Params.KeyIndex] = CacheValue; + m_CacheStore.Put(Params.CacheKey.Bucket, Params.CacheKey.Hash, {.Value = CacheValue}); + m_CacheStats.HitCount++; m_CacheStats.UpstreamHitCount++; } -- cgit v1.2.3 From fc277d6dfb301c6e65c949b5cda81020aad9a22a Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Sat, 13 Nov 2021 14:39:50 +0100 Subject: Format fix. --- zenserver/upstream/upstreamapply.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'zenserver') diff --git a/zenserver/upstream/upstreamapply.h b/zenserver/upstream/upstreamapply.h index 8f72660c7..98f193c02 100644 --- a/zenserver/upstream/upstreamapply.h +++ b/zenserver/upstream/upstreamapply.h @@ -89,7 +89,7 @@ struct UpstreamApplyStatus std::chrono::steady_clock::time_point ExpireTime{}; }; -using UpstreamApplyTasks = std::unordered_map>; +using UpstreamApplyTasks = std::unordered_map>; struct UpstreamEndpointHealth { -- cgit v1.2.3 From 033fd189b876197e263e669c81703bdb550b3ec0 Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Sat, 13 Nov 2021 15:33:22 +0100 Subject: Fixed bug in upstream jupiter endpoint. --- zenserver/upstream/upstreamcache.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'zenserver') diff --git a/zenserver/upstream/upstreamcache.cpp b/zenserver/upstream/upstreamcache.cpp index 174c3d17a..085932685 100644 --- a/zenserver/upstream/upstreamcache.cpp +++ b/zenserver/upstream/upstreamcache.cpp @@ -158,7 +158,7 @@ namespace detail { { const CacheKey& CacheKey = CacheKeys[Index]; CbPackage Package; - CbObjectView Record; + CbObject Record; if (!Result.Error) { @@ -170,7 +170,7 @@ namespace detail { const CbValidateError ValidationResult = ValidateCompactBinary(RefResult.Response, CbValidateMode::All); if (ValidationResult == CbValidateError::None) { - Record = CbObjectView(RefResult.Response.GetData()); + Record = LoadCompactBinaryObject(RefResult.Response); Record.IterateAttachments([this, &Session, &Result, &Package](CbFieldView AttachmentHash) { CloudCacheResult BlobResult = Session.GetCompressedBlob(AttachmentHash.AsHash()); AppendResult(BlobResult, Result); -- cgit v1.2.3 From d65be761d6c0b7ef74e854862514de55db6a2722 Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Sun, 14 Nov 2021 08:51:29 +0100 Subject: Parse chunk ID from chunk request. --- zenserver/cache/structuredcache.cpp | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) (limited to 'zenserver') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index feb8efb2e..79d370740 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -1000,7 +1000,7 @@ HttpStructuredCacheService::HandleRpcGetCachePayloads(zen::HttpServerRequest& Re CbObjectView RequestObject = RequestView.AsObjectView(); CbObjectView KeyObject = RequestObject["Key"sv].AsObjectView(); const CacheKey Key = CacheKey::Create(KeyObject["Bucket"sv].AsString(), KeyObject["Hash"sv].AsHash()); - const IoHash ChunkId = IoHash::Zero; + const IoHash ChunkId = RequestObject["ChunkId"sv].AsHash(); const Oid PayloadId = RequestObject["PayloadId"sv].AsObjectId(); const uint64_t RawOffset = RequestObject["RawoffSet"sv].AsUInt64(); const uint64_t RawSize = RequestObject["RawSize"sv].AsUInt64(); @@ -1016,12 +1016,13 @@ HttpStructuredCacheService::HandleRpcGetCachePayloads(zen::HttpServerRequest& Re Chunks.resize(ChunkRequests.size()); - // Try to find the uncompressed raw hash from the payload ID. + // Unreal uses a 12 byte ID to address cache record payloads. When the uncompressed hash (ChunkId) + // is missing, load the cache record and try to find the raw hash from the payload ID. { const auto GetChunkIdFromPayloadId = [](CbObjectView Record, const Oid& PayloadId) -> IoHash { - if (CbObjectView ValueObject = Record["Value"].AsObjectView()) + if (CbObjectView ValueObject = Record["Value"sv].AsObjectView()) { - const Oid Id = ValueObject["Id"].AsObjectId(); + const Oid Id = ValueObject["Id"sv].AsObjectId(); if (Id == PayloadId) { return ValueObject["RawHash"sv].AsHash(); @@ -1031,7 +1032,7 @@ HttpStructuredCacheService::HandleRpcGetCachePayloads(zen::HttpServerRequest& Re for (CbFieldView AttachmentView : Record["Attachments"sv]) { CbObjectView AttachmentObject = AttachmentView.AsObjectView(); - const Oid Id = AttachmentObject["Id"].AsObjectId(); + const Oid Id = AttachmentObject["Id"sv].AsObjectId(); if (Id == PayloadId) { @@ -1049,6 +1050,11 @@ HttpStructuredCacheService::HandleRpcGetCachePayloads(zen::HttpServerRequest& Re for (CacheChunkRequest& ChunkRequest : ChunkRequests) { + if (ChunkRequest.ChunkId != IoHash::Zero) + { + continue; + } + if (ChunkRequest.Key != CurrentKey) { CurrentKey = ChunkRequest.Key; -- cgit v1.2.3 From bfa621a2d3a3e45879728577b0dc3509bbb26ee3 Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Sun, 14 Nov 2021 08:53:48 +0100 Subject: Fixed typo in object key. --- zenserver/cache/structuredcache.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'zenserver') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 79d370740..41a3e807b 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -1002,7 +1002,7 @@ HttpStructuredCacheService::HandleRpcGetCachePayloads(zen::HttpServerRequest& Re const CacheKey Key = CacheKey::Create(KeyObject["Bucket"sv].AsString(), KeyObject["Hash"sv].AsHash()); const IoHash ChunkId = RequestObject["ChunkId"sv].AsHash(); const Oid PayloadId = RequestObject["PayloadId"sv].AsObjectId(); - const uint64_t RawOffset = RequestObject["RawoffSet"sv].AsUInt64(); + const uint64_t RawOffset = RequestObject["RawOffset"sv].AsUInt64(); const uint64_t RawSize = RequestObject["RawSize"sv].AsUInt64(); const uint32_t ChunkPolicy = RequestObject["Policy"sv].AsUInt32(); -- cgit v1.2.3 From 6732450dfc7d488030cea1a1c6d0b28241534db4 Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Sun, 14 Nov 2021 15:05:58 +0100 Subject: Removed sorting of chunk requests. --- zenserver/cache/structuredcache.cpp | 2 -- 1 file changed, 2 deletions(-) (limited to 'zenserver') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 41a3e807b..eb31ce5e6 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -1046,8 +1046,6 @@ HttpStructuredCacheService::HandleRpcGetCachePayloads(zen::HttpServerRequest& Re CacheKey CurrentKey = CacheKey::Empty; IoBuffer CurrentRecordBuffer; - std::stable_sort(ChunkRequests.begin(), ChunkRequests.end()); - for (CacheChunkRequest& ChunkRequest : ChunkRequests) { if (ChunkRequest.ChunkId != IoHash::Zero) -- cgit v1.2.3 From 6fcd94a6dd98d2643251edf29cf078e6d8641403 Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Mon, 15 Nov 2021 08:20:10 +0100 Subject: Updated cache policy according to UE. --- zenserver/cache/structuredcache.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'zenserver') diff --git a/zenserver/cache/structuredcache.h b/zenserver/cache/structuredcache.h index 3d9914606..51073d05d 100644 --- a/zenserver/cache/structuredcache.h +++ b/zenserver/cache/structuredcache.h @@ -20,7 +20,7 @@ class CasStore; class CidStore; class UpstreamCache; class ZenCacheStore; -enum class CachePolicy : uint8_t; +enum class CachePolicy : uint32_t; /** * Structured cache service. Imposes constraints on keys, supports blobs and -- cgit v1.2.3 From a9298af536e76ce922411f94a6cb4683e5da09ad Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Mon, 15 Nov 2021 09:47:03 +0100 Subject: Handle 'partial on error' cache policy. --- zenserver/cache/structuredcache.cpp | 50 ++++++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 15 deletions(-) (limited to 'zenserver') diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index eb31ce5e6..726bd7cdb 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -473,7 +473,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request TotalCount++; }); - ZEN_DEBUG("PUT - '{}/{}' {} '{}' attachments '{}/{}' (Valid/Total)", + ZEN_DEBUG("PUT - '{}/{}' {} '{}' attachments '{}/{}' (valid/total)", Ref.BucketSegment, Ref.HashKey, NiceBytes(Body.Size()), @@ -552,7 +552,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid attachment(s)"sv); } - ZEN_DEBUG("PUT - '{}/{}' {} '{}', attachments '{}/{}/{}' (New/Valid/Total)", + ZEN_DEBUG("PUT - '{}/{}' {} '{}', attachments '{}/{}/{}' (new/valid/total)", Ref.BucketSegment, Ref.HashKey, NiceBytes(Body.GetSize()), @@ -831,8 +831,9 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Req CacheRecordPolicy::Load(Params["Policy"sv].AsObjectView(), Policy); - const bool SkipAttachments = (Policy.GetRecordPolicy() & CachePolicy::SkipAttachments) == CachePolicy::SkipAttachments; - const bool QueryRemote = m_UpstreamCache && ((Policy.GetRecordPolicy() & CachePolicy::QueryRemote) == CachePolicy::QueryRemote); + const bool PartialOnError = Policy.HasRecordPolicy(CachePolicy::PartialOnError); + const bool SkipAttachments = Policy.HasRecordPolicy(CachePolicy::SkipAttachments); + const bool QueryRemote = Policy.HasRecordPolicy(CachePolicy::QueryRemote) && m_UpstreamCache; for (CbFieldView KeyView : Params["CacheKeys"sv]) { @@ -850,27 +851,36 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Req for (size_t KeyIndex = 0; const CacheKey& Key : CacheKeys) { ZenCacheValue CacheValue; + uint32_t MissingCount = 0; + if (m_CacheStore.Get(Key.Bucket, Key.Hash, CacheValue)) { CbObjectView CacheRecord(CacheValue.Value.Data()); if (!SkipAttachments) { - CacheRecord.IterateAttachments([this, &RpcResponse](CbFieldView AttachmentHash) { + CacheRecord.IterateAttachments([this, &MissingCount, &RpcResponse](CbFieldView AttachmentHash) { if (IoBuffer Chunk = m_CidStore.FindChunkByCid(AttachmentHash.AsHash())) { RpcResponse.AddAttachment(CbAttachment(CompressedBuffer::FromCompressed(SharedBuffer(Chunk)))); } + else + { + MissingCount++; + } }); } + } + if (CacheValue.Value && (MissingCount == 0 || PartialOnError)) + { ZEN_DEBUG("HIT - '{}/{}' {} '{}' (LOCAL)", Key.Bucket, Key.Hash, NiceBytes(CacheValue.Value.Size()), ToString(CacheValue.Value.GetContentType())); - CacheValues[KeyIndex] = CacheValue.Value; + CacheValues[KeyIndex] = std::move(CacheValue.Value); m_CacheStats.HitCount++; } else if (QueryRemote) @@ -879,7 +889,7 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Req } else { - ZEN_DEBUG("MISS - '{}/{}'", Key.Bucket, Key.Hash); + ZEN_DEBUG("MISS - '{}/{}' {}", Key.Bucket, Key.Hash, MissingCount ? "(partial)"sv : ""sv); m_CacheStats.MissCount++; } @@ -889,10 +899,14 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Req if (!UpstreamRequests.empty() && m_UpstreamCache) { const auto OnCacheRecordGetComplete = - [this, &CacheKeys, &CacheValues, &RpcResponse, SkipAttachments](CacheRecordGetCompleteParams&& Params) { + [this, &CacheKeys, &CacheValues, &RpcResponse, PartialOnError, SkipAttachments](CacheRecordGetCompleteParams&& Params) { + ZEN_ASSERT(Params.KeyIndex < CacheValues.size()); + + IoBuffer CacheValue; + AttachmentCount Count; + if (Params.Record) { - AttachmentCount Count; Params.Record.IterateAttachments([this, &RpcResponse, SkipAttachments, &Params, &Count](CbFieldView HashView) { if (const CbAttachment* Attachment = Params.Package.FindAttachment(HashView.AsHash())) { @@ -926,18 +940,23 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Req Count.Total++; }); - ZEN_DEBUG("HIT - '{}/{}' {} '{}' attachments '{}/{}/{}' (New/Valid/Total) (UPSTREAM)", + if ((Count.Valid == Count.Total) || PartialOnError) + { + CacheValue = CbObject::Clone(Params.Record).GetBuffer().AsIoBuffer(); + } + } + + if (CacheValue) + { + ZEN_DEBUG("HIT - '{}/{}' {} '{}' attachments '{}/{}/{}' (new/valid/total) (UPSTREAM)", Params.CacheKey.Bucket, Params.CacheKey.Hash, - NiceBytes(Params.Record.GetView().GetSize()), + NiceBytes(CacheValue.GetSize()), ToString(HttpContentType::kCbPackage), Count.New, Count.Valid, Count.Total); - ZEN_ASSERT(Params.KeyIndex < CacheValues.size()); - - IoBuffer CacheValue = CbObject::Clone(Params.Record).GetBuffer().AsIoBuffer(); CacheValue.SetContentType(ZenContentType::kCbObject); CacheValues[Params.KeyIndex] = CacheValue; @@ -948,7 +967,8 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Req } else { - ZEN_DEBUG("MISS - '{}/{}'", Params.CacheKey.Bucket, Params.CacheKey.Hash); + const bool IsPartial = Count.Valid != Count.Total; + ZEN_DEBUG("MISS - '{}/{}' {}", Params.CacheKey.Bucket, Params.CacheKey.Hash, IsPartial ? "(partial)"sv : ""sv); m_CacheStats.MissCount++; } }; -- cgit v1.2.3 From d1b8635b9b50f7b55b4caea193edaa29b6da0544 Mon Sep 17 00:00:00 2001 From: Joe Kirchoff Date: Mon, 15 Nov 2021 09:48:04 -0800 Subject: Horde Apply: Pass through Currently only Windows is supported Add some additional error messaging on failure --- zenserver/upstream/upstreamapply.cpp | 45 ++++++++++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) (limited to 'zenserver') diff --git a/zenserver/upstream/upstreamapply.cpp b/zenserver/upstream/upstreamapply.cpp index 3f1b0d8f9..a9b0e7fd0 100644 --- a/zenserver/upstream/upstreamapply.cpp +++ b/zenserver/upstream/upstreamapply.cpp @@ -279,6 +279,28 @@ namespace detail { Exception = 6, }; + std::string_view ComputeTaskOutcomeToString(const ComputeTaskOutcome Outcome) + { + switch (Outcome) + { + case ComputeTaskOutcome::Success: + return "Success"sv; + case ComputeTaskOutcome::Failed: + return "Failed"sv; + case ComputeTaskOutcome::Cancelled: + return "Cancelled"sv; + case ComputeTaskOutcome::NoResult: + return "NoResult"sv; + case ComputeTaskOutcome::Exipred: + return "Exipred"sv; + case ComputeTaskOutcome::BlobNotFound: + return "BlobNotFound"sv; + case ComputeTaskOutcome::Exception: + return "Exception"sv; + }; + return "Unknown"sv; + } + virtual GetUpstreamApplyUpdatesResult GetUpdates() override { int64_t Bytes{}; @@ -410,8 +432,15 @@ namespace detail { if (Outcome != ComputeTaskOutcome::Success) { + using namespace fmt::literals; const std::string_view Detail = TaskStatus["d"sv].AsString(); - return {.Error{.ErrorCode = -1, .Reason = std::string(Detail)}}; + if (!Detail.empty()) + { + return {.Error{.ErrorCode = -1, + .Reason = "Task {}: {}"_format(ComputeTaskOutcomeToString(Outcome), std::string(Detail))}}; + } + return { + .Error{.ErrorCode = -1, .Reason = "Task {}"_format(ComputeTaskOutcomeToString(Outcome))}}; } const IoHash TaskId = TaskStatus["h"sv].AsObjectAttachment(); @@ -767,7 +796,19 @@ namespace detail { const IoHash SandboxHash = Sandbox.GetHash(); Data.Objects[SandboxHash] = std::move(Sandbox); - CbObject Requirements = BuildRequirements("OSFamily == 'Windows'"sv, {}, false); + + std::string_view HostPlatform = ApplyRecord.WorkerDescriptor["host"sv].AsString(); + CbObject Requirements; + if (HostPlatform == "Win64"sv) + { + Requirements = BuildRequirements("OSFamily == 'Windows'"sv, {}, false); + } + else + { + Log().warn("process apply upstream FAILED, unsupported Host Platform '{}'", HostPlatform); + return false; + } + const IoHash RequirementsId = Requirements.GetHash(); Data.Objects[RequirementsId] = std::move(Requirements); Data.RequirementsId = RequirementsId; -- cgit v1.2.3 From 20874d3801c4ed142dfd5246a850452f24043512 Mon Sep 17 00:00:00 2001 From: Joe Kirchoff Date: Mon, 15 Nov 2021 15:25:17 -0800 Subject: Use upstream apply with Horde --- zenserver/compute/apply.cpp | 107 ++++++++++++++++++++++++++++++++++- zenserver/compute/apply.h | 20 ++++--- zenserver/upstream/upstreamapply.cpp | 27 ++++----- 3 files changed, 127 insertions(+), 27 deletions(-) (limited to 'zenserver') diff --git a/zenserver/compute/apply.cpp b/zenserver/compute/apply.cpp index 053c262c2..d2ae2febc 100644 --- a/zenserver/compute/apply.cpp +++ b/zenserver/compute/apply.cpp @@ -2,6 +2,8 @@ #include "apply.h" +#include +#include #include #include #include @@ -331,8 +333,20 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, , m_SandboxPath(BaseDir / "scratch") , m_FunctionPath(BaseDir / "func") { + m_UpstreamApply = MakeUpstreamApply({}, m_CasStore, m_CidStore); + + CloudCacheClientOptions Options = {.ServiceUrl = "https://horde.devtools-dev.epicgames.com"sv, + .DdcNamespace = "default"sv, + .BlobStoreNamespace = "default"sv, + .AccessToken = "ServiceAccount 0f8056b30bd0df0959be55fc3338159b6f938456d3474aed0087fb965268d079"sv}; + + auto HordeUpstreamEndpoint = MakeHordeUpstreamEndpoint(Options, m_CasStore, m_CidStore); + m_UpstreamApply->RegisterEndpoint(std::move(HordeUpstreamEndpoint)); + m_UpstreamApply->Initialize(); + m_Router.AddPattern("job", "([[:digit:]]+)"); m_Router.AddPattern("worker", "([[:xdigit:]]{40})"); + m_Router.AddPattern("action", "([[:xdigit:]]{40})"); m_Router.RegisterRoute( "workers/{worker}", @@ -487,6 +501,30 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, }, HttpVerb::kGet | HttpVerb::kPost); + m_Router.RegisterRoute( + "jobs/{worker}/{action}", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + const IoHash WorkerId = IoHash::FromHexString(Req.GetCapture(1)); + const IoHash ActionId = IoHash::FromHexString(Req.GetCapture(2)); + + switch (HttpReq.RequestVerb()) + { + case HttpVerb::kGet: + { + CbPackage Output; + HttpResponseCode ResponseCode = ExecActionUpstreamResult(WorkerId, ActionId, Output); + if (ResponseCode != HttpResponseCode::OK) + { + return HttpReq.WriteResponse(ResponseCode); + } + return HttpReq.WriteResponse(HttpResponseCode::OK, Output); + } + break; + } + }, + HttpVerb::kGet); + m_Router.RegisterRoute( "jobs/{worker}", [this](HttpRouterRequest& Req) { @@ -541,7 +579,7 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, { // We already have everything - CbPackage Output = ExecAction(Worker, RequestObject); + CbObject Output = ExecActionUpstream(Worker, RequestObject); return HttpReq.WriteResponse(HttpResponseCode::OK, Output); } @@ -600,7 +638,7 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, zen::NiceBytes(TotalNewBytes), NewAttachmentCount); - CbPackage Output = ExecAction(Worker, ActionObj); + CbObject Output = ExecActionUpstream(Worker, ActionObj); return HttpReq.WriteResponse(HttpResponseCode::OK, Output); } @@ -843,4 +881,69 @@ HttpFunctionService::ExecAction(const WorkerDesc& Worker, CbObject Action) return OutputPackage; } +CbObject +HttpFunctionService::ExecActionUpstream(const WorkerDesc& Worker, CbObject Action) +{ + const IoHash WorkerId = Worker.Descriptor.GetHash(); + const IoHash ActionId = Action.GetHash(); + + Action.MakeOwned(); + + ZEN_INFO("Action {}/{} being processed...", WorkerId.ToHexString(), ActionId.ToHexString()); + + auto EnqueueResult = m_UpstreamApply->EnqueueUpstream({.WorkerDescriptor = Worker.Descriptor, .Action = std::move(Action)}); + + if (!EnqueueResult.Success) + { + throw std::runtime_error("Error enqueuing upstream task"); + } + + CbObjectWriter Writer; + Writer.AddHash("worker", WorkerId); + Writer.AddHash("action", ActionId); + + return std::move(Writer.Save()); +} + +HttpResponseCode +HttpFunctionService::ExecActionUpstreamResult(const IoHash& WorkerId, const IoHash& ActionId, CbPackage& Package) +{ + using namespace fmt::literals; + auto Status = m_UpstreamApply->GetStatus(WorkerId, ActionId); + if (!Status.Success) + { + // throw std::runtime_error("Action {}/{} not found"_format(WorkerId.ToHexString(), ActionId.ToHexString()).c_str()); + return HttpResponseCode::NotFound; + } + + if (Status.Status.State != UpstreamApplyState::Complete) + { + return HttpResponseCode::Accepted; + } + + GetUpstreamApplyResult& Completed = Status.Status.Result; + if (!Completed.Success || Completed.Error.ErrorCode != 0) + { + ZEN_ERROR("Action {}/{} failed:\n stdout: {} \n stderr: {} \n reason: {}", + WorkerId.ToHexString(), + ActionId.ToHexString(), + Completed.StdOut, + Completed.StdErr, + Completed.Error.Reason); + // throw std::runtime_error( + // "Action {}/{} failed: {}"_format(WorkerId.ToHexString(), ActionId.ToHexString(), Completed.Error.Reason).c_str()); + return HttpResponseCode::NotFound; + } + + ZEN_INFO("Action {}/{} completed with {} attachments ({} compressed, {} uncompressed)", + WorkerId.ToHexString(), + ActionId.ToHexString(), + Completed.OutputPackage.GetAttachments().size(), + NiceBytes(Completed.TotalAttachmentBytes), + NiceBytes(Completed.TotalRawAttachmentBytes)); + + Package = std::move(Completed.OutputPackage); + return HttpResponseCode::OK; +} + } // namespace zen diff --git a/zenserver/compute/apply.h b/zenserver/compute/apply.h index 86b262213..15cda4750 100644 --- a/zenserver/compute/apply.h +++ b/zenserver/compute/apply.h @@ -14,6 +14,7 @@ namespace zen { class CasStore; class CidStore; +class UpstreamApply; /** * Lambda style compute function service @@ -28,14 +29,15 @@ public: virtual void HandleRequest(HttpServerRequest& Request) override; private: - spdlog::logger& Log() { return m_Log; } - spdlog::logger& m_Log; - HttpRequestRouter m_Router; - CasStore& m_CasStore; - CidStore& m_CidStore; - std::filesystem::path m_SandboxPath; - std::filesystem::path m_FunctionPath; - std::atomic m_SandboxCount{0}; + spdlog::logger& Log() { return m_Log; } + spdlog::logger& m_Log; + HttpRequestRouter m_Router; + CasStore& m_CasStore; + CidStore& m_CidStore; + std::filesystem::path m_SandboxPath; + std::filesystem::path m_FunctionPath; + std::atomic m_SandboxCount{0}; + std::unique_ptr m_UpstreamApply; struct WorkerDesc { @@ -44,6 +46,8 @@ private: [[nodiscard]] std::filesystem::path CreateNewSandbox(); [[nodiscard]] CbPackage ExecAction(const WorkerDesc& Worker, CbObject Action); + [[nodiscard]] CbObject ExecActionUpstream(const WorkerDesc& Worker, CbObject Action); + [[nodiscard]] HttpResponseCode ExecActionUpstreamResult(const IoHash& WorkerId, const IoHash& ActionId, CbPackage& Package); RwLock m_WorkerLock; std::unordered_map m_WorkerMap; diff --git a/zenserver/upstream/upstreamapply.cpp b/zenserver/upstream/upstreamapply.cpp index a9b0e7fd0..038353683 100644 --- a/zenserver/upstream/upstreamapply.cpp +++ b/zenserver/upstream/upstreamapply.cpp @@ -439,8 +439,7 @@ namespace detail { return {.Error{.ErrorCode = -1, .Reason = "Task {}: {}"_format(ComputeTaskOutcomeToString(Outcome), std::string(Detail))}}; } - return { - .Error{.ErrorCode = -1, .Reason = "Task {}"_format(ComputeTaskOutcomeToString(Outcome))}}; + return {.Error{.ErrorCode = -1, .Reason = "Task {}"_format(ComputeTaskOutcomeToString(Outcome))}}; } const IoHash TaskId = TaskStatus["h"sv].AsObjectAttachment(); @@ -796,29 +795,23 @@ namespace detail { const IoHash SandboxHash = Sandbox.GetHash(); Data.Objects[SandboxHash] = std::move(Sandbox); - - std::string_view HostPlatform = ApplyRecord.WorkerDescriptor["host"sv].AsString(); - CbObject Requirements; - if (HostPlatform == "Win64"sv) - { - Requirements = BuildRequirements("OSFamily == 'Windows'"sv, {}, false); - } - else { - Log().warn("process apply upstream FAILED, unsupported Host Platform '{}'", HostPlatform); - return false; + using namespace fmt::literals; + std::string_view HostPlatform = ApplyRecord.WorkerDescriptor["host"sv].AsString(); + // TODO: Enable when Horde accepts the UE style Host Platforms (Win64, Linux, Mac) + //CbObject Requirements = BuildRequirements("OSFamily == '{}'"_format(HostPlatform), {}, false); + CbObject Requirements = BuildRequirements("OSFamily == 'Windows'", {}, false); + const IoHash RequirementsId = Requirements.GetHash(); + Data.Objects[RequirementsId] = std::move(Requirements); + Data.RequirementsId = RequirementsId; } - const IoHash RequirementsId = Requirements.GetHash(); - Data.Objects[RequirementsId] = std::move(Requirements); - Data.RequirementsId = RequirementsId; - CbObject Task = BuildTask(ExecutablePath, {"-Build=build.action"}, Environment, {}, SandboxHash, - RequirementsId, + Data.RequirementsId, {"Build.output", "Outputs"}); const IoHash TaskId = Task.GetHash(); -- cgit v1.2.3 From 1fda861670f67b3220fc661e7975f160f99e6aed Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Tue, 16 Nov 2021 21:07:17 +0100 Subject: Added upstream connect/transfer timeout options. --- zenserver/config.cpp | 16 +++++++++++++++- zenserver/config.h | 8 +++++--- zenserver/upstream/upstreamcache.cpp | 21 ++++++++++++++------- zenserver/upstream/upstreamcache.h | 3 ++- zenserver/upstream/zen.cpp | 34 ++++++++++++++++++++++++++-------- zenserver/upstream/zen.h | 18 +++++++++++++++--- zenserver/zenserver.cpp | 5 ++++- 7 files changed, 81 insertions(+), 24 deletions(-) (limited to 'zenserver') diff --git a/zenserver/config.cpp b/zenserver/config.cpp index f512f8015..3e85daa9e 100644 --- a/zenserver/config.cpp +++ b/zenserver/config.cpp @@ -243,7 +243,7 @@ ParseGlobalCliOptions(int argc, char* argv[], ZenServerOptions& GlobalOptions, Z "", "upstream-thread-count", "Number of threads used for upstream procsssing", - cxxopts::value(ServiceConfig.UpstreamCacheConfig.UpstreamThreadCount)->default_value("4"), + cxxopts::value(ServiceConfig.UpstreamCacheConfig.UpstreamThreadCount)->default_value("4"), ""); options.add_option("cache", @@ -253,6 +253,20 @@ ParseGlobalCliOptions(int argc, char* argv[], ZenServerOptions& GlobalOptions, Z cxxopts::value(ServiceConfig.UpstreamCacheConfig.StatsEnabled)->default_value("false"), ""); + options.add_option("cache", + "", + "upstream-connect-timeout-ms", + "Connect timeout in millisecond(s). Default 5000 ms.", + cxxopts::value(ServiceConfig.UpstreamCacheConfig.ConnectTimeoutMilliseconds)->default_value("5000"), + ""); + + options.add_option("cache", + "", + "upstream-timeout-ms", + "Timeout in millisecond(s). Default 0 ms", + cxxopts::value(ServiceConfig.UpstreamCacheConfig.TimeoutMilliseconds)->default_value("0"), + ""); + try { auto result = options.parse(argc, argv); diff --git a/zenserver/config.h b/zenserver/config.h index 72a4f31bb..36156a570 100644 --- a/zenserver/config.h +++ b/zenserver/config.h @@ -57,9 +57,11 @@ struct ZenUpstreamCacheConfig { ZenUpstreamJupiterConfig JupiterConfig; ZenUpstreamZenConfig ZenConfig; - int UpstreamThreadCount = 4; - UpstreamCachePolicy CachePolicy = UpstreamCachePolicy::ReadWrite; - bool StatsEnabled = false; + int32_t UpstreamThreadCount = 4; + int32_t ConnectTimeoutMilliseconds = 5000; + int32_t TimeoutMilliseconds = 0; + UpstreamCachePolicy CachePolicy = UpstreamCachePolicy::ReadWrite; + bool StatsEnabled = false; }; struct ZenServiceConfig diff --git a/zenserver/upstream/upstreamcache.cpp b/zenserver/upstream/upstreamcache.cpp index 085932685..2741e3e51 100644 --- a/zenserver/upstream/upstreamcache.cpp +++ b/zenserver/upstream/upstreamcache.cpp @@ -442,9 +442,13 @@ namespace detail { }; public: - ZenUpstreamEndpoint(std::span Urls) : m_Log(zen::logging::Get("upstream")), m_DisplayName("ZEN") + ZenUpstreamEndpoint(const ZenClientOptions& Options) + : m_Log(zen::logging::Get("upstream")) + , m_DisplayName("ZEN") + , m_ConnectTimeout(Options.ConnectTimeout) + , m_Timeout(Options.Timeout) { - for (const auto& Url : Urls) + for (const auto& Url : Options.Urls) { m_Endpoints.push_back({.Url = Url}); } @@ -461,7 +465,7 @@ namespace detail { { m_ServiceUrl = Ep.Url; m_DisplayName = "ZEN - {}"_format(m_ServiceUrl); - m_Client = new ZenStructuredCacheClient(m_ServiceUrl); + m_Client = new ZenStructuredCacheClient({.Url = m_ServiceUrl, .ConnectTimeout = m_ConnectTimeout, .Timeout = m_Timeout}); m_HealthOk = true; return {.Ok = true}; @@ -486,7 +490,8 @@ namespace detail { { m_ServiceUrl = Ep.Url; m_DisplayName = "ZEN - {}"_format(m_ServiceUrl); - m_Client = new ZenStructuredCacheClient(m_ServiceUrl); + m_Client = + new ZenStructuredCacheClient({.Url = m_ServiceUrl, .ConnectTimeout = m_ConnectTimeout, .Timeout = m_Timeout}); m_HealthOk = true; return {.Ok = true}; @@ -825,7 +830,7 @@ namespace detail { { for (ZenEndpoint& Ep : m_Endpoints) { - ZenStructuredCacheClient Client(Ep.Url); + ZenStructuredCacheClient Client({.Url = Ep.Url, .ConnectTimeout = std::chrono::milliseconds(1000)}); ZenStructuredCacheSession Session(Client); const int32_t SampleCount = 2; @@ -858,6 +863,8 @@ namespace detail { std::string m_ServiceUrl; std::vector m_Endpoints; std::string m_DisplayName; + std::chrono::milliseconds m_ConnectTimeout; + std::chrono::milliseconds m_Timeout; RefPtr m_Client; UpstreamEndpointStats m_Stats; std::atomic_bool m_HealthOk{false}; @@ -1347,9 +1354,9 @@ MakeJupiterUpstreamEndpoint(const CloudCacheClientOptions& Options) } std::unique_ptr -MakeZenUpstreamEndpoint(std::span Urls) +MakeZenUpstreamEndpoint(const ZenClientOptions& Options) { - return std::make_unique(Urls); + return std::make_unique(Options); } } // namespace zen diff --git a/zenserver/upstream/upstreamcache.h b/zenserver/upstream/upstreamcache.h index e5c3521b9..c2770a930 100644 --- a/zenserver/upstream/upstreamcache.h +++ b/zenserver/upstream/upstreamcache.h @@ -20,6 +20,7 @@ class CbObjectWriter; class CidStore; class ZenCacheStore; struct CloudCacheClientOptions; +struct ZenClientOptions; struct UpstreamCacheRecord { @@ -174,6 +175,6 @@ std::unique_ptr MakeUpstreamCache(const UpstreamCacheOptions& Opt std::unique_ptr MakeJupiterUpstreamEndpoint(const CloudCacheClientOptions& Options); -std::unique_ptr MakeZenUpstreamEndpoint(std::span Urls); +std::unique_ptr MakeZenUpstreamEndpoint(const ZenClientOptions& Options); } // namespace zen diff --git a/zenserver/upstream/zen.cpp b/zenserver/upstream/zen.cpp index 9ba767098..3e5a42c22 100644 --- a/zenserver/upstream/zen.cpp +++ b/zenserver/upstream/zen.cpp @@ -313,6 +313,15 @@ namespace detail { void Reset() {} + cpr::Session& GetSession() + { + OwnerClient.InitializeSessionState(*this); + return Session; + } + + private: + friend class ZenStructuredCacheClient; + ZenStructuredCacheClient& OwnerClient; cpr::Session Session; }; @@ -321,9 +330,11 @@ namespace detail { ////////////////////////////////////////////////////////////////////////// -ZenStructuredCacheClient::ZenStructuredCacheClient(std::string_view ServiceUrl) +ZenStructuredCacheClient::ZenStructuredCacheClient(const ZenClientOptions& Options) : m_Log(logging::Get(std::string_view("zenclient"))) -, m_ServiceUrl(ServiceUrl) +, m_ServiceUrl(Options.Url) +, m_ConnectTimeout(Options.ConnectTimeout) +, m_Timeout(Options.Timeout) { } @@ -359,6 +370,13 @@ ZenStructuredCacheClient::FreeSessionState(detail::ZenCacheSessionState* State) m_SessionStateCache.push_front(State); } +void +ZenStructuredCacheClient::InitializeSessionState(detail::ZenCacheSessionState& State) +{ + State.Session.SetConnectTimeout(m_ConnectTimeout); + State.Session.SetTimeout(m_Timeout); +} + ////////////////////////////////////////////////////////////////////////// using namespace std::literals; @@ -381,7 +399,7 @@ ZenStructuredCacheSession::CheckHealth() ExtendableStringBuilder<256> Uri; Uri << m_Client.ServiceUrl() << "/health/check"; - cpr::Session& Session = m_SessionState->Session; + cpr::Session& Session = m_SessionState->GetSession(); Session.SetOption(cpr::Url{Uri.c_str()}); cpr::Response Response = Session.Get(); @@ -399,7 +417,7 @@ ZenStructuredCacheSession::GetCacheRecord(std::string_view BucketId, const IoHas ExtendableStringBuilder<256> Uri; Uri << m_Client.ServiceUrl() << "/z$/" << BucketId << "/" << Key.ToHexString(); - cpr::Session& Session = m_SessionState->Session; + cpr::Session& Session = m_SessionState->GetSession(); Session.SetOption(cpr::Url{Uri.c_str()}); Session.SetHeader(cpr::Header{{"Accept", @@ -427,7 +445,7 @@ ZenStructuredCacheSession::GetCachePayload(std::string_view BucketId, const IoHa ExtendableStringBuilder<256> Uri; Uri << m_Client.ServiceUrl() << "/z$/" << BucketId << "/" << Key.ToHexString() << "/" << PayloadId.ToHexString(); - cpr::Session& Session = m_SessionState->Session; + cpr::Session& Session = m_SessionState->GetSession(); Session.SetOption(cpr::Url{Uri.c_str()}); Session.SetHeader(cpr::Header{{"Accept", "application/x-ue-comp"}}); @@ -452,7 +470,7 @@ ZenStructuredCacheSession::PutCacheRecord(std::string_view BucketId, const IoHas ExtendableStringBuilder<256> Uri; Uri << m_Client.ServiceUrl() << "/z$/" << BucketId << "/" << Key.ToHexString(); - cpr::Session& Session = m_SessionState->Session; + cpr::Session& Session = m_SessionState->GetSession(); Session.SetOption(cpr::Url{Uri.c_str()}); Session.SetHeader(cpr::Header{{"Content-Type", @@ -480,7 +498,7 @@ ZenStructuredCacheSession::PutCachePayload(std::string_view BucketId, const IoHa ExtendableStringBuilder<256> Uri; Uri << m_Client.ServiceUrl() << "/z$/" << BucketId << "/" << Key.ToHexString() << "/" << PayloadId.ToHexString(); - cpr::Session& Session = m_SessionState->Session; + cpr::Session& Session = m_SessionState->GetSession(); Session.SetOption(cpr::Url{Uri.c_str()}); Session.SetHeader(cpr::Header{{"Content-Type", "application/x-ue-comp"}}); @@ -508,7 +526,7 @@ ZenStructuredCacheSession::InvokeRpc(const CbObjectView& Request) BinaryWriter Body; Request.CopyTo(Body); - cpr::Session& Session = m_SessionState->Session; + cpr::Session& Session = m_SessionState->GetSession(); Session.SetOption(cpr::Url{Uri.c_str()}); Session.SetHeader(cpr::Header{{"Content-Type", "application/x-ue-cb"}, {"Accept", "application/x-ue-cbpkg"}}); diff --git a/zenserver/upstream/zen.h b/zenserver/upstream/zen.h index 5efe19094..d3df09e06 100644 --- a/zenserver/upstream/zen.h +++ b/zenserver/upstream/zen.h @@ -101,6 +101,14 @@ struct ZenCacheResult bool Success = false; }; +struct ZenClientOptions +{ + std::string_view Url; + std::span Urls; + std::chrono::milliseconds ConnectTimeout{}; + std::chrono::milliseconds Timeout{}; +}; + /** Zen Structured Cache session * * This provides a context in which cache queries can be performed @@ -136,7 +144,7 @@ private: class ZenStructuredCacheClient : public RefCounted { public: - ZenStructuredCacheClient(std::string_view ServiceUrl); + ZenStructuredCacheClient(const ZenClientOptions& Options); ~ZenStructuredCacheClient(); std::string_view ServiceUrl() const { return m_ServiceUrl; } @@ -144,16 +152,20 @@ public: inline spdlog::logger& Log() { return m_Log; } private: - spdlog::logger& m_Log; - std::string m_ServiceUrl; + spdlog::logger& m_Log; + std::string m_ServiceUrl; + std::chrono::milliseconds m_ConnectTimeout; + std::chrono::milliseconds m_Timeout; RwLock m_SessionStateLock; std::list m_SessionStateCache; detail::ZenCacheSessionState* AllocSessionState(); void FreeSessionState(detail::ZenCacheSessionState*); + void InitializeSessionState(detail::ZenCacheSessionState& State); friend class ZenStructuredCacheSession; + friend struct detail::ZenCacheSessionState; }; } // namespace zen diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp index 9a8090fc0..ca3195616 100644 --- a/zenserver/zenserver.cpp +++ b/zenserver/zenserver.cpp @@ -664,7 +664,10 @@ ZenServer::InitializeStructuredCache(ZenServiceConfig& ServiceConfig) if (!ZenUrls.empty()) { - std::unique_ptr ZenEndpoint = zen::MakeZenUpstreamEndpoint(ZenUrls); + std::unique_ptr ZenEndpoint = + zen::MakeZenUpstreamEndpoint({.Urls = ZenUrls, + .ConnectTimeout = std::chrono::milliseconds(UpstreamConfig.ConnectTimeoutMilliseconds), + .Timeout = std::chrono::milliseconds(UpstreamConfig.TimeoutMilliseconds)}); UpstreamCache->RegisterEndpoint(std::move(ZenEndpoint)); } } -- cgit v1.2.3 From b64cdb97a9fca9aac0df85d6f976ed01ed6f00de Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Tue, 16 Nov 2021 21:10:18 +0100 Subject: Format fix. --- zenserver/upstream/upstreamapply.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'zenserver') diff --git a/zenserver/upstream/upstreamapply.cpp b/zenserver/upstream/upstreamapply.cpp index 038353683..3267b408e 100644 --- a/zenserver/upstream/upstreamapply.cpp +++ b/zenserver/upstream/upstreamapply.cpp @@ -797,13 +797,13 @@ namespace detail { { using namespace fmt::literals; - std::string_view HostPlatform = ApplyRecord.WorkerDescriptor["host"sv].AsString(); + std::string_view HostPlatform = ApplyRecord.WorkerDescriptor["host"sv].AsString(); // TODO: Enable when Horde accepts the UE style Host Platforms (Win64, Linux, Mac) - //CbObject Requirements = BuildRequirements("OSFamily == '{}'"_format(HostPlatform), {}, false); - CbObject Requirements = BuildRequirements("OSFamily == 'Windows'", {}, false); - const IoHash RequirementsId = Requirements.GetHash(); - Data.Objects[RequirementsId] = std::move(Requirements); - Data.RequirementsId = RequirementsId; + // CbObject Requirements = BuildRequirements("OSFamily == '{}'"_format(HostPlatform), {}, false); + CbObject Requirements = BuildRequirements("OSFamily == 'Windows'", {}, false); + const IoHash RequirementsId = Requirements.GetHash(); + Data.Objects[RequirementsId] = std::move(Requirements); + Data.RequirementsId = RequirementsId; } CbObject Task = BuildTask(ExecutablePath, -- cgit v1.2.3 From f45fb6c13407e98434db255b36e3cff402387588 Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Wed, 17 Nov 2021 12:21:28 +0100 Subject: Added connect/transfer timeout options for Jupiter client. --- zenserver/upstream/jupiter.cpp | 47 +++++++++++++++++++++--------------- zenserver/upstream/jupiter.h | 42 +++++++++++++++++--------------- zenserver/upstream/upstreamcache.cpp | 4 +-- zenserver/upstream/upstreamcache.h | 4 +-- zenserver/upstream/zen.cpp | 25 +++++++------------ zenserver/upstream/zen.h | 6 ++--- zenserver/zenserver.cpp | 34 +++++++++++++++----------- 7 files changed, 86 insertions(+), 76 deletions(-) (limited to 'zenserver') diff --git a/zenserver/upstream/jupiter.cpp b/zenserver/upstream/jupiter.cpp index 4caa5c8df..9223ea0f4 100644 --- a/zenserver/upstream/jupiter.cpp +++ b/zenserver/upstream/jupiter.cpp @@ -46,13 +46,20 @@ namespace detail { void InvalidateAccessToken() { AccessToken = {}; } - void Reset() + void Reset(std::chrono::milliseconds ConnectTimeout, std::chrono::milliseconds Timeout) { Session.SetBody({}); Session.SetHeader({}); + Session.SetConnectTimeout(ConnectTimeout); + Session.SetTimeout(Timeout); AccessToken = GetAccessToken(); } + cpr::Session& GetSession() { return Session; } + + private: + friend class CloudCacheClient; + CloudCacheClient& OwnerClient; CloudCacheAccessToken AccessToken; cpr::Session Session; @@ -89,7 +96,7 @@ CloudCacheSession::GetDerivedData(std::string_view BucketId, std::string_view Ke ExtendableStringBuilder<256> Uri; Uri << m_CacheClient->ServiceUrl() << "/api/v1/c/ddc/" << m_CacheClient->DdcNamespace() << "/" << BucketId << "/" << Key; - cpr::Session& Session = m_SessionState->Session; + cpr::Session& Session = m_SessionState->GetSession(); Session.SetOption(cpr::Url{Uri.c_str()}); Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}, {"Accept", "application/octet-stream"}}); @@ -133,7 +140,7 @@ CloudCacheSession::GetRef(std::string_view BucketId, const IoHash& Key, ZenConte Uri << m_CacheClient->ServiceUrl() << "/api/v1/refs/" << m_CacheClient->BlobStoreNamespace() << "/" << BucketId << "/" << Key.ToHexString(); - cpr::Session& Session = m_SessionState->Session; + cpr::Session& Session = m_SessionState->GetSession(); Session.SetOption(cpr::Url{Uri.c_str()}); Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}, {"Accept", ContentType}}); @@ -169,7 +176,7 @@ CloudCacheSession::GetBlob(const IoHash& Key) ExtendableStringBuilder<256> Uri; Uri << m_CacheClient->ServiceUrl() << "/api/v1/blobs/" << m_CacheClient->BlobStoreNamespace() << "/" << Key.ToHexString(); - cpr::Session& Session = m_SessionState->Session; + cpr::Session& Session = m_SessionState->GetSession(); Session.SetOption(cpr::Url{Uri.c_str()}); Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}, {"Accept", "application/octet-stream"}}); @@ -206,7 +213,7 @@ CloudCacheSession::GetCompressedBlob(const IoHash& Key) ExtendableStringBuilder<256> Uri; Uri << m_CacheClient->ServiceUrl() << "/api/v1/compressed-blobs/" << m_CacheClient->BlobStoreNamespace() << "/" << Key.ToHexString(); - cpr::Session& Session = m_SessionState->Session; + cpr::Session& Session = m_SessionState->GetSession(); Session.SetOption(cpr::Url{Uri.c_str()}); Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}, {"Accept", "application/x-ue-comp"}}); @@ -242,7 +249,7 @@ CloudCacheSession::GetObject(const IoHash& Key) ExtendableStringBuilder<256> Uri; Uri << m_CacheClient->ServiceUrl() << "/api/v1/objects/" << m_CacheClient->BlobStoreNamespace() << "/" << Key.ToHexString(); - cpr::Session& Session = m_SessionState->Session; + cpr::Session& Session = m_SessionState->GetSession(); Session.SetOption(cpr::Url{Uri.c_str()}); Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}, {"Accept", "application/x-ue-cb"}}); @@ -280,7 +287,7 @@ CloudCacheSession::PutDerivedData(std::string_view BucketId, std::string_view Ke ExtendableStringBuilder<256> Uri; Uri << m_CacheClient->ServiceUrl() << "/api/v1/c/ddc/" << m_CacheClient->DdcNamespace() << "/" << BucketId << "/" << Key; - auto& Session = m_SessionState->Session; + auto& Session = m_SessionState->GetSession(); Session.SetOption(cpr::Url{Uri.c_str()}); Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}, @@ -331,7 +338,7 @@ CloudCacheSession::PutRef(std::string_view BucketId, const IoHash& Key, IoBuffer Uri << m_CacheClient->ServiceUrl() << "/api/v1/refs/" << m_CacheClient->BlobStoreNamespace() << "/" << BucketId << "/" << Key.ToHexString(); - cpr::Session& Session = m_SessionState->Session; + cpr::Session& Session = m_SessionState->GetSession(); Session.SetOption(cpr::Url{Uri.c_str()}); Session.SetOption( @@ -394,7 +401,7 @@ CloudCacheSession::FinalizeRef(std::string_view BucketId, const IoHash& Key, con Uri << m_CacheClient->ServiceUrl() << "/api/v1/refs/" << m_CacheClient->BlobStoreNamespace() << "/" << BucketId << "/" << Key.ToHexString() << "/finalize/" << RefHash.ToHexString(); - cpr::Session& Session = m_SessionState->Session; + cpr::Session& Session = m_SessionState->GetSession(); Session.SetOption(cpr::Url{Uri.c_str()}); Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}, @@ -454,7 +461,7 @@ CloudCacheSession::PutBlob(const IoHash& Key, IoBuffer Blob) ExtendableStringBuilder<256> Uri; Uri << m_CacheClient->ServiceUrl() << "/api/v1/blobs/" << m_CacheClient->BlobStoreNamespace() << "/" << Key.ToHexString(); - cpr::Session& Session = m_SessionState->Session; + cpr::Session& Session = m_SessionState->GetSession(); Session.SetOption(cpr::Url{Uri.c_str()}); Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}, {"Content-Type", "application/octet-stream"}}); @@ -489,7 +496,7 @@ CloudCacheSession::PutCompressedBlob(const IoHash& Key, IoBuffer Blob) ExtendableStringBuilder<256> Uri; Uri << m_CacheClient->ServiceUrl() << "/api/v1/compressed-blobs/" << m_CacheClient->BlobStoreNamespace() << "/" << Key.ToHexString(); - cpr::Session& Session = m_SessionState->Session; + cpr::Session& Session = m_SessionState->GetSession(); Session.SetOption(cpr::Url{Uri.c_str()}); Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}, {"Content-Type", "application/x-ue-comp"}}); @@ -524,7 +531,7 @@ CloudCacheSession::PutObject(const IoHash& Key, IoBuffer Object) ExtendableStringBuilder<256> Uri; Uri << m_CacheClient->ServiceUrl() << "/api/v1/objects/" << m_CacheClient->BlobStoreNamespace() << "/" << Key.ToHexString(); - cpr::Session& Session = m_SessionState->Session; + cpr::Session& Session = m_SessionState->GetSession(); Session.SetOption(cpr::Url{Uri.c_str()}); Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}, {"Content-Type", "application/x-ue-cb"}}); @@ -560,7 +567,7 @@ CloudCacheSession::RefExists(std::string_view BucketId, const IoHash& Key) Uri << m_CacheClient->ServiceUrl() << "/api/v1/refs/" << m_CacheClient->BlobStoreNamespace() << "/" << BucketId << "/" << Key.ToHexString(); - cpr::Session& Session = m_SessionState->Session; + cpr::Session& Session = m_SessionState->GetSession(); Session.SetOption(cpr::Url{Uri.c_str()}); Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}}); @@ -629,7 +636,7 @@ CloudCacheSession::PostComputeTasks(std::string_view ChannelId, IoBuffer TasksDa ExtendableStringBuilder<256> Uri; Uri << m_CacheClient->ServiceUrl() << "/api/v1/compute/" << ChannelId; - auto& Session = m_SessionState->Session; + auto& Session = m_SessionState->GetSession(); Session.SetOption(cpr::Url{Uri.c_str()}); Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}, {"Content-Type", "application/x-ue-cb"}}); @@ -662,7 +669,7 @@ CloudCacheSession::GetComputeUpdates(std::string_view ChannelId, const uint32_t ExtendableStringBuilder<256> Uri; Uri << m_CacheClient->ServiceUrl() << "/api/v1/compute/" << ChannelId << "/updates?wait=" << WaitSeconds; - auto& Session = m_SessionState->Session; + auto& Session = m_SessionState->GetSession(); Session.SetOption(cpr::Url{Uri.c_str()}); Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}, {"Accept", "application/x-ue-cb"}}); @@ -697,7 +704,7 @@ CloudCacheSession::GetObjectTree(const IoHash& Key) ExtendableStringBuilder<256> Uri; Uri << m_CacheClient->ServiceUrl() << "/api/v1/objects/" << m_CacheClient->BlobStoreNamespace() << "/" << Key.ToHexString() << "/tree"; - cpr::Session& Session = m_SessionState->Session; + cpr::Session& Session = m_SessionState->GetSession(); Session.SetOption(cpr::Url{Uri.c_str()}); Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}, {"Accept", "application/octet-stream"}}); @@ -761,7 +768,7 @@ CloudCacheSession::CacheTypeExists(std::string_view TypeId, const IoHash& Key) ExtendableStringBuilder<256> Uri; Uri << m_CacheClient->ServiceUrl() << "/api/v1/" << TypeId << "/" << m_CacheClient->BlobStoreNamespace() << "/" << Key.ToHexString(); - cpr::Session& Session = m_SessionState->Session; + cpr::Session& Session = m_SessionState->GetSession(); Session.SetOption(cpr::Url{Uri.c_str()}); Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}}); @@ -799,7 +806,7 @@ CloudCacheSession::CacheTypeExists(std::string_view TypeId, const std::set Uri; Uri << m_CacheClient->ServiceUrl() << "/api/v1/" << TypeId << "/" << m_CacheClient->BlobStoreNamespace() << "/exists?" << Query; - cpr::Session& Session = m_SessionState->Session; + cpr::Session& Session = m_SessionState->GetSession(); Session.SetOption(cpr::Url{Uri.c_str()}); Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}, {"Accept", "application/x-ue-cb"}}); @@ -853,6 +860,8 @@ CloudCacheClient::CloudCacheClient(const CloudCacheClientOptions& Options) , m_OAuthClientId(Options.OAuthClientId) , m_OAuthSecret(Options.OAuthSecret) , m_AccessToken(Options.AccessToken) +, m_ConnectTimeout(Options.ConnectTimeout) +, m_Timeout(Options.Timeout) { if (!Options.AccessToken.empty()) { @@ -955,7 +964,7 @@ CloudCacheClient::AllocSessionState() State = new detail::CloudCacheSessionState(*this); } - State->Reset(); + State->Reset(m_ConnectTimeout, m_Timeout); return State; } diff --git a/zenserver/upstream/jupiter.h b/zenserver/upstream/jupiter.h index 13d65587e..68c7361e0 100644 --- a/zenserver/upstream/jupiter.h +++ b/zenserver/upstream/jupiter.h @@ -132,14 +132,16 @@ private: struct CloudCacheClientOptions { - std::string_view ServiceUrl; - std::string_view DdcNamespace; - std::string_view BlobStoreNamespace; - std::string_view OAuthProvider; - std::string_view OAuthClientId; - std::string_view OAuthSecret; - std::string_view AccessToken; - bool UseLegacyDdc = false; + std::string_view ServiceUrl; + std::string_view DdcNamespace; + std::string_view BlobStoreNamespace; + std::string_view OAuthProvider; + std::string_view OAuthClientId; + std::string_view OAuthSecret; + std::string_view AccessToken; + std::chrono::milliseconds ConnectTimeout{5000}; + std::chrono::milliseconds Timeout{}; + bool UseLegacyDdc = false; }; /** @@ -160,17 +162,19 @@ public: spdlog::logger& Logger() { return m_Log; } private: - spdlog::logger& m_Log; - std::string m_ServiceUrl; - std::string m_OAuthDomain; - std::string m_OAuthUriPath; - std::string m_OAuthFullUri; - std::string m_DdcNamespace; - std::string m_BlobStoreNamespace; - std::string m_OAuthClientId; - std::string m_OAuthSecret; - std::string m_AccessToken; - bool m_IsValid = false; + spdlog::logger& m_Log; + std::string m_ServiceUrl; + std::string m_OAuthDomain; + std::string m_OAuthUriPath; + std::string m_OAuthFullUri; + std::string m_DdcNamespace; + std::string m_BlobStoreNamespace; + std::string m_OAuthClientId; + std::string m_OAuthSecret; + std::string m_AccessToken; + std::chrono::milliseconds m_ConnectTimeout{}; + std::chrono::milliseconds m_Timeout{}; + bool m_IsValid = false; RwLock m_SessionStateLock; std::list m_SessionStateCache; diff --git a/zenserver/upstream/upstreamcache.cpp b/zenserver/upstream/upstreamcache.cpp index 2741e3e51..0290723f0 100644 --- a/zenserver/upstream/upstreamcache.cpp +++ b/zenserver/upstream/upstreamcache.cpp @@ -442,7 +442,7 @@ namespace detail { }; public: - ZenUpstreamEndpoint(const ZenClientOptions& Options) + ZenUpstreamEndpoint(const ZenStructuredCacheClientOptions& Options) : m_Log(zen::logging::Get("upstream")) , m_DisplayName("ZEN") , m_ConnectTimeout(Options.ConnectTimeout) @@ -1354,7 +1354,7 @@ MakeJupiterUpstreamEndpoint(const CloudCacheClientOptions& Options) } std::unique_ptr -MakeZenUpstreamEndpoint(const ZenClientOptions& Options) +MakeZenUpstreamEndpoint(const ZenStructuredCacheClientOptions& Options) { return std::make_unique(Options); } diff --git a/zenserver/upstream/upstreamcache.h b/zenserver/upstream/upstreamcache.h index c2770a930..fb2550cad 100644 --- a/zenserver/upstream/upstreamcache.h +++ b/zenserver/upstream/upstreamcache.h @@ -20,7 +20,7 @@ class CbObjectWriter; class CidStore; class ZenCacheStore; struct CloudCacheClientOptions; -struct ZenClientOptions; +struct ZenStructuredCacheClientOptions; struct UpstreamCacheRecord { @@ -175,6 +175,6 @@ std::unique_ptr MakeUpstreamCache(const UpstreamCacheOptions& Opt std::unique_ptr MakeJupiterUpstreamEndpoint(const CloudCacheClientOptions& Options); -std::unique_ptr MakeZenUpstreamEndpoint(const ZenClientOptions& Options); +std::unique_ptr MakeZenUpstreamEndpoint(const ZenStructuredCacheClientOptions& Options); } // namespace zen diff --git a/zenserver/upstream/zen.cpp b/zenserver/upstream/zen.cpp index 3e5a42c22..cd7f48334 100644 --- a/zenserver/upstream/zen.cpp +++ b/zenserver/upstream/zen.cpp @@ -311,17 +311,17 @@ namespace detail { ZenCacheSessionState(ZenStructuredCacheClient& Client) : OwnerClient(Client) {} ~ZenCacheSessionState() {} - void Reset() {} - - cpr::Session& GetSession() + void Reset(std::chrono::milliseconds ConnectTimeout, std::chrono::milliseconds Timeout) { - OwnerClient.InitializeSessionState(*this); - return Session; + Session.SetBody({}); + Session.SetHeader({}); + Session.SetConnectTimeout(ConnectTimeout); + Session.SetTimeout(Timeout); } - private: - friend class ZenStructuredCacheClient; + cpr::Session& GetSession() { return Session; } + private: ZenStructuredCacheClient& OwnerClient; cpr::Session Session; }; @@ -330,7 +330,7 @@ namespace detail { ////////////////////////////////////////////////////////////////////////// -ZenStructuredCacheClient::ZenStructuredCacheClient(const ZenClientOptions& Options) +ZenStructuredCacheClient::ZenStructuredCacheClient(const ZenStructuredCacheClientOptions& Options) : m_Log(logging::Get(std::string_view("zenclient"))) , m_ServiceUrl(Options.Url) , m_ConnectTimeout(Options.ConnectTimeout) @@ -358,7 +358,7 @@ ZenStructuredCacheClient::AllocSessionState() State = new detail::ZenCacheSessionState(*this); } - State->Reset(); + State->Reset(m_ConnectTimeout, m_Timeout); return State; } @@ -370,13 +370,6 @@ ZenStructuredCacheClient::FreeSessionState(detail::ZenCacheSessionState* State) m_SessionStateCache.push_front(State); } -void -ZenStructuredCacheClient::InitializeSessionState(detail::ZenCacheSessionState& State) -{ - State.Session.SetConnectTimeout(m_ConnectTimeout); - State.Session.SetTimeout(m_Timeout); -} - ////////////////////////////////////////////////////////////////////////// using namespace std::literals; diff --git a/zenserver/upstream/zen.h b/zenserver/upstream/zen.h index d3df09e06..d549c2fc4 100644 --- a/zenserver/upstream/zen.h +++ b/zenserver/upstream/zen.h @@ -101,7 +101,7 @@ struct ZenCacheResult bool Success = false; }; -struct ZenClientOptions +struct ZenStructuredCacheClientOptions { std::string_view Url; std::span Urls; @@ -144,7 +144,7 @@ private: class ZenStructuredCacheClient : public RefCounted { public: - ZenStructuredCacheClient(const ZenClientOptions& Options); + ZenStructuredCacheClient(const ZenStructuredCacheClientOptions& Options); ~ZenStructuredCacheClient(); std::string_view ServiceUrl() const { return m_ServiceUrl; } @@ -162,10 +162,8 @@ private: detail::ZenCacheSessionState* AllocSessionState(); void FreeSessionState(detail::ZenCacheSessionState*); - void InitializeSessionState(detail::ZenCacheSessionState& State); friend class ZenStructuredCacheSession; - friend struct detail::ZenCacheSessionState; }; } // namespace zen diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp index ca3195616..a330fb558 100644 --- a/zenserver/zenserver.cpp +++ b/zenserver/zenserver.cpp @@ -677,23 +677,29 @@ ZenServer::InitializeStructuredCache(ZenServiceConfig& ServiceConfig) zen::CloudCacheClientOptions Options; if (UpstreamConfig.JupiterConfig.UseProductionSettings) { - Options = zen::CloudCacheClientOptions{.ServiceUrl = "https://jupiter.devtools.epicgames.com"sv, - .DdcNamespace = "ue.ddc"sv, - .BlobStoreNamespace = "ue.ddc"sv, - .OAuthProvider = "https://epicgames.okta.com/oauth2/auso645ojjWVdRI3d0x7/v1/token"sv, - .OAuthClientId = "0oao91lrhqPiAlaGD0x7"sv, - .OAuthSecret = "-GBWjjenhCgOwhxL5yBKNJECVIoDPH0MK4RDuN7d"sv, - .UseLegacyDdc = false}; + Options = + zen::CloudCacheClientOptions{.ServiceUrl = "https://jupiter.devtools.epicgames.com"sv, + .DdcNamespace = "ue.ddc"sv, + .BlobStoreNamespace = "ue.ddc"sv, + .OAuthProvider = "https://epicgames.okta.com/oauth2/auso645ojjWVdRI3d0x7/v1/token"sv, + .OAuthClientId = "0oao91lrhqPiAlaGD0x7"sv, + .OAuthSecret = "-GBWjjenhCgOwhxL5yBKNJECVIoDPH0MK4RDuN7d"sv, + .ConnectTimeout = std::chrono::milliseconds(UpstreamConfig.ConnectTimeoutMilliseconds), + .Timeout = std::chrono::milliseconds(UpstreamConfig.TimeoutMilliseconds), + .UseLegacyDdc = false}; } else if (UpstreamConfig.JupiterConfig.UseDevelopmentSettings) { - Options = zen::CloudCacheClientOptions{.ServiceUrl = "https://jupiter.devtools-dev.epicgames.com"sv, - .DdcNamespace = "ue4.ddc"sv, - .BlobStoreNamespace = "test.ddc"sv, - .OAuthProvider = "https://epicgames.okta.com/oauth2/auso645ojjWVdRI3d0x7/v1/token"sv, - .OAuthClientId = "0oao91lrhqPiAlaGD0x7"sv, - .OAuthSecret = "-GBWjjenhCgOwhxL5yBKNJECVIoDPH0MK4RDuN7d"sv, - .UseLegacyDdc = false}; + Options = + zen::CloudCacheClientOptions{.ServiceUrl = "https://jupiter.devtools-dev.epicgames.com"sv, + .DdcNamespace = "ue4.ddc"sv, + .BlobStoreNamespace = "test.ddc"sv, + .OAuthProvider = "https://epicgames.okta.com/oauth2/auso645ojjWVdRI3d0x7/v1/token"sv, + .OAuthClientId = "0oao91lrhqPiAlaGD0x7"sv, + .OAuthSecret = "-GBWjjenhCgOwhxL5yBKNJECVIoDPH0MK4RDuN7d"sv, + .ConnectTimeout = std::chrono::milliseconds(UpstreamConfig.ConnectTimeoutMilliseconds), + .Timeout = std::chrono::milliseconds(UpstreamConfig.TimeoutMilliseconds), + .UseLegacyDdc = false}; } Options.ServiceUrl = ValueOrDefault(UpstreamConfig.JupiterConfig.Url, Options.ServiceUrl); -- cgit v1.2.3 From 200913d1b0d9716e73bf465b7b42802cc7b508cc Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Wed, 17 Nov 2021 12:35:34 +0100 Subject: Check both success and error code when initializing Jupiter upstream. --- zenserver/upstream/upstreamcache.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'zenserver') diff --git a/zenserver/upstream/upstreamcache.cpp b/zenserver/upstream/upstreamcache.cpp index 0290723f0..0fa497faf 100644 --- a/zenserver/upstream/upstreamcache.cpp +++ b/zenserver/upstream/upstreamcache.cpp @@ -58,7 +58,7 @@ namespace detail { CloudCacheSession Session(m_Client); const CloudCacheResult Result = Session.Authenticate(); - m_HealthOk = Result.ErrorCode == 0; + m_HealthOk = Result.Success && Result.ErrorCode == 0; return {.Reason = std::move(Result.Reason), .Ok = Result.Success}; } -- cgit v1.2.3 From 731e523ab939c8aebf33ae5f7e127b01599bf575 Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Wed, 17 Nov 2021 15:29:58 +0100 Subject: Log upstream HTTP errors as errors. --- zenserver/upstream/upstreamcache.cpp | 68 +++++++++++++++++++++++++++++------- 1 file changed, 55 insertions(+), 13 deletions(-) (limited to 'zenserver') diff --git a/zenserver/upstream/upstreamcache.cpp b/zenserver/upstream/upstreamcache.cpp index 0fa497faf..726ef331d 100644 --- a/zenserver/upstream/upstreamcache.cpp +++ b/zenserver/upstream/upstreamcache.cpp @@ -814,7 +814,10 @@ namespace detail { TotalElapsedSeconds += Result.ElapsedSeconds; } - return {.Bytes = TotalBytes, .ElapsedSeconds = TotalElapsedSeconds, .Success = Result.Success}; + return {.Reason = std::move(Result.Reason), + .Bytes = TotalBytes, + .ElapsedSeconds = TotalElapsedSeconds, + .Success = Result.Success}; } catch (std::exception& e) { @@ -1027,6 +1030,14 @@ public: { return Result; } + + if (Result.Error) + { + ZEN_ERROR("get cache record FAILED, endpoint '{}', reason '{}', error code '{}'", + Endpoint->DisplayName(), + Result.Error.Reason, + Result.Error.ErrorCode); + } } } } @@ -1060,6 +1071,14 @@ public: } }); + if (Result.Error) + { + ZEN_ERROR("get cache record(s) (rpc) FAILED, endpoint '{}', reason '{}', error code '{}'", + Endpoint->DisplayName(), + Result.Error.Reason, + Result.Error.ErrorCode); + } + m_Stats.Add(m_Log, *Endpoint, Result, m_Endpoints); MissingKeys = std::move(Missing); } @@ -1098,6 +1117,14 @@ public: } }); + if (Result.Error) + { + ZEN_ERROR("get cache payloads(s) (rpc) FAILED, endpoint '{}', reason '{}', error code '{}'", + Endpoint->DisplayName(), + Result.Error.Reason, + Result.Error.ErrorCode); + } + m_Stats.Add(m_Log, *Endpoint, Result, m_Endpoints); MissingPayloads = std::move(Missing); } @@ -1125,6 +1152,14 @@ public: { return Result; } + + if (Result.Error) + { + ZEN_ERROR("get cache payload FAILED, endpoint '{}', reason '{}', error code '{}'", + Endpoint->DisplayName(), + Result.Error.Reason, + Result.Error.ErrorCode); + } } } } @@ -1243,10 +1278,10 @@ private: } catch (std::exception& e) { - ZEN_WARN("upload cache record '{}/{}' FAILED, reason '{}'", - CacheRecord.CacheKey.Bucket, - CacheRecord.CacheKey.Hash, - e.what()); + ZEN_ERROR("upload cache record '{}/{}' FAILED, reason '{}'", + CacheRecord.CacheKey.Bucket, + CacheRecord.CacheKey.Hash, + e.what()); } } @@ -1269,20 +1304,27 @@ private: } } - for (auto& Endpoint : m_Endpoints) + try { - if (!Endpoint->IsHealthy()) + for (auto& Endpoint : m_Endpoints) { - if (const UpstreamEndpointHealth Health = Endpoint->CheckHealth(); Health.Ok) + if (!Endpoint->IsHealthy()) { - ZEN_INFO("health check endpoint '{}' OK", Endpoint->DisplayName(), Health.Reason); - } - else - { - ZEN_WARN("health check endpoint '{}' FAILED, reason '{}'", Endpoint->DisplayName(), Health.Reason); + if (const UpstreamEndpointHealth Health = Endpoint->CheckHealth(); Health.Ok) + { + ZEN_INFO("health check endpoint '{}' OK", Endpoint->DisplayName(), Health.Reason); + } + else + { + ZEN_WARN("health check endpoint '{}' FAILED, reason '{}'", Endpoint->DisplayName(), Health.Reason); + } } } } + catch (std::exception& e) + { + ZEN_ERROR("check endpoint(s) health FAILED, reason '{}'", e.what()); + } } } -- cgit v1.2.3 From 1b9d83b3fa3a6cc6e2fa0f1af72050de2ccf4ea9 Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Wed, 17 Nov 2021 20:31:41 +0100 Subject: Added health check and return missing error message. --- zenserver/upstream/upstreamcache.cpp | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) (limited to 'zenserver') diff --git a/zenserver/upstream/upstreamcache.cpp b/zenserver/upstream/upstreamcache.cpp index 726ef331d..065471c07 100644 --- a/zenserver/upstream/upstreamcache.cpp +++ b/zenserver/upstream/upstreamcache.cpp @@ -286,7 +286,12 @@ namespace detail { } } - return {.Bytes = Result.Bytes, .ElapsedSeconds = Result.ElapsedSeconds, .Success = Result.Success}; + m_HealthOk = Result.ErrorCode == 0; + + return {.Reason = std::move(Result.Reason), + .Bytes = Result.Bytes, + .ElapsedSeconds = Result.ElapsedSeconds, + .Success = Result.Success}; } else { @@ -312,6 +317,8 @@ namespace detail { BlobResult = Session.PutCompressedBlob(CacheRecord.PayloadIds[Idx], Payloads[Idx]); } + m_HealthOk = BlobResult.ErrorCode == 0; + if (!BlobResult.Success) { OutReason = "upload payload '{}' FAILED, reason '{}'"_format(PayloadId, BlobResult.Reason); @@ -332,6 +339,8 @@ namespace detail { Session.PutRef(CacheRecord.CacheKey.Bucket, CacheRecord.CacheKey.Hash, RecordValue, ZenContentType::kCbObject); } + m_HealthOk = RefResult.ErrorCode == 0; + if (!RefResult.Success) { return {.Reason = "upload cache record '{}/{}' FAILED, reason '{}'"_format(CacheRecord.CacheKey.Bucket, @@ -351,6 +360,7 @@ namespace detail { const IoHash RefHash = IoHash::HashBuffer(RecordValue); FinalizeRefResult FinalizeResult = Session.FinalizeRef(CacheRecord.CacheKey.Bucket, CacheRecord.CacheKey.Hash, RefHash); + m_HealthOk = FinalizeResult.ErrorCode == 0; if (!FinalizeResult.Success) { @@ -368,6 +378,7 @@ namespace detail { } FinalizeResult = Session.FinalizeRef(CacheRecord.CacheKey.Bucket, CacheRecord.CacheKey.Hash, RefHash); + m_HealthOk = FinalizeResult.ErrorCode == 0; if (!FinalizeResult.Success) { @@ -400,6 +411,7 @@ namespace detail { } catch (std::exception& Err) { + m_HealthOk = false; return {.Reason = std::string(Err.what()), .Success = false}; } } @@ -819,10 +831,10 @@ namespace detail { .ElapsedSeconds = TotalElapsedSeconds, .Success = Result.Success}; } - catch (std::exception& e) + catch (std::exception& Err) { m_HealthOk = false; - return {.Reason = std::string(e.what()), .Success = false}; + return {.Reason = std::string(Err.what()), .Success = false}; } } @@ -1276,12 +1288,12 @@ private: { ProcessCacheRecord(std::move(CacheRecord)); } - catch (std::exception& e) + catch (std::exception& Err) { ZEN_ERROR("upload cache record '{}/{}' FAILED, reason '{}'", CacheRecord.CacheKey.Bucket, CacheRecord.CacheKey.Hash, - e.what()); + Err.what()); } } @@ -1321,9 +1333,9 @@ private: } } } - catch (std::exception& e) + catch (std::exception& Err) { - ZEN_ERROR("check endpoint(s) health FAILED, reason '{}'", e.what()); + ZEN_ERROR("check endpoint(s) health FAILED, reason '{}'", Err.what()); } } } -- cgit v1.2.3 From a3b24501aa2758b0621806086bb6e3ffc83c2b97 Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Wed, 17 Nov 2021 21:06:17 +0100 Subject: Changed upstream DisplayName to UpstreamEndpointInfo with name and url. --- zenserver/upstream/upstreamcache.cpp | 59 +++++++++++++++++++----------------- zenserver/upstream/upstreamcache.h | 10 ++++-- 2 files changed, 39 insertions(+), 30 deletions(-) (limited to 'zenserver') diff --git a/zenserver/upstream/upstreamcache.cpp b/zenserver/upstream/upstreamcache.cpp index 065471c07..e2dc09872 100644 --- a/zenserver/upstream/upstreamcache.cpp +++ b/zenserver/upstream/upstreamcache.cpp @@ -40,13 +40,15 @@ namespace detail { : m_Log(zen::logging::Get("upstream")) , m_UseLegacyDdc(Options.UseLegacyDdc) { - using namespace fmt::literals; - m_DisplayName = "Jupiter - '{}'"_format(Options.ServiceUrl); - m_Client = new CloudCacheClient(Options); + m_Info.Name = "Horde"sv; + m_Info.Url = Options.ServiceUrl; + m_Client = new CloudCacheClient(Options); } virtual ~JupiterUpstreamEndpoint() = default; + virtual const UpstreamEndpointInfo& GetEndpointInfo() const { return m_Info; } + virtual UpstreamEndpointHealth Initialize() override { return CheckHealth(); } virtual bool IsHealthy() const override { return m_HealthOk.load(); } @@ -68,8 +70,6 @@ namespace detail { } } - virtual std::string_view DisplayName() const override { return m_DisplayName; } - virtual GetUpstreamCacheResult GetCacheRecord(CacheKey CacheKey, ZenContentType Type) override { try @@ -434,6 +434,7 @@ namespace detail { spdlog::logger& Log() { return m_Log; } spdlog::logger& m_Log; + UpstreamEndpointInfo m_Info; bool m_UseLegacyDdc; std::string m_DisplayName; RefPtr m_Client; @@ -456,7 +457,7 @@ namespace detail { public: ZenUpstreamEndpoint(const ZenStructuredCacheClientOptions& Options) : m_Log(zen::logging::Get("upstream")) - , m_DisplayName("ZEN") + , m_Info({.Name = std::string("Zen")}) , m_ConnectTimeout(Options.ConnectTimeout) , m_Timeout(Options.Timeout) { @@ -468,6 +469,8 @@ namespace detail { ~ZenUpstreamEndpoint() = default; + virtual const UpstreamEndpointInfo& GetEndpointInfo() const { return m_Info; } + virtual UpstreamEndpointHealth Initialize() override { using namespace fmt::literals; @@ -475,9 +478,8 @@ namespace detail { const ZenEndpoint& Ep = GetEndpoint(); if (Ep.Ok) { - m_ServiceUrl = Ep.Url; - m_DisplayName = "ZEN - {}"_format(m_ServiceUrl); - m_Client = new ZenStructuredCacheClient({.Url = m_ServiceUrl, .ConnectTimeout = m_ConnectTimeout, .Timeout = m_Timeout}); + m_Info.Url = Ep.Url; + m_Client = new ZenStructuredCacheClient({.Url = m_Info.Url, .ConnectTimeout = m_ConnectTimeout, .Timeout = m_Timeout}); m_HealthOk = true; return {.Ok = true}; @@ -500,10 +502,9 @@ namespace detail { const ZenEndpoint& Ep = GetEndpoint(); if (Ep.Ok) { - m_ServiceUrl = Ep.Url; - m_DisplayName = "ZEN - {}"_format(m_ServiceUrl); + m_Info.Url = Ep.Url; m_Client = - new ZenStructuredCacheClient({.Url = m_ServiceUrl, .ConnectTimeout = m_ConnectTimeout, .Timeout = m_Timeout}); + new ZenStructuredCacheClient({.Url = m_Info.Url, .ConnectTimeout = m_ConnectTimeout, .Timeout = m_Timeout}); m_HealthOk = true; return {.Ok = true}; @@ -530,8 +531,6 @@ namespace detail { } } - virtual std::string_view DisplayName() const override { return m_DisplayName; } - virtual GetUpstreamCacheResult GetCacheRecord(CacheKey CacheKey, ZenContentType Type) override { try @@ -866,7 +865,7 @@ namespace detail { for (const auto& Ep : m_Endpoints) { - ZEN_INFO("ping ZEN endpoint '{}' latency '{:.3}s' {}", Ep.Url, Ep.Latency, Ep.Ok ? "OK" : Ep.Reason); + ZEN_INFO("ping 'Zen' endpoint '{}' latency '{:.3}s' {}", Ep.Url, Ep.Latency, Ep.Ok ? "OK" : Ep.Reason); } return m_Endpoints.front(); @@ -875,9 +874,8 @@ namespace detail { spdlog::logger& Log() { return m_Log; } spdlog::logger& m_Log; - std::string m_ServiceUrl; + UpstreamEndpointInfo m_Info; std::vector m_Endpoints; - std::string m_DisplayName; std::chrono::milliseconds m_ConnectTimeout; std::chrono::milliseconds m_Timeout; RefPtr m_Client; @@ -966,7 +964,7 @@ struct UpstreamStats const double HitRate = TotalCount > 0 ? (double(HitCount) / double(TotalCount)) : 0.0; Logger.debug("STATS - '{}', Hit rate: {:.2f}%, DOWN: '{:.2f} MiB {:.2f} MiB/s', UP: '{:.2f} MiB {:.2f} MiB/s'", - Ep->DisplayName(), + Ep->GetEndpointInfo().Name, HitRate, DownBytes, DownSpeed, @@ -1000,13 +998,15 @@ public: for (auto& Endpoint : m_Endpoints) { const UpstreamEndpointHealth Health = Endpoint->Initialize(); + const UpstreamEndpointInfo& Info = Endpoint->GetEndpointInfo(); + if (Health.Ok) { - ZEN_INFO("initialize endpoint '{}' OK", Endpoint->DisplayName()); + ZEN_INFO("'{}' endpoint '{}' OK", Info.Name, Info.Url); } else { - ZEN_WARN("initialize endpoint '{}' FAILED, reason '{}'", Endpoint->DisplayName(), Health.Reason); + ZEN_WARN("'{}' endpoint '{}' FAILED, reason '{}'", Info.Name, Info.Url, Health.Reason); } } @@ -1046,7 +1046,7 @@ public: if (Result.Error) { ZEN_ERROR("get cache record FAILED, endpoint '{}', reason '{}', error code '{}'", - Endpoint->DisplayName(), + Endpoint->GetEndpointInfo().Url, Result.Error.Reason, Result.Error.ErrorCode); } @@ -1086,7 +1086,7 @@ public: if (Result.Error) { ZEN_ERROR("get cache record(s) (rpc) FAILED, endpoint '{}', reason '{}', error code '{}'", - Endpoint->DisplayName(), + Endpoint->GetEndpointInfo().Url, Result.Error.Reason, Result.Error.ErrorCode); } @@ -1132,7 +1132,7 @@ public: if (Result.Error) { ZEN_ERROR("get cache payloads(s) (rpc) FAILED, endpoint '{}', reason '{}', error code '{}'", - Endpoint->DisplayName(), + Endpoint->GetEndpointInfo().Url, Result.Error.Reason, Result.Error.ErrorCode); } @@ -1168,7 +1168,7 @@ public: if (Result.Error) { ZEN_ERROR("get cache payload FAILED, endpoint '{}', reason '{}', error code '{}'", - Endpoint->DisplayName(), + Endpoint->GetEndpointInfo().Url, Result.Error.Reason, Result.Error.ErrorCode); } @@ -1208,8 +1208,10 @@ public: Status.BeginArray("endpoints"); for (const auto& Ep : m_Endpoints) { + const UpstreamEndpointInfo& Info = Ep->GetEndpointInfo(); Status.BeginObject(); - Status << "name" << Ep->DisplayName(); + Status << "name" << Info.Name; + Status << "url" << Info.Url; Status << "health" << (Ep->IsHealthy() ? "ok"sv : "inactive"sv); UpstreamEndpointStats& Stats = Ep->Stats(); @@ -1270,7 +1272,7 @@ private: ZEN_WARN("upload cache record '{}/{}' FAILED, endpoint '{}', reason '{}'", CacheRecord.CacheKey.Bucket, CacheRecord.CacheKey.Hash, - Endpoint->DisplayName(), + Endpoint->GetEndpointInfo().Url, Result.Reason); } } @@ -1322,13 +1324,14 @@ private: { if (!Endpoint->IsHealthy()) { + const UpstreamEndpointInfo& Info = Endpoint->GetEndpointInfo(); if (const UpstreamEndpointHealth Health = Endpoint->CheckHealth(); Health.Ok) { - ZEN_INFO("health check endpoint '{}' OK", Endpoint->DisplayName(), Health.Reason); + ZEN_INFO("health check endpoint '{} - {}' OK", Info.Name, Info.Url, Health.Reason); } else { - ZEN_WARN("health check endpoint '{}' FAILED, reason '{}'", Endpoint->DisplayName(), Health.Reason); + ZEN_WARN("health check endpoint '{} - {}' FAILED, reason '{}'", Info.Name, Info.Url, Health.Reason); } } } diff --git a/zenserver/upstream/upstreamcache.h b/zenserver/upstream/upstreamcache.h index fb2550cad..12287198d 100644 --- a/zenserver/upstream/upstreamcache.h +++ b/zenserver/upstream/upstreamcache.h @@ -100,6 +100,12 @@ struct CachePayloadGetCompleteParams using OnCachePayloadGetComplete = std::function; +struct UpstreamEndpointInfo +{ + std::string Name; + std::string Url; +}; + /** * The upstream endpont is responsible for handling upload/downloading of cache records. */ @@ -108,14 +114,14 @@ class UpstreamEndpoint public: virtual ~UpstreamEndpoint() = default; + virtual const UpstreamEndpointInfo& GetEndpointInfo() const = 0; + virtual UpstreamEndpointHealth Initialize() = 0; virtual bool IsHealthy() const = 0; virtual UpstreamEndpointHealth CheckHealth() = 0; - virtual std::string_view DisplayName() const = 0; - virtual GetUpstreamCacheResult GetCacheRecord(CacheKey CacheKey, ZenContentType Type) = 0; virtual GetUpstreamCacheResult GetCacheRecords(std::span CacheKeys, -- cgit v1.2.3 From f16668694fe5f565a55860bed27233f1a3650f80 Mon Sep 17 00:00:00 2001 From: Joe Kirchoff Date: Wed, 17 Nov 2021 14:32:43 -0800 Subject: Set horde worker requirements, limit to Win-RemoteExec pool --- zenserver/upstream/upstreamapply.cpp | 41 ++++++++++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 4 deletions(-) (limited to 'zenserver') diff --git a/zenserver/upstream/upstreamapply.cpp b/zenserver/upstream/upstreamapply.cpp index 3267b408e..651c2dcc8 100644 --- a/zenserver/upstream/upstreamapply.cpp +++ b/zenserver/upstream/upstreamapply.cpp @@ -798,9 +798,42 @@ namespace detail { { using namespace fmt::literals; std::string_view HostPlatform = ApplyRecord.WorkerDescriptor["host"sv].AsString(); - // TODO: Enable when Horde accepts the UE style Host Platforms (Win64, Linux, Mac) - // CbObject Requirements = BuildRequirements("OSFamily == '{}'"_format(HostPlatform), {}, false); - CbObject Requirements = BuildRequirements("OSFamily == 'Windows'", {}, false); + if (HostPlatform.empty()) + { + Log().warn("process apply upstream FAILED, 'host' platform not provided"); + return false; + } + + int32_t LogicalCores = ApplyRecord.WorkerDescriptor["cores"sv].AsInt32(); + int64_t Memory = ApplyRecord.WorkerDescriptor["memory"sv].AsInt64(); + bool Exclusive = ApplyRecord.WorkerDescriptor["exclusive"sv].AsBool(); + + // TODO: Remove override when Horde accepts the UE style Host Platforms (Win64, Linux, Mac) + std::string Condition; + if (HostPlatform == "Win64" || HostPlatform == "Windows") + { + Condition = "OSFamily == 'Windows' && Pool == 'Win-RemoteExec'"; + } + else if (HostPlatform == "Mac") + { + Condition = "OSFamily == 'MacOS'"; + } + else + { + Condition = "OSFamily == '{}'"_format(HostPlatform); + } + + std::map Resources; + if (LogicalCores > 0) + { + Resources["LogicalCores"sv] = LogicalCores; + } + if (Memory > 0) + { + Resources["RAM"sv] = std::max(Memory / 1024 / 1024 / 1024, 1LL); + } + + CbObject Requirements = BuildRequirements(Condition, Resources, Exclusive); const IoHash RequirementsId = Requirements.GetHash(); Data.Objects[RequirementsId] = std::move(Requirements); Data.RequirementsId = RequirementsId; @@ -954,7 +987,7 @@ namespace detail { } [[nodiscard]] CbObject BuildRequirements(const std::string_view Condition, - const std::map& Resources, + const std::map& Resources, const bool Exclusive) { CbObjectWriter Writer; -- cgit v1.2.3 From a18ba147f04207810c9d2c034e219a4337b2310c Mon Sep 17 00:00:00 2001 From: Joe Kirchoff Date: Wed, 17 Nov 2021 14:51:46 -0800 Subject: Change error code for failed upsteam apply --- zenserver/compute/apply.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'zenserver') diff --git a/zenserver/compute/apply.cpp b/zenserver/compute/apply.cpp index d2ae2febc..8ad14a1ed 100644 --- a/zenserver/compute/apply.cpp +++ b/zenserver/compute/apply.cpp @@ -932,7 +932,7 @@ HttpFunctionService::ExecActionUpstreamResult(const IoHash& WorkerId, const IoHa Completed.Error.Reason); // throw std::runtime_error( // "Action {}/{} failed: {}"_format(WorkerId.ToHexString(), ActionId.ToHexString(), Completed.Error.Reason).c_str()); - return HttpResponseCode::NotFound; + return HttpResponseCode::BadRequest; } ZEN_INFO("Action {}/{} completed with {} attachments ({} compressed, {} uncompressed)", -- cgit v1.2.3