From ca09abbeef5b1788f4a52b61eedd2f3dd07f81f2 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Tue, 14 Oct 2025 11:32:16 +0200 Subject: move all storage-related services into storage tree (#571) * move all storage-related services into storage tree * move config into config/ * also move admin service into storage since it mostly has storage related functionality * header consolidation --- src/zenserver/admin/admin.cpp | 805 ----- src/zenserver/admin/admin.h | 46 - src/zenserver/buildstore/httpbuildstore.cpp | 561 ---- src/zenserver/buildstore/httpbuildstore.h | 68 - src/zenserver/cache/httpstructuredcache.cpp | 2052 ------------ src/zenserver/cache/httpstructuredcache.h | 138 - src/zenserver/config.cpp | 512 --- src/zenserver/config.h | 87 - src/zenserver/config/config.cpp | 512 +++ src/zenserver/config/config.h | 87 + src/zenserver/diag/logging.cpp | 2 +- src/zenserver/main.cpp | 20 +- src/zenserver/objectstore/objectstore.cpp | 618 ---- src/zenserver/objectstore/objectstore.h | 53 - src/zenserver/projectstore/httpprojectstore.cpp | 3307 -------------------- src/zenserver/projectstore/httpprojectstore.h | 111 - src/zenserver/stats/statsreporter.h | 2 +- src/zenserver/storage/admin/admin.cpp | 804 +++++ src/zenserver/storage/admin/admin.h | 46 + .../storage/buildstore/httpbuildstore.cpp | 561 ++++ src/zenserver/storage/buildstore/httpbuildstore.h | 68 + .../storage/cache/httpstructuredcache.cpp | 2052 ++++++++++++ src/zenserver/storage/cache/httpstructuredcache.h | 138 + src/zenserver/storage/objectstore/objectstore.cpp | 618 ++++ src/zenserver/storage/objectstore/objectstore.h | 53 + .../storage/projectstore/httpprojectstore.cpp | 3307 ++++++++++++++++++++ .../storage/projectstore/httpprojectstore.h | 111 + src/zenserver/storage/storageconfig.cpp | 1055 +++++++ src/zenserver/storage/storageconfig.h | 203 ++ src/zenserver/storage/upstream/upstream.h | 7 + src/zenserver/storage/upstream/upstreamcache.cpp | 2134 +++++++++++++ src/zenserver/storage/upstream/upstreamcache.h | 167 + src/zenserver/storage/upstream/upstreamservice.cpp | 55 + src/zenserver/storage/upstream/upstreamservice.h | 27 + src/zenserver/storage/upstream/zen.cpp | 251 ++ src/zenserver/storage/upstream/zen.h | 103 + src/zenserver/storage/vfs/vfsservice.cpp | 179 ++ src/zenserver/storage/vfs/vfsservice.h | 48 + .../storage/workspaces/httpworkspaces.cpp | 1211 +++++++ src/zenserver/storage/workspaces/httpworkspaces.h | 97 + src/zenserver/storage/zenstorageserver.cpp | 961 ++++++ src/zenserver/storage/zenstorageserver.h | 113 + src/zenserver/storageconfig.cpp | 1055 ------- src/zenserver/storageconfig.h | 203 -- src/zenserver/upstream/upstream.h | 7 - src/zenserver/upstream/upstreamcache.cpp | 2134 ------------- src/zenserver/upstream/upstreamcache.h | 167 - src/zenserver/upstream/upstreamservice.cpp | 55 - src/zenserver/upstream/upstreamservice.h | 27 - src/zenserver/upstream/zen.cpp | 251 -- src/zenserver/upstream/zen.h | 103 - src/zenserver/vfs/vfsservice.cpp | 179 -- src/zenserver/vfs/vfsservice.h | 48 - src/zenserver/workspaces/httpworkspaces.cpp | 1211 ------- src/zenserver/workspaces/httpworkspaces.h | 97 - src/zenserver/zenserver.cpp | 10 +- src/zenserver/zenstorageserver.cpp | 961 ------ src/zenserver/zenstorageserver.h | 113 - 58 files changed, 14979 insertions(+), 14992 deletions(-) delete mode 100644 src/zenserver/admin/admin.cpp delete mode 100644 src/zenserver/admin/admin.h delete mode 100644 src/zenserver/buildstore/httpbuildstore.cpp delete mode 100644 src/zenserver/buildstore/httpbuildstore.h delete mode 100644 src/zenserver/cache/httpstructuredcache.cpp delete mode 100644 src/zenserver/cache/httpstructuredcache.h delete mode 100644 src/zenserver/config.cpp delete mode 100644 src/zenserver/config.h create mode 100644 src/zenserver/config/config.cpp create mode 100644 src/zenserver/config/config.h delete mode 100644 src/zenserver/objectstore/objectstore.cpp delete mode 100644 src/zenserver/objectstore/objectstore.h delete mode 100644 src/zenserver/projectstore/httpprojectstore.cpp delete mode 100644 src/zenserver/projectstore/httpprojectstore.h create mode 100644 src/zenserver/storage/admin/admin.cpp create mode 100644 src/zenserver/storage/admin/admin.h create mode 100644 src/zenserver/storage/buildstore/httpbuildstore.cpp create mode 100644 src/zenserver/storage/buildstore/httpbuildstore.h create mode 100644 src/zenserver/storage/cache/httpstructuredcache.cpp create mode 100644 src/zenserver/storage/cache/httpstructuredcache.h create mode 100644 src/zenserver/storage/objectstore/objectstore.cpp create mode 100644 src/zenserver/storage/objectstore/objectstore.h create mode 100644 src/zenserver/storage/projectstore/httpprojectstore.cpp create mode 100644 src/zenserver/storage/projectstore/httpprojectstore.h create mode 100644 src/zenserver/storage/storageconfig.cpp create mode 100644 src/zenserver/storage/storageconfig.h create mode 100644 src/zenserver/storage/upstream/upstream.h create mode 100644 src/zenserver/storage/upstream/upstreamcache.cpp create mode 100644 src/zenserver/storage/upstream/upstreamcache.h create mode 100644 src/zenserver/storage/upstream/upstreamservice.cpp create mode 100644 src/zenserver/storage/upstream/upstreamservice.h create mode 100644 src/zenserver/storage/upstream/zen.cpp create mode 100644 src/zenserver/storage/upstream/zen.h create mode 100644 src/zenserver/storage/vfs/vfsservice.cpp create mode 100644 src/zenserver/storage/vfs/vfsservice.h create mode 100644 src/zenserver/storage/workspaces/httpworkspaces.cpp create mode 100644 src/zenserver/storage/workspaces/httpworkspaces.h create mode 100644 src/zenserver/storage/zenstorageserver.cpp create mode 100644 src/zenserver/storage/zenstorageserver.h delete mode 100644 src/zenserver/storageconfig.cpp delete mode 100644 src/zenserver/storageconfig.h delete mode 100644 src/zenserver/upstream/upstream.h delete mode 100644 src/zenserver/upstream/upstreamcache.cpp delete mode 100644 src/zenserver/upstream/upstreamcache.h delete mode 100644 src/zenserver/upstream/upstreamservice.cpp delete mode 100644 src/zenserver/upstream/upstreamservice.h delete mode 100644 src/zenserver/upstream/zen.cpp delete mode 100644 src/zenserver/upstream/zen.h delete mode 100644 src/zenserver/vfs/vfsservice.cpp delete mode 100644 src/zenserver/vfs/vfsservice.h delete mode 100644 src/zenserver/workspaces/httpworkspaces.cpp delete mode 100644 src/zenserver/workspaces/httpworkspaces.h delete mode 100644 src/zenserver/zenstorageserver.cpp delete mode 100644 src/zenserver/zenstorageserver.h (limited to 'src') diff --git a/src/zenserver/admin/admin.cpp b/src/zenserver/admin/admin.cpp deleted file mode 100644 index 97522e892..000000000 --- a/src/zenserver/admin/admin.cpp +++ /dev/null @@ -1,805 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "admin.h" - -#include -#include -#include -#include -#include -#include - -#if ZEN_WITH_TRACE -# include -#endif // ZEN_WITH_TRACE - -#if ZEN_USE_MIMALLOC -# include -#endif - -#include - -#include -#include -#include "config.h" - -#include - -namespace zen { - -struct DirStats -{ - uint64_t FileCount = 0; - uint64_t DirCount = 0; - uint64_t ByteCount = 0; -}; - -DirStats -GetStatsForDirectory(std::filesystem::path Dir) -{ - if (!IsDir(Dir)) - return {}; - - struct StatsTraversal : public GetDirectoryContentVisitor - { - virtual void AsyncVisitDirectory(const std::filesystem::path& RelativeRoot, DirectoryContent&& Content) override - { - ZEN_UNUSED(RelativeRoot); - - uint64_t FileCount = Content.FileNames.size(); - uint64_t DirCount = Content.DirectoryNames.size(); - uint64_t FilesSize = 0; - for (uint64_t FileSize : Content.FileSizes) - { - FilesSize += FileSize; - } - TotalBytes += FilesSize; - TotalFileCount += FileCount; - TotalDirCount += DirCount; - } - - std::atomic_uint64_t TotalBytes = 0; - std::atomic_uint64_t TotalFileCount = 0; - std::atomic_uint64_t TotalDirCount = 0; - - DirStats GetStats() - { - return {.FileCount = TotalFileCount.load(), .DirCount = TotalDirCount.load(), .ByteCount = TotalBytes.load()}; - } - } DirTraverser; - - Latch PendingWorkCount(1); - - GetDirectoryContent(Dir, - DirectoryContentFlags::IncludeAllEntries | DirectoryContentFlags::IncludeFileSizes, - DirTraverser, - GetSmallWorkerPool(EWorkloadType::Background), - PendingWorkCount); - PendingWorkCount.CountDown(); - PendingWorkCount.Wait(); - - return DirTraverser.GetStats(); -} - -struct StateDiskStats -{ - DirStats CacheStats; - DirStats CasStats; - DirStats ProjectStats; -}; - -StateDiskStats -GetStatsForStateDirectory(std::filesystem::path StateDir) -{ - StateDiskStats Stats; - Stats.CacheStats = GetStatsForDirectory(StateDir / "cache"); - Stats.CasStats = GetStatsForDirectory(StateDir / "cas"); - Stats.ProjectStats = GetStatsForDirectory(StateDir / "projects"); - return Stats; -} - -HttpAdminService::HttpAdminService(GcScheduler& Scheduler, - JobQueue& BackgroundJobQueue, - ZenCacheStore* CacheStore, - std::function&& FlushFunction, - const LogPaths& LogPaths, - const ZenServerOptions& ServerOptions) -: m_GcScheduler(Scheduler) -, m_BackgroundJobQueue(BackgroundJobQueue) -, m_CacheStore(CacheStore) -, m_FlushFunction(std::move(FlushFunction)) -, m_LogPaths(LogPaths) -, m_ServerOptions(ServerOptions) -{ - using namespace std::literals; - - m_Router.RegisterRoute( - "health", - [](HttpRouterRequest& Req) { - CbObjectWriter Obj; - Obj.AddBool("ok", true); - Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); - }, - HttpVerb::kGet); - - m_Router.AddPattern("jobid", "([[:digit:]]+?)"); - - m_Router.RegisterRoute( - "jobs", - [&](HttpRouterRequest& Req) { - std::vector Jobs = m_BackgroundJobQueue.GetJobs(); - CbObjectWriter Obj; - Obj.BeginArray("jobs"); - for (const auto& Job : Jobs) - { - Obj.BeginObject(); - Obj.AddInteger("Id", Job.Id.Id); - Obj.AddString("Status", JobQueue::ToString(Job.Status)); - Obj.EndObject(); - } - Obj.EndArray(); - Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); - }, - HttpVerb::kGet); - - m_Router.RegisterRoute( - "jobs/{jobid}", - [&](HttpRouterRequest& Req) { - const auto& JobIdString = Req.GetCapture(1); - std::optional JobIdArg = ParseInt(JobIdString); - if (!JobIdArg) - { - Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); - } - JobId Id{.Id = JobIdArg.value_or(0)}; - if (Id.Id == 0) - { - return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, - ZenContentType::kText, - fmt::format("Invalid Job Id: {}", Id.Id)); - } - - std::optional CurrentState = m_BackgroundJobQueue.Get(Id); - if (!CurrentState) - { - return Req.ServerRequest().WriteResponse(HttpResponseCode::NotFound); - } - - auto WriteState = [](CbObjectWriter& Obj, const JobQueue::State& State) { - if (!State.CurrentOp.empty()) - { - Obj.AddString( - "CurrentOp"sv, - State.CurrentOpDetails.empty() ? State.CurrentOp : fmt::format("{}: {}", State.CurrentOp, State.CurrentOpDetails)); - Obj.AddString("Op"sv, State.CurrentOp); - if (!State.CurrentOpDetails.empty()) - { - Obj.AddString("Details"sv, State.CurrentOpDetails); - } - Obj.AddInteger("TotalCount"sv, gsl::narrow(State.TotalCount)); - Obj.AddInteger("RemainingCount"sv, gsl::narrow(State.RemainingCount)); - Obj.AddInteger("CurrentOpPercentComplete"sv, - State.TotalCount > 0 - ? gsl::narrow((100 * (State.TotalCount - State.RemainingCount)) / State.TotalCount) - : 0); - } - if (!State.Messages.empty()) - { - Obj.BeginArray("Messages"); - for (const std::string& Message : State.Messages) - { - Obj.AddString(Message); - } - Obj.EndArray(); - } - if (!State.AbortReason.empty()) - { - Obj.AddString("AbortReason"sv, State.AbortReason); - } - }; - - auto GetAgeAsSeconds = [](std::chrono::system_clock::time_point Start, std::chrono::system_clock::time_point End) { - auto Age = End - Start; - auto Milliseconds = std::chrono::duration_cast(Age); - return Milliseconds.count() / 1000.0; - }; - - const std::chrono::system_clock::time_point Now = std::chrono::system_clock::now(); - - switch (CurrentState->Status) - { - case JobQueue::Status::Queued: - { - CbObjectWriter Obj; - Obj.AddString("Name"sv, CurrentState->Name); - Obj.AddString("Status"sv, "Queued"sv); - Obj.AddFloat("QueueTimeS", GetAgeAsSeconds(CurrentState->CreateTime, Now)); - Obj.AddInteger("WorkerThread", CurrentState->WorkerThreadId); - Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); - } - break; - case JobQueue::Status::Running: - { - CbObjectWriter Obj; - Obj.AddString("Name"sv, CurrentState->Name); - Obj.AddString("Status"sv, "Running"sv); - WriteState(Obj, CurrentState->State); - Obj.AddFloat("QueueTimeS", GetAgeAsSeconds(CurrentState->CreateTime, CurrentState->StartTime)); - Obj.AddFloat("RunTimeS", GetAgeAsSeconds(CurrentState->StartTime, Now)); - Obj.AddInteger("WorkerThread", CurrentState->WorkerThreadId); - Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); - } - break; - case JobQueue::Status::Aborted: - { - CbObjectWriter Obj; - Obj.AddString("Name"sv, CurrentState->Name); - Obj.AddString("Status"sv, "Aborted"sv); - - WriteState(Obj, CurrentState->State); - Obj.AddFloat("QueueTimeS", GetAgeAsSeconds(CurrentState->CreateTime, CurrentState->StartTime)); - Obj.AddFloat("RunTimeS", GetAgeAsSeconds(CurrentState->StartTime, CurrentState->EndTime)); - Obj.AddFloat("CompleteTimeS", GetAgeAsSeconds(CurrentState->EndTime, Now)); - Obj.AddInteger("ReturnCode", CurrentState->ReturnCode); - Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); - } - break; - case JobQueue::Status::Completed: - { - CbObjectWriter Obj; - Obj.AddString("Name"sv, CurrentState->Name); - Obj.AddString("Status"sv, "Complete"sv); - WriteState(Obj, CurrentState->State); - Obj.AddFloat("QueueTimeS", GetAgeAsSeconds(CurrentState->CreateTime, CurrentState->StartTime)); - Obj.AddFloat("RunTimeS", GetAgeAsSeconds(CurrentState->StartTime, CurrentState->EndTime)); - Obj.AddFloat("CompleteTimeS", GetAgeAsSeconds(CurrentState->EndTime, Now)); - Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); - } - break; - } - }, - HttpVerb::kGet); - - m_Router.RegisterRoute( - "jobs/{jobid}", - [&](HttpRouterRequest& Req) { - const auto& JobIdString = Req.GetCapture(1); - std::optional JobIdArg = ParseInt(JobIdString); - if (!JobIdArg) - { - Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); - } - JobId Id{.Id = JobIdArg.value_or(0)}; - if (m_BackgroundJobQueue.CancelJob(Id)) - { - Req.ServerRequest().WriteResponse(HttpResponseCode::OK); - } - else - { - Req.ServerRequest().WriteResponse(HttpResponseCode::NotFound); - } - }, - HttpVerb::kDelete); - - m_Router.RegisterRoute( - "gc", - [this](HttpRouterRequest& Req) { - const GcSchedulerState State = m_GcScheduler.GetState(); - - const HttpServerRequest::QueryParams Params = Req.ServerRequest().GetQueryParams(); - - bool Details = false; - if (auto Param = Params.GetValue("details"); Param == "true") - { - Details = true; - } - - CbObjectWriter Response; - Response << "Status"sv << (GcSchedulerStatus::kIdle == State.Status ? "Idle"sv : "Running"sv); - Response.BeginObject("Config"); - { - Response << "RootDirectory" << State.Config.RootDirectory.string(); - Response << "MonitorInterval" << ToTimeSpan(State.Config.MonitorInterval); - Response << "Interval" << ToTimeSpan(State.Config.Interval); - Response << "MaxCacheDuration" << ToTimeSpan(State.Config.MaxCacheDuration); - Response << "MaxProjectStoreDuration" << ToTimeSpan(State.Config.MaxProjectStoreDuration); - Response << "MaxBuildStoreDuration" << ToTimeSpan(State.Config.MaxBuildStoreDuration); - Response << "CollectSmallObjects" << State.Config.CollectSmallObjects; - Response << "Enabled" << State.Config.Enabled; - Response << "DiskReserveSize" << NiceBytes(State.Config.DiskReserveSize); - Response << "DiskSizeSoftLimit" << NiceBytes(State.Config.DiskSizeSoftLimit); - Response << "MinimumFreeDiskSpaceToAllowWrites" << NiceBytes(State.Config.MinimumFreeDiskSpaceToAllowWrites); - Response << "LightweightInterval" << ToTimeSpan(State.Config.LightweightInterval); - Response << "UseGCVersion" << ((State.Config.UseGCVersion == GcVersion::kV1_Deprecated) ? "1" : "2"); - Response << "CompactBlockUsageThresholdPercent" << State.Config.CompactBlockUsageThresholdPercent; - Response << "Verbose" << State.Config.Verbose; - Response << "SingleThreaded" << State.Config.SingleThreaded; - Response << "AttachmentPassCount" << State.Config.AttachmentPassCount; - } - Response.EndObject(); - Response << "AreDiskWritesBlocked" << State.AreDiskWritesBlocked; - Response << "HasDiskReserve" << State.HasDiskReserve; - Response << "DiskSize" << NiceBytes(State.DiskSize); - Response << "DiskUsed" << NiceBytes(State.DiskUsed); - Response << "DiskFree" << NiceBytes(State.DiskFree); - - Response.BeginObject("FullGC"); - { - Response << "LastTime" << ToDateTime(State.LastFullGcTime); - Response << "TimeToNext" << ToTimeSpan(State.RemainingTimeUntilFullGc); - if (State.Config.DiskSizeSoftLimit != 0) - { - Response << "SpaceToNext" << NiceBytes(State.RemainingSpaceUntilFullGC); - } - if (State.LastFullGCV2Result) - { - const bool HumanReadable = true; - WriteGCResult(Response, State.LastFullGCV2Result.value(), HumanReadable, Details); - } - else - { - Response << "LastDuration" << ToTimeSpan(State.LastFullGcDuration); - Response << "LastDiskFreed" << NiceBytes(State.LastFullGCDiff.DiskSize); - Response << "LastMemoryFreed" << NiceBytes(State.LastFullGCDiff.MemorySize); - } - if (State.LastFullAttachmentRangeMin != IoHash::Zero || State.LastFullAttachmentRangeMax != IoHash::Max) - { - Response << "AttachmentRangeMin" << State.LastFullAttachmentRangeMin; - Response << "AttachmentRangeMax" << State.LastFullAttachmentRangeMax; - } - } - Response.EndObject(); - Response.BeginObject("LightweightGC"); - { - Response << "LastTime" << ToDateTime(State.LastLightweightGcTime); - Response << "TimeToNext" << ToTimeSpan(State.RemainingTimeUntilLightweightGc); - - if (State.LastLightweightGCV2Result) - { - const bool HumanReadable = true; - WriteGCResult(Response, State.LastLightweightGCV2Result.value(), HumanReadable, Details); - } - else - { - Response << "LastDuration" << ToTimeSpan(State.LastLightweightGcDuration); - Response << "LastDiskFreed" << NiceBytes(State.LastLightweightGCDiff.DiskSize); - Response << "LastMemoryFreed" << NiceBytes(State.LastLightweightGCDiff.MemorySize); - } - } - Response.EndObject(); - Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Response.Save()); - }, - HttpVerb::kGet); - - m_Router.RegisterRoute( - "gc", - [this](HttpRouterRequest& Req) { - HttpServerRequest& HttpReq = Req.ServerRequest(); - const HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); - GcScheduler::TriggerGcParams GcParams; - - if (auto Param = Params.GetValue("smallobjects"); Param.empty() == false) - { - GcParams.CollectSmallObjects = Param == "true"sv; - } - - if (auto Param = Params.GetValue("maxcacheduration"); Param.empty() == false) - { - if (auto Value = ParseInt(Param)) - { - GcParams.MaxCacheDuration = std::chrono::seconds(Value.value()); - } - } - - if (auto Param = Params.GetValue("maxprojectstoreduration"); Param.empty() == false) - { - if (auto Value = ParseInt(Param)) - { - GcParams.MaxProjectStoreDuration = std::chrono::seconds(Value.value()); - } - } - - if (auto Param = Params.GetValue("maxbuildstoreduration"); Param.empty() == false) - { - if (auto Value = ParseInt(Param)) - { - GcParams.MaxBuildStoreDuration = std::chrono::seconds(Value.value()); - } - } - - if (auto Param = Params.GetValue("disksizesoftlimit"); Param.empty() == false) - { - if (auto Value = ParseInt(Param)) - { - GcParams.DiskSizeSoftLimit = Value.value(); - } - } - - if (auto Param = Params.GetValue("skipcid"); Param.empty() == false) - { - GcParams.SkipCid = Param == "true"sv; - } - - if (auto Param = Params.GetValue("skipdelete"); Param.empty() == false) - { - GcParams.SkipDelete = Param == "true"sv; - } - - if (auto Param = Params.GetValue("forceusegcv1"); Param.empty() == false) - { - GcParams.ForceGCVersion = GcVersion::kV1_Deprecated; - } - - if (auto Param = Params.GetValue("forceusegcv2"); Param.empty() == false) - { - GcParams.ForceGCVersion = GcVersion::kV2; - } - - if (auto Param = Params.GetValue("compactblockthreshold"); Param.empty() == false) - { - if (auto Value = ParseInt(Param)) - { - GcParams.CompactBlockUsageThresholdPercent = Value.value(); - } - } - - if (auto Param = Params.GetValue("verbose"); Param.empty() == false) - { - GcParams.Verbose = Param == "true"sv; - } - - if (auto Param = Params.GetValue("singlethreaded"); Param.empty() == false) - { - GcParams.SingleThreaded = Param == "true"sv; - } - - if (auto Param = Params.GetValue("referencehashlow"); Param.empty() == false) - { - GcParams.AttachmentRangeMin = IoHash::FromHexString(Param); - } - - if (auto Param = Params.GetValue("referencehashhigh"); Param.empty() == false) - { - GcParams.AttachmentRangeMax = IoHash::FromHexString(Param); - } - - if (auto Param = Params.GetValue("storecacheattachmentmetadata"); Param.empty() == false) - { - GcParams.StoreCacheAttachmentMetaData = Param == "true"sv; - } - - if (auto Param = Params.GetValue("storeprojectattachmentmetadata"); Param.empty() == false) - { - GcParams.StoreProjectAttachmentMetaData = Param == "true"sv; - } - - if (auto Param = Params.GetValue("enablevalidation"); Param.empty() == false) - { - GcParams.EnableValidation = Param == "true"sv; - } - - const bool Started = m_GcScheduler.TriggerGc(GcParams); - - CbObjectWriter Response; - Response << "Status"sv << (Started ? "Started"sv : "Running"sv); - HttpReq.WriteResponse(HttpResponseCode::Accepted, Response.Save()); - }, - HttpVerb::kPost); - - m_Router.RegisterRoute( - "gc-stop", - [this](HttpRouterRequest& Req) { - HttpServerRequest& HttpReq = Req.ServerRequest(); - if (m_GcScheduler.CancelGC()) - { - return HttpReq.WriteResponse(HttpResponseCode::Accepted); - } - HttpReq.WriteResponse(HttpResponseCode::OK); - }, - HttpVerb::kPost); - -#if ZEN_USE_MIMALLOC - m_Router.RegisterRoute( - "mi_collect", - [this](HttpRouterRequest& Req) { - HttpServerRequest& HttpReq = Req.ServerRequest(); - const HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); - - bool Force = false; - - if (auto Param = Params.GetValue("force"); Param.empty() == false) - { - Force = (Param == "true"sv); - } - - ExtendableStringBuilder<256> MiStats; - ExtendableStringBuilder<256> MiStatsAfter; - - auto MiOutputFun = [](const char* msg, void* arg) { - StringBuilderBase* StarsSb = reinterpret_cast(arg); - StarsSb->AppendAscii(msg); - }; - - mi_stats_print_out(MiOutputFun, static_cast(&MiStats)); - mi_collect(Force); - mi_stats_print_out(MiOutputFun, static_cast(&MiStatsAfter)); - - CbObjectWriter Response; - Response << "force"sv << Force; - Response << "stats_before"sv << MiStats; - Response << "stats_after"sv << MiStatsAfter; - HttpReq.WriteResponse(HttpResponseCode::OK, Response.Save()); - }, - HttpVerb::kPost); -#endif - - m_Router.RegisterRoute( - "scrub", - [this](HttpRouterRequest& Req) { - HttpServerRequest& HttpReq = Req.ServerRequest(); - const HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); - - GcScheduler::TriggerScrubParams ScrubParams; - ScrubParams.MaxTimeslice = std::chrono::seconds(100); - - if (auto Param = Params.GetValue("skipdelete"); Param.empty() == false) - { - ScrubParams.SkipDelete = (Param == "true"sv); - } - - if (auto Param = Params.GetValue("skipgc"); Param.empty() == false) - { - ScrubParams.SkipGc = (Param == "true"sv); - } - - if (auto Param = Params.GetValue("skipcid"); Param.empty() == false) - { - ScrubParams.SkipCas = (Param == "true"sv); - } - - m_GcScheduler.TriggerScrub(ScrubParams); - - CbObjectWriter Response; - Response << "ok"sv << true; - Response << "skip_delete" << ScrubParams.SkipDelete; - Response << "skip_gc" << ScrubParams.SkipGc; - Response << "skip_cas" << ScrubParams.SkipCas; - Response << "max_time" << TimeSpan(0, 0, gsl::narrow(ScrubParams.MaxTimeslice.count())); - HttpReq.WriteResponse(HttpResponseCode::OK, Response.Save()); - }, - HttpVerb::kPost); - - m_Router.RegisterRoute( - "", - [](HttpRouterRequest& Req) { - CbObject Payload = Req.ServerRequest().ReadPayloadObject(); - - CbObjectWriter Obj; - Obj.AddBool("ok", true); - Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); - }, - HttpVerb::kPost); - -#if ZEN_WITH_TRACE - m_Router.RegisterRoute( - "trace", - [this](HttpRouterRequest& Req) { - bool Enabled = IsTracing(); - return Req.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Enabled ? "enabled" : "disabled"); - }, - HttpVerb::kGet); - - m_Router.RegisterRoute( - "trace/start", - [this](HttpRouterRequest& Req) { - HttpServerRequest& HttpReq = Req.ServerRequest(); - const HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); - TraceOptions TraceOptions; - - if (!IsTracing()) - { - TraceInit("zenserver"); - } - - if (auto Channels = Params.GetValue("channels"); Channels.empty() == false) - { - TraceOptions.Channels = Channels; - } - - if (auto File = Params.GetValue("file"); File.empty() == false) - { - TraceOptions.File = File; - } - else if (auto Host = Params.GetValue("host"); Host.empty() == false) - { - TraceOptions.Host = Host; - } - else - { - return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - "Invalid trace type, use `file` or `host`"sv); - } - - TraceConfigure(TraceOptions); - - return Req.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kText, "Tracing started"); - }, - HttpVerb::kPost); - - m_Router.RegisterRoute( - "trace/stop", - [this](HttpRouterRequest& Req) { - if (!IsTracing()) - { - return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Tracing is not enabled"sv); - } - if (TraceStop()) - { - return Req.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kText, "Tracing stopped"); - } - else - { - return Req.ServerRequest().WriteResponse(HttpResponseCode::InternalServerError, - HttpContentType::kText, - "Failed stopping trace"); - } - }, - HttpVerb::kPost); -#endif // ZEN_WITH_TRACE - - m_Router.RegisterRoute( - "info", - [this](HttpRouterRequest& Req) { - CbObjectWriter Obj; - - Obj << "root" << m_ServerOptions.SystemRootDir.generic_wstring(); - Obj << "install" << (m_ServerOptions.SystemRootDir / "Install").generic_wstring(); - - Obj.BeginObject("primary"); - Obj << "data" << m_ServerOptions.DataDir.generic_wstring(); - - try - { - auto Stats = GetStatsForStateDirectory(m_ServerOptions.DataDir); - - auto EmitStats = [&](std::string_view Tag, const DirStats& Stats) { - Obj.BeginObject(Tag); - Obj << "bytes" << Stats.ByteCount; - Obj << "files" << Stats.FileCount; - Obj << "dirs" << Stats.DirCount; - Obj.EndObject(); - }; - - EmitStats("cache", Stats.CacheStats); - EmitStats("cas", Stats.CasStats); - EmitStats("project", Stats.ProjectStats); - } - catch (const std::exception& Ex) - { - ZEN_WARN("exception in disk stats gathering for '{}': {}", m_ServerOptions.DataDir, Ex.what()); - } - Obj.EndObject(); - - try - { - std::vector Manifests = ReadAllCentralManifests(m_ServerOptions.SystemRootDir); - - Obj.BeginArray("known"); - - for (const auto& Manifest : Manifests) - { - Obj.AddObject(Manifest); - } - - Obj.EndArray(); - } - catch (const std::exception& Ex) - { - ZEN_WARN("exception in state gathering for '{}': {}", m_ServerOptions.SystemRootDir, Ex.what()); - } - Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); - }, - HttpVerb::kGet); - - m_Router.RegisterRoute( - "logs", - [this](HttpRouterRequest& Req) { - CbObjectWriter Obj; - auto LogLevel = logging::level::ToStringView(logging::GetLogLevel()); - Obj.AddString("loglevel", std::string_view(LogLevel.data(), LogLevel.size())); - Obj.AddString("Logfile", PathToUtf8(m_LogPaths.AbsLogPath)); - Obj.BeginObject("cache"); - if (m_CacheStore) - { - const ZenCacheStore::Configuration& CacheConfig = m_CacheStore->GetConfiguration(); - Obj.AddString("Logfile", PathToUtf8(m_LogPaths.CacheLogPath)); - Obj.AddBool("Write", CacheConfig.Logging.EnableWriteLog); - Obj.AddBool("Access", CacheConfig.Logging.EnableAccessLog); - } - Obj.EndObject(); - Obj.BeginObject("http"); - { - Obj.AddString("Logfile", PathToUtf8(m_LogPaths.HttpLogPath)); - } - Obj.EndObject(); - Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); - }, - HttpVerb::kGet); - - m_Router.RegisterRoute( - "logs", - [this](HttpRouterRequest& Req) { - HttpServerRequest& HttpReq = Req.ServerRequest(); - const HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); - bool SetCacheLogConfig = false; - ExtendableStringBuilder<256> StringBuilder; - if (m_CacheStore) - { - ZenCacheStore::Configuration::LogConfig LoggingConfig = m_CacheStore->GetConfiguration().Logging; - if (std::string Param(Params.GetValue("cacheenablewritelog")); Param.empty() == false) - { - LoggingConfig.EnableWriteLog = StrCaseCompare(Param.c_str(), "true") == 0; - SetCacheLogConfig = true; - } - if (std::string Param(Params.GetValue("cacheenableaccesslog")); Param.empty() == false) - { - LoggingConfig.EnableAccessLog = StrCaseCompare(Param.c_str(), "true") == 0; - SetCacheLogConfig = true; - } - if (SetCacheLogConfig) - { - m_CacheStore->SetLoggingConfig(LoggingConfig); - StringBuilder.Append(fmt::format("cache write log: {}, cache access log: {}", - LoggingConfig.EnableWriteLog ? "true" : "false", - LoggingConfig.EnableAccessLog ? "true" : "false")); - } - } - if (std::string Param(Params.GetValue("loglevel")); Param.empty() == false) - { - logging::level::LogLevel NewLevel = logging::level::ParseLogLevelString(Param); - std::string_view LogLevel = logging::level::ToStringView(NewLevel); - if (LogLevel != Param) - { - return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Invalid log level '{}'", Param)); - } - logging::SetLogLevel(NewLevel); - if (StringBuilder.Size() > 0) - { - StringBuilder.Append(", "); - } - StringBuilder.Append("loglevel: "); - StringBuilder.Append(Param); - } - return Req.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kText, StringBuilder.ToView()); - }, - HttpVerb::kPost); - m_Router.RegisterRoute( - "flush", - [this](HttpRouterRequest& Req) { - HttpServerRequest& HttpReq = Req.ServerRequest(); - m_FlushFunction(); - HttpReq.WriteResponse(HttpResponseCode::OK); - }, - HttpVerb::kPost); -} - -HttpAdminService::~HttpAdminService() -{ -} - -const char* -HttpAdminService::BaseUri() const -{ - return "/admin/"; -} - -void -HttpAdminService::HandleRequest(zen::HttpServerRequest& Request) -{ - m_Router.HandleRequest(Request); -} - -} // namespace zen diff --git a/src/zenserver/admin/admin.h b/src/zenserver/admin/admin.h deleted file mode 100644 index 9a49f5120..000000000 --- a/src/zenserver/admin/admin.h +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include -#include -#include - -namespace zen { - -class GcScheduler; -class JobQueue; -class ZenCacheStore; -struct ZenServerOptions; - -class HttpAdminService : public zen::HttpService -{ -public: - struct LogPaths - { - std::filesystem::path AbsLogPath; - std::filesystem::path HttpLogPath; - std::filesystem::path CacheLogPath; - }; - HttpAdminService(GcScheduler& Scheduler, - JobQueue& BackgroundJobQueue, - ZenCacheStore* CacheStore, - std::function&& FlushFunction, - const LogPaths& LogPaths, - const ZenServerOptions& ServerOptions); - ~HttpAdminService(); - - virtual const char* BaseUri() const override; - virtual void HandleRequest(zen::HttpServerRequest& Request) override; - -private: - HttpRequestRouter m_Router; - GcScheduler& m_GcScheduler; - JobQueue& m_BackgroundJobQueue; - ZenCacheStore* m_CacheStore; - std::function m_FlushFunction; - LogPaths m_LogPaths; - const ZenServerOptions& m_ServerOptions; -}; - -} // namespace zen diff --git a/src/zenserver/buildstore/httpbuildstore.cpp b/src/zenserver/buildstore/httpbuildstore.cpp deleted file mode 100644 index bce993f17..000000000 --- a/src/zenserver/buildstore/httpbuildstore.cpp +++ /dev/null @@ -1,561 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "httpbuildstore.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -namespace zen { -using namespace std::literals; - -ZEN_DEFINE_LOG_CATEGORY_STATIC(LogBuilds, "builds"sv); - -HttpBuildStoreService::HttpBuildStoreService(HttpStatusService& StatusService, HttpStatsService& StatsService, BuildStore& Store) -: m_Log(logging::Get("builds")) -, m_StatusService(StatusService) -, m_StatsService(StatsService) -, m_BuildStore(Store) -{ - Initialize(); - - m_StatusService.RegisterHandler("builds", *this); - m_StatsService.RegisterHandler("builds", *this); -} - -HttpBuildStoreService::~HttpBuildStoreService() -{ - m_StatsService.UnregisterHandler("builds", *this); - m_StatusService.UnregisterHandler("builds", *this); -} - -const char* -HttpBuildStoreService::BaseUri() const -{ - return "/builds/"; -} - -void -HttpBuildStoreService::Initialize() -{ - ZEN_LOG_INFO(LogBuilds, "Initializing Builds Service"); - - m_Router.AddPattern("namespace", "([[:alnum:]\\-_.]+)"); - m_Router.AddPattern("bucket", "([[:alnum:]\\-_.]+)"); - m_Router.AddPattern("buildid", "([[:xdigit:]]{24})"); - m_Router.AddPattern("hash", "([[:xdigit:]]{40})"); - - m_Router.RegisterRoute( - "{namespace}/{bucket}/{buildid}/blobs/{hash}", - [this](HttpRouterRequest& Req) { PutBlobRequest(Req); }, - HttpVerb::kPut); - - m_Router.RegisterRoute( - "{namespace}/{bucket}/{buildid}/blobs/{hash}", - [this](HttpRouterRequest& Req) { GetBlobRequest(Req); }, - HttpVerb::kGet); - - m_Router.RegisterRoute( - "{namespace}/{bucket}/{buildid}/blobs/putBlobMetadata", - [this](HttpRouterRequest& Req) { PutMetadataRequest(Req); }, - HttpVerb::kPost); - - m_Router.RegisterRoute( - "{namespace}/{bucket}/{buildid}/blobs/getBlobMetadata", - [this](HttpRouterRequest& Req) { GetMetadatasRequest(Req); }, - HttpVerb::kPost); - - m_Router.RegisterRoute( - "{namespace}/{bucket}/{buildid}/blobs/exists", - [this](HttpRouterRequest& Req) { BlobsExistsRequest(Req); }, - HttpVerb::kPost); -} - -void -HttpBuildStoreService::HandleRequest(zen::HttpServerRequest& Request) -{ - ZEN_TRACE_CPU("HttpBuildStoreService::HandleRequest"); - metrics::OperationTiming::Scope $(m_HttpRequests); - - m_BuildStoreStats.RequestCount++; - if (m_Router.HandleRequest(Request) == false) - { - ZEN_LOG_WARN(LogBuilds, "No route found for {0}", Request.RelativeUri()); - return Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Not found"sv); - } -} - -void -HttpBuildStoreService::PutBlobRequest(HttpRouterRequest& Req) -{ - ZEN_TRACE_CPU("HttpBuildStoreService::PutBlobRequest"); - HttpServerRequest& ServerRequest = Req.ServerRequest(); - const std::string_view Namespace = Req.GetCapture(1); - const std::string_view Bucket = Req.GetCapture(2); - const std::string_view BuildId = Req.GetCapture(3); - const std::string_view Hash = Req.GetCapture(4); - ZEN_UNUSED(Namespace, Bucket, BuildId); - IoHash BlobHash; - if (!IoHash::TryParse(Hash, BlobHash)) - { - m_BuildStoreStats.BadRequestCount++; - return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Invalid blob hash '{}'", Hash)); - } - m_BuildStoreStats.BlobWriteCount++; - IoBuffer Payload = ServerRequest.ReadPayload(); - if (!Payload) - { - m_BuildStoreStats.BadRequestCount++; - return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Payload blob {} is empty", Hash)); - } - if (Payload.GetContentType() != HttpContentType::kCompressedBinary) - { - m_BuildStoreStats.BadRequestCount++; - return ServerRequest.WriteResponse( - HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Payload blob {} content type {} is invalid", Hash, ToString(Payload.GetContentType()))); - } - m_BuildStore.PutBlob(BlobHash, ServerRequest.ReadPayload()); - // ZEN_INFO("Stored blob {}. Size: {}", BlobHash, ServerRequest.ReadPayload().GetSize()); - return ServerRequest.WriteResponse(HttpResponseCode::OK); -} - -void -HttpBuildStoreService::GetBlobRequest(HttpRouterRequest& Req) -{ - ZEN_TRACE_CPU("HttpBuildStoreService::GetBlobRequest"); - HttpServerRequest& ServerRequest = Req.ServerRequest(); - std::string_view Namespace = Req.GetCapture(1); - std::string_view Bucket = Req.GetCapture(2); - std::string_view BuildId = Req.GetCapture(3); - std::string_view Hash = Req.GetCapture(4); - ZEN_UNUSED(Namespace, Bucket, BuildId); - IoHash BlobHash; - if (!IoHash::TryParse(Hash, BlobHash)) - { - m_BuildStoreStats.BadRequestCount++; - return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Invalid blob hash '{}'", Hash)); - } - zen::HttpRanges Ranges; - bool HasRange = ServerRequest.TryGetRanges(Ranges); - if (Ranges.size() > 1) - { - // Only a single range is supported - return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - "Multiple ranges in blob request is not supported"); - } - - m_BuildStoreStats.BlobReadCount++; - IoBuffer Blob = m_BuildStore.GetBlob(BlobHash); - if (!Blob) - { - return ServerRequest.WriteResponse(HttpResponseCode::NotFound, - HttpContentType::kText, - fmt::format("Blob with hash '{}' could not be found", Hash)); - } - // ZEN_INFO("Fetched blob {}. Size: {}", BlobHash, Blob.GetSize()); - m_BuildStoreStats.BlobHitCount++; - if (HasRange) - { - const HttpRange& Range = Ranges.front(); - const uint64_t BlobSize = Blob.GetSize(); - const uint64_t MaxBlobSize = Range.Start < BlobSize ? Range.Start - BlobSize : 0; - const uint64_t RangeSize = Min(Range.End - Range.Start + 1, MaxBlobSize); - if (Range.Start + RangeSize > BlobSize) - { - return ServerRequest.WriteResponse(HttpResponseCode::NoContent); - } - Blob = IoBuffer(Blob, Range.Start, RangeSize); - return ServerRequest.WriteResponse(HttpResponseCode::OK, ZenContentType::kBinary, Blob); - } - else - { - return ServerRequest.WriteResponse(HttpResponseCode::OK, Blob.GetContentType(), Blob); - } -} - -void -HttpBuildStoreService::PutMetadataRequest(HttpRouterRequest& Req) -{ - ZEN_TRACE_CPU("HttpBuildStoreService::PutMetadataRequest"); - HttpServerRequest& ServerRequest = Req.ServerRequest(); - std::string_view Namespace = Req.GetCapture(1); - std::string_view Bucket = Req.GetCapture(2); - std::string_view BuildId = Req.GetCapture(3); - - IoBuffer MetaPayload = ServerRequest.ReadPayload(); - if (MetaPayload.GetContentType() != ZenContentType::kCbPackage) - { - throw std::runtime_error(fmt::format("PutMetadataRequest payload has unexpected payload type '{}', expected '{}'", - ToString(MetaPayload.GetContentType()), - ToString(ZenContentType::kCbPackage))); - } - CbPackage Message = ParsePackageMessage(MetaPayload); - - CbObjectView MessageObject = Message.GetObject(); - if (!MessageObject) - { - throw std::runtime_error("PutMetadataRequest payload object is missing"); - } - CbArrayView BlobsArray = MessageObject["blobHashes"sv].AsArrayView(); - CbArrayView MetadataArray = MessageObject["metadatas"sv].AsArrayView(); - - const uint64_t BlobCount = BlobsArray.Num(); - if (BlobCount == 0) - { - throw std::runtime_error("PutMetadataRequest blobs array is empty"); - } - if (BlobCount != MetadataArray.Num()) - { - throw std::runtime_error( - fmt::format("PutMetadataRequest metadata array size {} does not match blobs array size {}", MetadataArray.Num(), BlobCount)); - } - - std::vector BlobHashes; - std::vector MetadataPayloads; - - BlobHashes.reserve(BlobCount); - MetadataPayloads.reserve(BlobCount); - - auto BlobsArrayIt = begin(BlobsArray); - auto MetadataArrayIt = begin(MetadataArray); - while (BlobsArrayIt != end(BlobsArray)) - { - const IoHash BlobHash = (*BlobsArrayIt).AsHash(); - const IoHash MetadataHash = (*MetadataArrayIt).AsAttachment(); - - const CbAttachment* Attachment = Message.FindAttachment(MetadataHash); - if (Attachment == nullptr) - { - throw std::runtime_error(fmt::format("Blob metadata attachment {} is missing", MetadataHash)); - } - BlobHashes.push_back(BlobHash); - if (Attachment->IsObject()) - { - MetadataPayloads.push_back(Attachment->AsObject().GetBuffer().MakeOwned().AsIoBuffer()); - MetadataPayloads.back().SetContentType(ZenContentType::kCbObject); - } - else if (Attachment->IsCompressedBinary()) - { - MetadataPayloads.push_back(Attachment->AsCompressedBinary().GetCompressed().Flatten().AsIoBuffer()); - MetadataPayloads.back().SetContentType(ZenContentType::kCompressedBinary); - } - else - { - ZEN_ASSERT(Attachment->IsBinary()); - MetadataPayloads.push_back(Attachment->AsBinary().AsIoBuffer()); - MetadataPayloads.back().SetContentType(ZenContentType::kBinary); - } - - BlobsArrayIt++; - MetadataArrayIt++; - } - m_BuildStore.PutMetadatas(BlobHashes, MetadataPayloads, &GetSmallWorkerPool(EWorkloadType::Burst)); - return ServerRequest.WriteResponse(HttpResponseCode::OK); -} - -void -HttpBuildStoreService::GetMetadatasRequest(HttpRouterRequest& Req) -{ - ZEN_TRACE_CPU("HttpBuildStoreService::GetMetadatasRequest"); - HttpServerRequest& ServerRequest = Req.ServerRequest(); - std::string_view Namespace = Req.GetCapture(1); - std::string_view Bucket = Req.GetCapture(2); - std::string_view BuildId = Req.GetCapture(3); - ZEN_UNUSED(Namespace, Bucket, BuildId); - IoBuffer RequestPayload = ServerRequest.ReadPayload(); - if (!RequestPayload) - { - m_BuildStoreStats.BadRequestCount++; - return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - "Expected compact binary body for metadata request, body is missing"); - } - if (RequestPayload.GetContentType() != HttpContentType::kCbObject) - { - m_BuildStoreStats.BadRequestCount++; - return ServerRequest.WriteResponse( - HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Expected compact binary body for metadata request, got {}", ToString(RequestPayload.GetContentType()))); - } - if (CbValidateError ValidateError = ValidateCompactBinary(RequestPayload.GetView(), CbValidateMode::Default); - ValidateError != CbValidateError::None) - { - m_BuildStoreStats.BadRequestCount++; - return ServerRequest.WriteResponse( - HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Compact binary body for metadata request is not valid, reason: {}", ToString(ValidateError))); - } - CbObject RequestObject = LoadCompactBinaryObject(RequestPayload); - CbArrayView BlobsArray = RequestObject["blobHashes"sv].AsArrayView(); - if (!BlobsArray) - { - m_BuildStoreStats.BadRequestCount++; - return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - "Compact binary body for metadata request is missing 'blobHashes' array"); - } - const uint64_t BlobCount = BlobsArray.Num(); - - std::vector BlobRawHashes; - BlobRawHashes.reserve(BlobCount); - for (CbFieldView BlockHashView : BlobsArray) - { - BlobRawHashes.push_back(BlockHashView.AsHash()); - if (BlobRawHashes.back() == IoHash::Zero) - { - const uint8_t Type = (uint8_t)BlockHashView.GetValue().GetType(); - return ServerRequest.WriteResponse( - HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Compact binary body for metadata 'blobHashes' array contains invalid field type: {}", Type)); - } - } - m_BuildStoreStats.BlobMetaReadCount += BlobRawHashes.size(); - std::vector BlockMetadatas = m_BuildStore.GetMetadatas(BlobRawHashes, &GetSmallWorkerPool(EWorkloadType::Burst)); - - CbPackage ResponsePackage; - std::vector Attachments; - tsl::robin_set AttachmentHashes; - Attachments.reserve(BlobCount); - AttachmentHashes.reserve(BlobCount); - { - CbObjectWriter ResponseWriter; - - ResponseWriter.BeginArray("blobHashes"); - for (size_t BlockHashIndex = 0; BlockHashIndex < BlobRawHashes.size(); BlockHashIndex++) - { - if (BlockMetadatas[BlockHashIndex]) - { - const IoHash& BlockHash = BlobRawHashes[BlockHashIndex]; - ResponseWriter.AddHash(BlockHash); - } - } - ResponseWriter.EndArray(); // blobHashes - - ResponseWriter.BeginArray("metadatas"); - - for (size_t BlockHashIndex = 0; BlockHashIndex < BlobRawHashes.size(); BlockHashIndex++) - { - if (IoBuffer Metadata = BlockMetadatas[BlockHashIndex]; Metadata) - { - switch (Metadata.GetContentType()) - { - case ZenContentType::kCbObject: - { - CbObject Object = CbObject(SharedBuffer(std::move(Metadata)).MakeOwned()); - const IoHash ObjectHash = Object.GetHash(); - ResponseWriter.AddBinaryAttachment(ObjectHash); - if (!AttachmentHashes.contains(ObjectHash)) - { - Attachments.push_back(CbAttachment(Object, ObjectHash)); - AttachmentHashes.insert(ObjectHash); - } - } - break; - case ZenContentType::kCompressedBinary: - { - IoHash RawHash; - uint64_t _; - CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(std::move(Metadata)), RawHash, _); - ResponseWriter.AddBinaryAttachment(RawHash); - if (!AttachmentHashes.contains(RawHash)) - { - Attachments.push_back(CbAttachment(Compressed, RawHash)); - AttachmentHashes.insert(RawHash); - } - } - break; - default: - { - const IoHash RawHash = IoHash::HashBuffer(Metadata); - ResponseWriter.AddBinaryAttachment(RawHash); - if (!AttachmentHashes.contains(RawHash)) - { - Attachments.push_back(CbAttachment(SharedBuffer(Metadata), RawHash)); - AttachmentHashes.insert(RawHash); - } - } - break; - } - } - } - - ResponseWriter.EndArray(); // metadatas - - ResponsePackage.SetObject(ResponseWriter.Save()); - } - ResponsePackage.AddAttachments(Attachments); - - CompositeBuffer RpcResponseBuffer = FormatPackageMessageBuffer(ResponsePackage); - ServerRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, RpcResponseBuffer); -} - -void -HttpBuildStoreService::BlobsExistsRequest(HttpRouterRequest& Req) -{ - ZEN_TRACE_CPU("HttpBuildStoreService::BlobsExistsRequest"); - HttpServerRequest& ServerRequest = Req.ServerRequest(); - std::string_view Namespace = Req.GetCapture(1); - std::string_view Bucket = Req.GetCapture(2); - std::string_view BuildId = Req.GetCapture(3); - ZEN_UNUSED(Namespace, Bucket, BuildId); - IoBuffer RequestPayload = ServerRequest.ReadPayload(); - if (!RequestPayload) - { - m_BuildStoreStats.BadRequestCount++; - return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - "Expected compact binary body for blob exists request, body is missing"); - } - if (RequestPayload.GetContentType() != HttpContentType::kCbObject) - { - m_BuildStoreStats.BadRequestCount++; - return ServerRequest.WriteResponse( - HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Expected compact binary body for blob exists request, got {}", ToString(RequestPayload.GetContentType()))); - } - if (CbValidateError ValidateError = ValidateCompactBinary(RequestPayload.GetView(), CbValidateMode::Default); - ValidateError != CbValidateError::None) - { - m_BuildStoreStats.BadRequestCount++; - return ServerRequest.WriteResponse( - HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Compact binary body for blob exists request is not valid, reason: {}", ToString(ValidateError))); - } - CbObject RequestObject = LoadCompactBinaryObject(RequestPayload); - CbArrayView BlobsArray = RequestObject["blobHashes"sv].AsArrayView(); - if (!BlobsArray) - { - m_BuildStoreStats.BadRequestCount++; - return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - "Compact binary body for blob exists request is missing 'blobHashes' array"); - } - - std::vector BlobRawHashes; - BlobRawHashes.reserve(BlobsArray.Num()); - for (CbFieldView BlockHashView : BlobsArray) - { - BlobRawHashes.push_back(BlockHashView.AsHash()); - if (BlobRawHashes.back() == IoHash::Zero) - { - const uint8_t Type = (uint8_t)BlockHashView.GetValue().GetType(); - return ServerRequest.WriteResponse( - HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Compact binary body for blob exists request 'blobHashes' array contains invalid field type: {}", Type)); - } - } - - m_BuildStoreStats.BlobExistsCount += BlobRawHashes.size(); - std::vector BlobsExists = m_BuildStore.BlobsExists(BlobRawHashes); - CbObjectWriter ResponseWriter(9 * BlobsExists.size()); - ResponseWriter.BeginArray("blobExists"sv); - for (const BuildStore::BlobExistsResult& BlobExists : BlobsExists) - { - ResponseWriter.AddBool(BlobExists.HasBody); - if (BlobExists.HasBody) - { - m_BuildStoreStats.BlobExistsBodyHitCount++; - } - } - ResponseWriter.EndArray(); // blobExist - ResponseWriter.BeginArray("metadataExists"sv); - for (const BuildStore::BlobExistsResult& BlobExists : BlobsExists) - { - ResponseWriter.AddBool(BlobExists.HasMetadata); - if (BlobExists.HasMetadata) - { - m_BuildStoreStats.BlobExistsMetaHitCount++; - } - } - ResponseWriter.EndArray(); // metadataExists - CbObject ResponseObject = ResponseWriter.Save(); - return ServerRequest.WriteResponse(HttpResponseCode::OK, ResponseObject); -} - -void -HttpBuildStoreService::HandleStatsRequest(HttpServerRequest& Request) -{ - ZEN_TRACE_CPU("HttpBuildStoreService::Stats"); - CbObjectWriter Cbo; - - EmitSnapshot("requests", m_HttpRequests, Cbo); - - Cbo.BeginObject("builds"); - { - Cbo.BeginObject("blobs"); - { - Cbo << "readcount" << m_BuildStoreStats.BlobReadCount << "writecount" << m_BuildStoreStats.BlobWriteCount << "hitcount" - << m_BuildStoreStats.BlobHitCount; - } - Cbo.EndObject(); - - Cbo.BeginObject("metadata"); - { - Cbo << "readcount" << m_BuildStoreStats.BlobMetaReadCount << "writecount" << m_BuildStoreStats.BlobMetaWriteCount << "hitcount" - << m_BuildStoreStats.BlobMetaHitCount; - } - Cbo.EndObject(); - - Cbo << "requestcount" << m_BuildStoreStats.RequestCount; - Cbo << "badrequestcount" << m_BuildStoreStats.BadRequestCount; - } - Cbo.EndObject(); - - Cbo.BeginObject("size"); - { - BuildStore::StorageStats StorageStats = m_BuildStore.GetStorageStats(); - - Cbo << "count" << StorageStats.EntryCount; - Cbo << "bytes" << StorageStats.BlobBytes + StorageStats.MetadataByteCount; - Cbo.BeginObject("blobs"); - { - Cbo << "count" << StorageStats.BlobCount; - Cbo << "bytes" << StorageStats.BlobBytes; - } - Cbo.EndObject(); // blobs - - Cbo.BeginObject("metadata"); - { - Cbo << "count" << StorageStats.MetadataCount; - Cbo << "bytes" << StorageStats.MetadataByteCount; - } - Cbo.EndObject(); // metadata - } - Cbo.EndObject(); // size - - return Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); -} - -void -HttpBuildStoreService::HandleStatusRequest(HttpServerRequest& Request) -{ - ZEN_TRACE_CPU("HttpBuildStoreService::Status"); - CbObjectWriter Cbo; - Cbo << "ok" << true; - Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); -} - -} // namespace zen diff --git a/src/zenserver/buildstore/httpbuildstore.h b/src/zenserver/buildstore/httpbuildstore.h deleted file mode 100644 index 50cb5db12..000000000 --- a/src/zenserver/buildstore/httpbuildstore.h +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include -#include -#include -#include - -#include - -namespace zen { - -class BuildStore; - -class HttpBuildStoreService final : public zen::HttpService, public IHttpStatusProvider, public IHttpStatsProvider -{ -public: - HttpBuildStoreService(HttpStatusService& StatusService, HttpStatsService& StatsService, BuildStore& Store); - virtual ~HttpBuildStoreService(); - - virtual const char* BaseUri() const override; - virtual void HandleRequest(zen::HttpServerRequest& Request) override; - - virtual void HandleStatsRequest(HttpServerRequest& Request) override; - virtual void HandleStatusRequest(HttpServerRequest& Request) override; - -private: - struct BuildStoreStats - { - std::atomic_uint64_t BlobReadCount{}; - std::atomic_uint64_t BlobHitCount{}; - std::atomic_uint64_t BlobWriteCount{}; - std::atomic_uint64_t BlobMetaReadCount{}; - std::atomic_uint64_t BlobMetaHitCount{}; - std::atomic_uint64_t BlobMetaWriteCount{}; - std::atomic_uint64_t BlobExistsCount{}; - std::atomic_uint64_t BlobExistsBodyHitCount{}; - std::atomic_uint64_t BlobExistsMetaHitCount{}; - std::atomic_uint64_t RequestCount{}; - std::atomic_uint64_t BadRequestCount{}; - }; - - void Initialize(); - - inline LoggerRef Log() { return m_Log; } - - LoggerRef m_Log; - - void PutBlobRequest(HttpRouterRequest& Req); - void GetBlobRequest(HttpRouterRequest& Req); - - void PutMetadataRequest(HttpRouterRequest& Req); - void GetMetadatasRequest(HttpRouterRequest& Req); - - void BlobsExistsRequest(HttpRouterRequest& Req); - - HttpRequestRouter m_Router; - - HttpStatusService& m_StatusService; - HttpStatsService& m_StatsService; - - BuildStore& m_BuildStore; - BuildStoreStats m_BuildStoreStats; - metrics::OperationTiming m_HttpRequests; -}; - -} // namespace zen diff --git a/src/zenserver/cache/httpstructuredcache.cpp b/src/zenserver/cache/httpstructuredcache.cpp deleted file mode 100644 index 9f87c208c..000000000 --- a/src/zenserver/cache/httpstructuredcache.cpp +++ /dev/null @@ -1,2052 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "httpstructuredcache.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "upstream/upstreamcache.h" -#include "upstream/zen.h" -#include "zenstore/cidstore.h" -#include "zenstore/scrubcontext.h" - -#include -#include -#include -#include -#include - -#include - -namespace zen { - -const FLLMTag& -GetCacheHttpTag() -{ - static FLLMTag CacheHttpTag("http", FLLMTag("cache")); - - return CacheHttpTag; -} - -extern const FLLMTag& GetCacheRpcTag(); - -using namespace std::literals; - -////////////////////////////////////////////////////////////////////////// - -CachePolicy -ParseCachePolicy(const HttpServerRequest::QueryParams& QueryParams) -{ - std::string_view PolicyText = QueryParams.GetValue("Policy"sv); - return !PolicyText.empty() ? zen::ParseCachePolicy(PolicyText) : CachePolicy::Default; -} - -namespace { - static constinit std::string_view HttpZCacheRPCPrefix = "$rpc"sv; - static constinit std::string_view HttpZCacheUtilStartRecording = "exec$/start-recording"sv; - static constinit std::string_view HttpZCacheUtilStopRecording = "exec$/stop-recording"sv; - static constinit std::string_view HttpZCacheUtilReplayRecording = "exec$/replay-recording"sv; - static constinit std::string_view HttpZCacheDetailsPrefix = "details$"sv; -} // namespace - -////////////////////////////////////////////////////////////////////////// - -HttpStructuredCacheService::HttpStructuredCacheService(ZenCacheStore& InCacheStore, - CidStore& InCidStore, - HttpStatsService& StatsService, - HttpStatusService& StatusService, - UpstreamCache& UpstreamCache, - const DiskWriteBlocker* InDiskWriteBlocker, - OpenProcessCache& InOpenProcessCache) -: m_Log(logging::Get("cache")) -, m_CacheStore(InCacheStore) -, m_StatsService(StatsService) -, m_StatusService(StatusService) -, m_CidStore(InCidStore) -, m_UpstreamCache(UpstreamCache) -, m_DiskWriteBlocker(InDiskWriteBlocker) -, m_OpenProcessCache(InOpenProcessCache) -, m_RpcHandler(m_Log, m_CacheStats, UpstreamCache, InCacheStore, InCidStore, InDiskWriteBlocker) -{ - m_StatsService.RegisterHandler("z$", *this); - m_StatusService.RegisterHandler("z$", *this); -} - -HttpStructuredCacheService::~HttpStructuredCacheService() -{ - ZEN_INFO("closing structured cache"); - { - RwLock::ExclusiveLockScope _(m_RequestRecordingLock); - m_RequestRecordingEnabled.store(false); - m_RequestRecorder.reset(); - } - - m_StatsService.UnregisterHandler("z$", *this); - m_StatusService.UnregisterHandler("z$", *this); -} - -const char* -HttpStructuredCacheService::BaseUri() const -{ - return "/z$/"; -} - -void -HttpStructuredCacheService::Flush() -{ - m_CacheStore.Flush(); -} - -void -HttpStructuredCacheService::HandleDetailsRequest(HttpServerRequest& Request) -{ - std::string_view Key = Request.RelativeUri(); - std::vector Tokens; - uint32_t TokenCount = ForEachStrTok(Key, '/', [&Tokens](std::string_view Token) { - Tokens.push_back(std::string(Token)); - return true; - }); - std::string FilterNamespace; - std::string FilterBucket; - std::string FilterValue; - switch (TokenCount) - { - case 1: - break; - case 2: - { - FilterNamespace = Tokens[1]; - if (FilterNamespace.empty()) - { - m_CacheStats.BadRequestCount++; - return Request.WriteResponse(HttpResponseCode::BadRequest); // invalid URL - } - } - break; - case 3: - { - FilterNamespace = Tokens[1]; - if (FilterNamespace.empty()) - { - m_CacheStats.BadRequestCount++; - return Request.WriteResponse(HttpResponseCode::BadRequest); // invalid URL - } - FilterBucket = Tokens[2]; - if (FilterBucket.empty()) - { - m_CacheStats.BadRequestCount++; - return Request.WriteResponse(HttpResponseCode::BadRequest); // invalid URL - } - } - break; - case 4: - { - FilterNamespace = Tokens[1]; - if (FilterNamespace.empty()) - { - m_CacheStats.BadRequestCount++; - return Request.WriteResponse(HttpResponseCode::BadRequest); // invalid URL - } - FilterBucket = Tokens[2]; - if (FilterBucket.empty()) - { - m_CacheStats.BadRequestCount++; - return Request.WriteResponse(HttpResponseCode::BadRequest); // invalid URL - } - FilterValue = Tokens[3]; - if (FilterValue.empty()) - { - m_CacheStats.BadRequestCount++; - return Request.WriteResponse(HttpResponseCode::BadRequest); // invalid URL - } - } - break; - default: - m_CacheStats.BadRequestCount++; - return Request.WriteResponse(HttpResponseCode::BadRequest); // invalid URL - } - - HttpServerRequest::QueryParams Params = Request.GetQueryParams(); - bool CSV = Params.GetValue("csv") == "true"; - bool Details = Params.GetValue("details") == "true"; - bool AttachmentDetails = Params.GetValue("attachmentdetails") == "true"; - - std::chrono::seconds NowSeconds = std::chrono::duration_cast(GcClock::Now().time_since_epoch()); - CacheValueDetails ValueDetails = m_CacheStore.GetValueDetails(FilterNamespace, FilterBucket, FilterValue); - - if (CSV) - { - ExtendableStringBuilder<4096> CSVWriter; - if (AttachmentDetails) - { - CSVWriter << "Namespace, Bucket, Key, Cid, Size"; - } - else if (Details) - { - CSVWriter << "Namespace, Bucket, Key, Size, RawSize, RawHash, ContentType, Age, AttachmentsCount, AttachmentsSize"; - } - else - { - CSVWriter << "Namespace, Bucket, Key"; - } - for (const auto& NamespaceIt : ValueDetails.Namespaces) - { - const std::string& Namespace = NamespaceIt.first; - for (const auto& BucketIt : NamespaceIt.second.Buckets) - { - const std::string& Bucket = BucketIt.first; - for (const auto& ValueIt : BucketIt.second.Values) - { - if (AttachmentDetails) - { - for (const IoHash& Hash : ValueIt.second.Attachments) - { - IoBuffer Payload = m_CidStore.FindChunkByCid(Hash); - CSVWriter << "\r\n" - << Namespace << "," << Bucket << "," << ValueIt.first.ToHexString() << ", " << Hash.ToHexString() - << ", " << gsl::narrow(Payload.GetSize()); - } - } - else if (Details) - { - std::chrono::seconds LastAccessedSeconds = std::chrono::duration_cast( - GcClock::TimePointFromTick(ValueIt.second.LastAccess).time_since_epoch()); - CSVWriter << "\r\n" - << Namespace << "," << Bucket << "," << ValueIt.first.ToHexString() << ", " << ValueIt.second.Size << "," - << ValueIt.second.RawSize << "," << ValueIt.second.RawHash.ToHexString() << ", " - << ToString(ValueIt.second.ContentType) << ", " << (NowSeconds.count() - LastAccessedSeconds.count()) - << ", " << gsl::narrow(ValueIt.second.Attachments.size()); - size_t AttachmentsSize = 0; - for (const IoHash& Hash : ValueIt.second.Attachments) - { - IoBuffer Payload = m_CidStore.FindChunkByCid(Hash); - AttachmentsSize += Payload.GetSize(); - } - CSVWriter << ", " << gsl::narrow(AttachmentsSize); - } - else - { - CSVWriter << "\r\n" << Namespace << "," << Bucket << "," << ValueIt.first.ToHexString(); - } - } - } - } - return Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView()); - } - else - { - CbObjectWriter Cbo; - Cbo.BeginArray("namespaces"); - { - for (const auto& NamespaceIt : ValueDetails.Namespaces) - { - const std::string& Namespace = NamespaceIt.first; - Cbo.BeginObject(); - { - Cbo.AddString("name", Namespace); - Cbo.BeginArray("buckets"); - { - for (const auto& BucketIt : NamespaceIt.second.Buckets) - { - const std::string& Bucket = BucketIt.first; - Cbo.BeginObject(); - { - Cbo.AddString("name", Bucket); - Cbo.BeginArray("values"); - { - for (const auto& ValueIt : BucketIt.second.Values) - { - std::chrono::seconds LastAccessedSeconds = std::chrono::duration_cast( - GcClock::TimePointFromTick(ValueIt.second.LastAccess).time_since_epoch()); - Cbo.BeginObject(); - { - Cbo.AddHash("key", ValueIt.first); - if (Details) - { - Cbo.AddInteger("size", ValueIt.second.Size); - if (ValueIt.second.Size > 0 && ValueIt.second.RawSize != 0 && - ValueIt.second.RawSize != ValueIt.second.Size) - { - Cbo.AddInteger("rawsize", ValueIt.second.RawSize); - Cbo.AddHash("rawhash", ValueIt.second.RawHash); - } - Cbo.AddString("contenttype", ToString(ValueIt.second.ContentType)); - Cbo.AddInteger("age", - gsl::narrow(NowSeconds.count() - LastAccessedSeconds.count())); - if (ValueIt.second.Attachments.size() > 0) - { - if (AttachmentDetails) - { - Cbo.BeginArray("attachments"); - { - for (const IoHash& Hash : ValueIt.second.Attachments) - { - Cbo.BeginObject(); - Cbo.AddHash("cid", Hash); - IoBuffer Payload = m_CidStore.FindChunkByCid(Hash); - Cbo.AddInteger("size", gsl::narrow(Payload.GetSize())); - Cbo.EndObject(); - } - } - Cbo.EndArray(); - } - else - { - Cbo.AddInteger("attachmentcount", - gsl::narrow(ValueIt.second.Attachments.size())); - size_t AttachmentsSize = 0; - for (const IoHash& Hash : ValueIt.second.Attachments) - { - IoBuffer Payload = m_CidStore.FindChunkByCid(Hash); - AttachmentsSize += Payload.GetSize(); - } - Cbo.AddInteger("attachmentssize", gsl::narrow(AttachmentsSize)); - } - } - } - } - Cbo.EndObject(); - } - } - Cbo.EndArray(); - } - Cbo.EndObject(); - } - } - Cbo.EndArray(); - } - Cbo.EndObject(); - } - } - Cbo.EndArray(); - Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); - } -} - -void -HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request) -{ - ZEN_TRACE_CPU("z$::Http::HandleRequest"); - - ZEN_MEMSCOPE(GetCacheHttpTag()); - - metrics::OperationTiming::Scope $(m_HttpRequests); - - const std::string_view Key = Request.RelativeUri(); - - std::string_view UriNamespace; - - if (Key.ends_with(HttpZCacheRPCPrefix)) - { - const size_t RpcOffset = Key.length() - HttpZCacheRPCPrefix.length(); - - if (RpcOffset) - { - std::string_view KeyPrefix = Key.substr(0, RpcOffset); - - if (KeyPrefix.back() == '/') - { - KeyPrefix.remove_suffix(1); - - UriNamespace = KeyPrefix; - } - } - - return HandleRpcRequest(Request, UriNamespace); - } - - if (Key == HttpZCacheUtilStartRecording) - { - HttpServerRequest::QueryParams Params = Request.GetQueryParams(); - - std::string RecordPath = UrlDecode(Params.GetValue("path")); - - { - RwLock::ExclusiveLockScope _(m_RequestRecordingLock); - m_RequestRecordingEnabled.store(false); - m_RequestRecorder.reset(); - - m_RequestRecorder = cache::MakeDiskRequestRecorder(RecordPath); - m_RequestRecordingEnabled.store(true); - } - ZEN_INFO("cache RPC recording STARTED -> '{}'", RecordPath); - Request.WriteResponse(HttpResponseCode::OK); - return; - } - - if (Key == HttpZCacheUtilStopRecording) - { - { - RwLock::ExclusiveLockScope _(m_RequestRecordingLock); - m_RequestRecordingEnabled.store(false); - m_RequestRecorder.reset(); - } - ZEN_INFO("cache RPC recording STOPPED"); - Request.WriteResponse(HttpResponseCode::OK); - return; - } - - if (Key == HttpZCacheUtilReplayRecording) - { - CacheRequestContext RequestContext = {.SessionId = Request.SessionId(), .RequestId = Request.RequestId()}; - - { - RwLock::ExclusiveLockScope _(m_RequestRecordingLock); - m_RequestRecordingEnabled.store(false); - m_RequestRecorder.reset(); - } - - HttpServerRequest::QueryParams Params = Request.GetQueryParams(); - - std::string RecordPath = UrlDecode(Params.GetValue("path")); - - uint32_t ThreadCount = GetHardwareConcurrency(); - if (auto Param = Params.GetValue("thread_count"); Param.empty() == false) - { - if (auto Value = ParseInt(Param)) - { - ThreadCount = gsl::narrow(Value.value()); - } - } - - ZEN_INFO("initiating cache RPC replay using {} threads, from '{}'", ThreadCount, RecordPath); - - std::unique_ptr Replayer(cache::MakeDiskRequestReplayer(RecordPath, false)); - ReplayRequestRecorder(RequestContext, *Replayer, ThreadCount < 1 ? 1 : ThreadCount); - - ZEN_INFO("cache RPC replay COMPLETED"); - - Request.WriteResponse(HttpResponseCode::OK); - return; - } - - if (Key.starts_with(HttpZCacheDetailsPrefix)) - { - HandleDetailsRequest(Request); - return; - } - - HttpCacheRequestData RequestData; - if (!HttpCacheRequestParseRelativeUri(Key, ZenCacheStore::DefaultNamespace, RequestData)) - { - m_CacheStats.BadRequestCount++; - return Request.WriteResponse(HttpResponseCode::BadRequest); // invalid URL - } - - if (RequestData.ValueContentId.has_value()) - { - ZEN_ASSERT(RequestData.Namespace.has_value()); - ZEN_ASSERT(RequestData.Bucket.has_value()); - ZEN_ASSERT(RequestData.HashKey.has_value()); - CacheRef Ref = {.Namespace = RequestData.Namespace.value(), - .BucketSegment = RequestData.Bucket.value(), - .HashKey = RequestData.HashKey.value(), - .ValueContentId = RequestData.ValueContentId.value()}; - return HandleCacheChunkRequest(Request, Ref, ParseCachePolicy(Request.GetQueryParams())); - } - - if (RequestData.HashKey.has_value()) - { - ZEN_ASSERT(RequestData.Namespace.has_value()); - ZEN_ASSERT(RequestData.Bucket.has_value()); - CacheRef Ref = {.Namespace = RequestData.Namespace.value(), - .BucketSegment = RequestData.Bucket.value(), - .HashKey = RequestData.HashKey.value(), - .ValueContentId = IoHash::Zero}; - return HandleCacheRecordRequest(Request, Ref, ParseCachePolicy(Request.GetQueryParams())); - } - - if (RequestData.Bucket.has_value()) - { - ZEN_ASSERT(RequestData.Namespace.has_value()); - return HandleCacheBucketRequest(Request, RequestData.Namespace.value(), RequestData.Bucket.value()); - } - - if (RequestData.Namespace.has_value()) - { - return HandleCacheNamespaceRequest(Request, RequestData.Namespace.value()); - } - return HandleCacheRequest(Request); -} - -void -HttpStructuredCacheService::HandleCacheRequest(HttpServerRequest& Request) -{ - switch (Request.RequestVerb()) - { - case HttpVerb::kHead: - case HttpVerb::kGet: - { - ZenCacheStore::Info Info = m_CacheStore.GetInfo(); - - CbObjectWriter ResponseWriter; - - ResponseWriter.BeginObject("Configuration"); - { - ExtendableStringBuilder<128> BasePathString; - BasePathString << Info.BasePath.u8string(); - ResponseWriter.AddString("BasePath"sv, BasePathString.ToView()); - ResponseWriter.AddBool("AllowAutomaticCreationOfNamespaces", Info.Config.AllowAutomaticCreationOfNamespaces); - ResponseWriter.BeginObject("Logging"); - { - ResponseWriter.AddBool("EnableWriteLog", Info.Config.Logging.EnableWriteLog); - ResponseWriter.AddBool("EnableAccessLog", Info.Config.Logging.EnableAccessLog); - } - ResponseWriter.EndObject(); - } - ResponseWriter.EndObject(); - - std::sort(begin(Info.NamespaceNames), end(Info.NamespaceNames), [](std::string_view L, std::string_view R) { - return L.compare(R) < 0; - }); - ResponseWriter.BeginArray("Namespaces"); - for (const std::string& NamespaceName : Info.NamespaceNames) - { - ResponseWriter.AddString(NamespaceName); - } - ResponseWriter.EndArray(); - ResponseWriter.BeginObject("StorageSize"); - { - ResponseWriter.AddInteger("DiskSize", Info.StorageSize.DiskSize); - ResponseWriter.AddInteger("MemorySize", Info.StorageSize.MemorySize); - } - - ResponseWriter.EndObject(); - - ResponseWriter.AddInteger("DiskEntryCount", Info.DiskEntryCount); - - return Request.WriteResponse(HttpResponseCode::OK, ResponseWriter.Save()); - } - break; - default: - m_CacheStats.BadRequestCount++; - break; - } -} - -void -HttpStructuredCacheService::HandleCacheNamespaceRequest(HttpServerRequest& Request, std::string_view NamespaceName) -{ - switch (Request.RequestVerb()) - { - case HttpVerb::kHead: - case HttpVerb::kGet: - { - std::optional Info = m_CacheStore.GetNamespaceInfo(NamespaceName); - if (!Info.has_value()) - { - return Request.WriteResponse(HttpResponseCode::NotFound); - } - - CbObjectWriter ResponseWriter; - - ResponseWriter.BeginObject("Configuration"); - { - ExtendableStringBuilder<128> BasePathString; - BasePathString << Info->RootDir.u8string(); - ResponseWriter.AddString("RootDir"sv, BasePathString.ToView()); - ResponseWriter.AddInteger("MaxBlockSize"sv, Info->Config.DiskLayerConfig.BucketConfig.MaxBlockSize); - ResponseWriter.AddInteger("PayloadAlignment"sv, Info->Config.DiskLayerConfig.BucketConfig.PayloadAlignment); - ResponseWriter.AddInteger("MemCacheSizeThreshold"sv, Info->Config.DiskLayerConfig.BucketConfig.MemCacheSizeThreshold); - ResponseWriter.AddInteger("LargeObjectThreshold"sv, Info->Config.DiskLayerConfig.BucketConfig.LargeObjectThreshold); - ResponseWriter.AddInteger("MemCacheTargetFootprintBytes"sv, Info->Config.DiskLayerConfig.MemCacheTargetFootprintBytes); - ResponseWriter.AddInteger("MemCacheTrimIntervalSeconds"sv, Info->Config.DiskLayerConfig.MemCacheTrimIntervalSeconds); - ResponseWriter.AddInteger("MemCacheMaxAgeSeconds"sv, Info->Config.DiskLayerConfig.MemCacheMaxAgeSeconds); - } - ResponseWriter.EndObject(); - - std::sort(begin(Info->BucketNames), end(Info->BucketNames), [](std::string_view L, std::string_view R) { - return L.compare(R) < 0; - }); - - ResponseWriter.BeginArray("Buckets"sv); - for (const std::string& BucketName : Info->BucketNames) - { - ResponseWriter.AddString(BucketName); - } - ResponseWriter.EndArray(); - - ResponseWriter.BeginObject("StorageSize"sv); - { - ResponseWriter.AddInteger("DiskSize"sv, Info->DiskLayerInfo.StorageSize.DiskSize); - ResponseWriter.AddInteger("MemorySize"sv, Info->DiskLayerInfo.StorageSize.MemorySize); - } - ResponseWriter.EndObject(); - - ResponseWriter.AddInteger("EntryCount", Info->DiskLayerInfo.EntryCount); - - if (auto Buckets = HttpServerRequest::Decode(Request.GetQueryParams().GetValue("bucketsizes")); !Buckets.empty()) - { - ResponseWriter.BeginObject("BucketSizes"); - - ResponseWriter.BeginArray("Buckets"); - - std::vector BucketNames; - if (Buckets == "*") // Get all - empty FieldFilter equal getting all fields - { - BucketNames = Info.value().BucketNames; - } - else - { - ForEachStrTok(Buckets, ',', [&](std::string_view BucketName) { - BucketNames.push_back(std::string(BucketName)); - return true; - }); - } - WorkerThreadPool& WorkerPool = GetMediumWorkerPool(EWorkloadType::Background); - std::vector AllAttachments; - for (const std::string& BucketName : BucketNames) - { - ResponseWriter.BeginObject(); - ResponseWriter << "Name" << BucketName; - CacheContentStats ContentStats; - bool Success = m_CacheStore.GetContentStats(NamespaceName, BucketName, ContentStats); - if (Success) - { - size_t ValuesSize = 0; - for (const uint64_t Size : ContentStats.ValueSizes) - { - ValuesSize += Size; - } - - std::sort(ContentStats.Attachments.begin(), ContentStats.Attachments.end()); - auto NewEnd = std::unique(ContentStats.Attachments.begin(), ContentStats.Attachments.end()); - ContentStats.Attachments.erase(NewEnd, ContentStats.Attachments.end()); - - ResponseWriter << "Count" << ContentStats.ValueSizes.size(); - ResponseWriter << "StructuredCount" << ContentStats.StructuredValuesCount; - ResponseWriter << "StandaloneCount" << ContentStats.StandaloneValuesCount; - ResponseWriter << "Size" << ValuesSize; - ResponseWriter << "AttachmentCount" << ContentStats.Attachments.size(); - - AllAttachments.insert(AllAttachments.end(), ContentStats.Attachments.begin(), ContentStats.Attachments.end()); - } - ResponseWriter.EndObject(); - } - - ResponseWriter.EndArray(); - - ResponseWriter.BeginObject("Attachments"); - std::sort(AllAttachments.begin(), AllAttachments.end()); - auto NewEnd = std::unique(AllAttachments.begin(), AllAttachments.end()); - AllAttachments.erase(NewEnd, AllAttachments.end()); - - uint64_t AttachmentsSize = 0; - - m_CidStore.IterateChunks( - AllAttachments, - [&](size_t Index, const IoBuffer& Payload) { - ZEN_UNUSED(Index); - AttachmentsSize += Payload.GetSize(); - return true; - }, - &WorkerPool, - 8u * 1024u); - - ResponseWriter << "Count" << AllAttachments.size(); - ResponseWriter << "Size" << AttachmentsSize; - - ResponseWriter.EndObject(); - - ResponseWriter.EndObject(); - } - - return Request.WriteResponse(HttpResponseCode::OK, ResponseWriter.Save()); - } - break; - - case HttpVerb::kDelete: - // Drop namespace - { - if (m_CacheStore.DropNamespace(NamespaceName)) - { - return Request.WriteResponse(HttpResponseCode::OK); - } - else - { - return Request.WriteResponse(HttpResponseCode::NotFound); - } - } - break; - - default: - break; - } -} - -void -HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, - std::string_view NamespaceName, - std::string_view BucketName) -{ - switch (Request.RequestVerb()) - { - case HttpVerb::kHead: - case HttpVerb::kGet: - { - std::optional Info = m_CacheStore.GetBucketInfo(NamespaceName, BucketName); - if (!Info.has_value()) - { - return Request.WriteResponse(HttpResponseCode::NotFound); - } - - CbObjectWriter ResponseWriter; - - ResponseWriter.BeginObject("StorageSize"); - { - ResponseWriter.AddInteger("DiskSize", Info->DiskLayerInfo.StorageSize.DiskSize); - ResponseWriter.AddInteger("MemorySize", Info->DiskLayerInfo.StorageSize.MemorySize); - } - ResponseWriter.EndObject(); - - ResponseWriter.AddInteger("DiskEntryCount", Info->DiskLayerInfo.EntryCount); - - if (auto GetBucketSize = Request.GetQueryParams().GetValue("bucketsize"); GetBucketSize == "true") - { - CacheContentStats ContentStats; - bool Success = m_CacheStore.GetContentStats(NamespaceName, BucketName, ContentStats); - if (Success) - { - size_t ValuesSize = 0; - for (const uint64_t Size : ContentStats.ValueSizes) - { - ValuesSize += Size; - } - - std::sort(ContentStats.Attachments.begin(), ContentStats.Attachments.end()); - auto NewEnd = std::unique(ContentStats.Attachments.begin(), ContentStats.Attachments.end()); - ContentStats.Attachments.erase(NewEnd, ContentStats.Attachments.end()); - - ResponseWriter << "Count" << ContentStats.ValueSizes.size(); - ResponseWriter << "StructuredCount" << ContentStats.StructuredValuesCount; - ResponseWriter << "StandaloneCount" << ContentStats.StandaloneValuesCount; - ResponseWriter << "Size" << ValuesSize; - ResponseWriter << "AttachmentCount" << ContentStats.Attachments.size(); - - uint64_t AttachmentsSize = 0; - - WorkerThreadPool& WorkerPool = GetMediumWorkerPool(EWorkloadType::Background); - - m_CidStore.IterateChunks( - ContentStats.Attachments, - [&](size_t Index, const IoBuffer& Payload) { - ZEN_UNUSED(Index); - AttachmentsSize += Payload.GetSize(); - return true; - }, - &WorkerPool, - 8u * 1024u); - - ResponseWriter << "AttachmentsSize" << AttachmentsSize; - } - } - - return Request.WriteResponse(HttpResponseCode::OK, ResponseWriter.Save()); - } - break; - - case HttpVerb::kDelete: - // Drop bucket - { - if (m_CacheStore.DropBucket(NamespaceName, BucketName)) - { - return Request.WriteResponse(HttpResponseCode::OK); - } - else - { - return Request.WriteResponse(HttpResponseCode::NotFound); - } - } - break; - - default: - break; - } -} - -void -HttpStructuredCacheService::HandleCacheRecordRequest(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl) -{ - switch (Request.RequestVerb()) - { - case HttpVerb::kHead: - case HttpVerb::kGet: - HandleGetCacheRecord(Request, Ref, PolicyFromUrl); - break; - - case HttpVerb::kPut: - HandlePutCacheRecord(Request, Ref, PolicyFromUrl); - break; - - default: - break; - } -} - -void -HttpStructuredCacheService::HandleGetCacheRecord(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl) -{ - const ZenContentType AcceptType = Request.AcceptContentType(); - const bool SkipData = EnumHasAllFlags(PolicyFromUrl, CachePolicy::SkipData); - const bool PartialRecord = EnumHasAllFlags(PolicyFromUrl, CachePolicy::PartialRecord); - - bool Success = false; - uint32_t MissingCount = 0; - ZenCacheValue ClientResultValue; - if (!EnumHasAnyFlags(PolicyFromUrl, CachePolicy::Query)) - { - return Request.WriteResponse(HttpResponseCode::OK); - } - - const bool HasUpstream = m_UpstreamCache.IsActive(); - - CacheRequestContext RequestContext = {.SessionId = Request.SessionId(), .RequestId = Request.RequestId()}; - Stopwatch Timer; - - if (EnumHasAllFlags(PolicyFromUrl, CachePolicy::QueryLocal) && - m_CacheStore.Get(RequestContext, Ref.Namespace, Ref.BucketSegment, Ref.HashKey, ClientResultValue)) - { - Success = true; - ZenContentType ContentType = ClientResultValue.Value.GetContentType(); - - if (AcceptType == ZenContentType::kCbPackage) - { - if (ContentType == ZenContentType::kCbObject) - { - CbPackage Package; - CbValidateError ValidateError = CbValidateError::None; - if (CbObject PackageObject = ValidateAndReadCompactBinaryObject(std::move(ClientResultValue.Value), ValidateError); - ValidateError == CbValidateError::None) - { - CbObjectView CacheRecord(ClientResultValue.Value.Data()); - CacheRecord.IterateAttachments([this, &MissingCount, &Package, SkipData](CbFieldView AttachmentHash) { - if (SkipData) - { - if (!m_CidStore.ContainsChunk(AttachmentHash.AsHash())) - { - MissingCount++; - } - } - else - { - if (IoBuffer Chunk = m_CidStore.FindChunkByCid(AttachmentHash.AsHash())) - { - CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(Chunk)); - if (Compressed) - { - Package.AddAttachment(CbAttachment(Compressed, AttachmentHash.AsHash())); - } - else - { - ZEN_WARN("invalid compressed binary returned for {}", AttachmentHash.AsHash()); - MissingCount++; - } - } - else - { - MissingCount++; - } - } - }); - - Success = MissingCount == 0 || PartialRecord; - } - else - { - ZEN_WARN("Invalid compact binary payload returned for {}/{}/{} ({}). Reason: '{}'", - Ref.Namespace, - Ref.BucketSegment, - Ref.HashKey, - Ref.ValueContentId, - ToString(ValidateError)); - Success = false; - } - - if (Success) - { - CbObject PackageObject = LoadCompactBinaryObject(std::move(ClientResultValue.Value)); - - Package.SetObject(std::move(PackageObject)); - - BinaryWriter MemStream; - Package.Save(MemStream); - - ClientResultValue.Value = IoBuffer(IoBuffer::Clone, MemStream.Data(), MemStream.Size()); - ClientResultValue.Value.SetContentType(HttpContentType::kCbPackage); - } - } - else - { - Success = false; - } - } - else if (AcceptType != ClientResultValue.Value.GetContentType() && AcceptType != ZenContentType::kUnknownContentType && - AcceptType != ZenContentType::kBinary) - { - Success = false; - } - } - - if (Success) - { - ZEN_DEBUG("GETCACHERECORD HIT - '{}/{}/{}' {} '{}' (LOCAL) in {}", - Ref.Namespace, - Ref.BucketSegment, - Ref.HashKey, - NiceBytes(ClientResultValue.Value.Size()), - ToString(ClientResultValue.Value.GetContentType()), - NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); - - m_CacheStats.HitCount++; - if (SkipData && AcceptType != ZenContentType::kCbPackage && AcceptType != ZenContentType::kCbObject) - { - return Request.WriteResponse(HttpResponseCode::OK); - } - else - { - // kCbPackage handled SkipData when constructing the ClientResultValue, kcbObject ignores SkipData - return Request.WriteResponse((MissingCount == 0) ? HttpResponseCode::OK : HttpResponseCode::PartialContent, - ClientResultValue.Value.GetContentType(), - ClientResultValue.Value); - } - } - else if (!HasUpstream || !EnumHasAllFlags(PolicyFromUrl, CachePolicy::QueryRemote)) - { - ZEN_DEBUG("GETCACHERECORD MISS - '{}/{}/{}' '{}' in {}", - Ref.Namespace, - Ref.BucketSegment, - Ref.HashKey, - ToString(AcceptType), - NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); - m_CacheStats.MissCount++; - return Request.WriteResponse(HttpResponseCode::NotFound); - } - - // Issue upstream query asynchronously in order to keep requests flowing without - // hogging I/O servicing threads with blocking work - - uint64_t LocalElapsedTimeUs = Timer.GetElapsedTimeUs(); - - Request.WriteResponseAsync([this, AcceptType, PolicyFromUrl, Ref, LocalElapsedTimeUs, RequestContext](HttpServerRequest& AsyncRequest) { - Stopwatch Timer; - bool Success = false; - const bool PartialRecord = EnumHasAllFlags(PolicyFromUrl, CachePolicy::PartialRecord); - const bool QueryLocal = EnumHasAllFlags(PolicyFromUrl, CachePolicy::QueryLocal); - const bool StoreLocal = EnumHasAllFlags(PolicyFromUrl, CachePolicy::StoreLocal) && AreDiskWritesAllowed(); - const bool SkipData = EnumHasAllFlags(PolicyFromUrl, CachePolicy::SkipData); - ZenCacheValue ClientResultValue; - - metrics::OperationTiming::Scope $(m_UpstreamGetRequestTiming); - - if (GetUpstreamCacheSingleResult UpstreamResult = - m_UpstreamCache.GetCacheRecord(Ref.Namespace, {Ref.BucketSegment, Ref.HashKey}, AcceptType); - UpstreamResult.Status.Success) - { - Success = true; - - ClientResultValue.Value = UpstreamResult.Value; - ClientResultValue.Value.SetContentType(AcceptType); - - if (AcceptType == ZenContentType::kBinary || AcceptType == ZenContentType::kCbObject) - { - if (AcceptType == ZenContentType::kCbObject) - { - const CbValidateError ValidationResult = ValidateCompactBinary(UpstreamResult.Value, CbValidateMode::All); - if (ValidationResult != CbValidateError::None) - { - Success = false; - ZEN_WARN("Get - '{}/{}/{}' '{}' FAILED, invalid compact binary object from upstream", - Ref.Namespace, - Ref.BucketSegment, - Ref.HashKey, - ToString(AcceptType)); - } - - // We do not do anything to the returned object for SkipData, only package attachments are cut when skipping data - } - - if (Success && StoreLocal) - { - const bool Overwrite = !EnumHasAllFlags(PolicyFromUrl, CachePolicy::QueryLocal); - ZenCacheStore::PutResult PutResult = - m_CacheStore - .Put(RequestContext, Ref.Namespace, Ref.BucketSegment, Ref.HashKey, ClientResultValue, {}, Overwrite, nullptr); - if (PutResult.Status == zen::PutStatus::Success) - { - m_CacheStats.WriteCount++; - } - } - } - else if (AcceptType == ZenContentType::kCbPackage) - { - CbPackage Package; - if (Package.TryLoad(ClientResultValue.Value)) - { - CbObject CacheRecord = Package.GetObject(); - AttachmentCount Count; - size_t NumAttachments = Package.GetAttachments().size(); - std::vector ReferencedAttachments; - std::vector WriteAttachmentBuffers; - WriteAttachmentBuffers.reserve(NumAttachments); - std::vector WriteRawHashes; - WriteRawHashes.reserve(NumAttachments); - - CacheRecord.IterateAttachments([this, - &Package, - &Ref, - &WriteAttachmentBuffers, - &WriteRawHashes, - &ReferencedAttachments, - &Count, - QueryLocal, - StoreLocal, - SkipData](CbFieldView HashView) { - IoHash Hash = HashView.AsHash(); - ReferencedAttachments.push_back(Hash); - if (const CbAttachment* Attachment = Package.FindAttachment(Hash)) - { - if (Attachment->IsCompressedBinary()) - { - if (StoreLocal) - { - const CompressedBuffer& Chunk = Attachment->AsCompressedBinary(); - WriteAttachmentBuffers.push_back(Chunk.GetCompressed().Flatten().AsIoBuffer()); - WriteRawHashes.push_back(Attachment->GetHash()); - } - Count.Valid++; - } - else - { - ZEN_WARN("Uncompressed value '{}' from upstream cache record '{}/{}'", - Hash, - Ref.BucketSegment, - Ref.HashKey); - Count.Invalid++; - } - } - else if (QueryLocal) - { - if (SkipData) - { - if (m_CidStore.ContainsChunk(Hash)) - { - Count.Valid++; - } - } - else if (IoBuffer Chunk = m_CidStore.FindChunkByCid(Hash)) - { - CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(Chunk)); - if (Compressed) - { - Package.AddAttachment(CbAttachment(Compressed, Hash)); - Count.Valid++; - } - else - { - ZEN_WARN("Uncompressed value '{}' stored in local cache '{}/{}'", Hash, Ref.BucketSegment, Ref.HashKey); - Count.Invalid++; - } - } - } - Count.Total++; - }); - - if ((Count.Valid == Count.Total) || PartialRecord) - { - ZenCacheValue CacheValue; - CacheValue.Value = CacheRecord.GetBuffer().AsIoBuffer(); - CacheValue.Value.SetContentType(ZenContentType::kCbObject); - - if (StoreLocal) - { - const bool Overwrite = !EnumHasAllFlags(PolicyFromUrl, CachePolicy::QueryLocal); - ZenCacheStore::PutResult PutResult = m_CacheStore.Put(RequestContext, - Ref.Namespace, - Ref.BucketSegment, - Ref.HashKey, - CacheValue, - ReferencedAttachments, - Overwrite, - nullptr); - if (PutResult.Status == zen::PutStatus::Success) - { - m_CacheStats.WriteCount++; - - if (!WriteAttachmentBuffers.empty()) - { - std::vector InsertResults = - m_CidStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes); - for (const CidStore::InsertResult& Result : InsertResults) - { - if (Result.New) - { - Count.New++; - } - } - } - - WriteAttachmentBuffers = {}; - WriteRawHashes = {}; - } - } - - BinaryWriter MemStream; - if (SkipData) - { - // Save a package containing only the object. - CbPackage(Package.GetObject()).Save(MemStream); - } - else - { - Package.Save(MemStream); - } - - ClientResultValue.Value = IoBuffer(IoBuffer::Clone, MemStream.Data(), MemStream.Size()); - ClientResultValue.Value.SetContentType(ZenContentType::kCbPackage); - } - else - { - Success = false; - ZEN_WARN("Get - '{}/{}' '{}' FAILED, attachments missing in upstream package", - Ref.BucketSegment, - Ref.HashKey, - ToString(AcceptType)); - } - } - else - { - Success = false; - ZEN_WARN("Get - '{}/{}/{}' '{}' FAILED, invalid upstream package", - Ref.Namespace, - Ref.BucketSegment, - Ref.HashKey, - ToString(AcceptType)); - } - } - } - - if (Success) - { - ZEN_DEBUG("GETCACHERECORD HIT - '{}/{}/{}' {} '{}' (UPSTREAM) in {}", - Ref.Namespace, - Ref.BucketSegment, - Ref.HashKey, - NiceBytes(ClientResultValue.Value.Size()), - ToString(ClientResultValue.Value.GetContentType()), - NiceLatencyNs((LocalElapsedTimeUs + Timer.GetElapsedTimeUs()) * 1000)); - - m_CacheStats.HitCount++; - m_CacheStats.UpstreamHitCount++; - - if (SkipData && AcceptType == ZenContentType::kBinary) - { - AsyncRequest.WriteResponse(HttpResponseCode::OK); - } - else - { - // Other methods modify ClientResultValue to a version that has skipped the data but keeps the Object and optionally - // metadata. - AsyncRequest.WriteResponse(HttpResponseCode::OK, ClientResultValue.Value.GetContentType(), ClientResultValue.Value); - } - } - else - { - ZEN_DEBUG("GETCACHERECORD MISS - '{}/{}/{}' '{}' in {}", - Ref.Namespace, - Ref.BucketSegment, - Ref.HashKey, - ToString(AcceptType), - NiceLatencyNs((LocalElapsedTimeUs + Timer.GetElapsedTimeUs()) * 1000)); - m_CacheStats.MissCount++; - AsyncRequest.WriteResponse(HttpResponseCode::NotFound); - } - }); -} - -void -HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl) -{ - IoBuffer Body = Request.ReadPayload(); - - if (!Body || Body.Size() == 0) - { - m_CacheStats.BadRequestCount++; - return Request.WriteResponse(HttpResponseCode::BadRequest); - } - if (!AreDiskWritesAllowed()) - { - return Request.WriteResponse(HttpResponseCode::InsufficientStorage); - } - - auto WriteFailureResponse = [&Request](const ZenCacheStore::PutResult& PutResult) { - ZEN_UNUSED(PutResult); - - HttpResponseCode ResponseCode = HttpResponseCode::InternalServerError; - switch (PutResult.Status) - { - case zen::PutStatus::Conflict: - ResponseCode = HttpResponseCode::Conflict; - break; - case zen::PutStatus::Invalid: - ResponseCode = HttpResponseCode::BadRequest; - break; - } - - if (PutResult.Details) - { - Request.WriteResponse(ResponseCode, PutResult.Details); - } - return Request.WriteResponse(ResponseCode); - }; - - const HttpContentType ContentType = Request.RequestContentType(); - - Body.SetContentType(ContentType); - - CacheRequestContext RequestContext = {.SessionId = Request.SessionId(), .RequestId = Request.RequestId()}; - - const bool HasUpstream = m_UpstreamCache.IsActive(); - - Stopwatch Timer; - - if (ContentType == HttpContentType::kBinary || ContentType == HttpContentType::kCompressedBinary) - { - IoHash RawHash = IoHash::Zero; - uint64_t RawSize = Body.GetSize(); - if (ContentType == HttpContentType::kCompressedBinary) - { - if (!CompressedBuffer::ValidateCompressedHeader(Body, RawHash, RawSize)) - { - m_CacheStats.BadRequestCount++; - return Request.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - "Payload is not a valid compressed binary"sv); - } - } - else - { - RawHash = IoHash::HashBuffer(SharedBuffer(Body)); - } - const bool Overwrite = !EnumHasAllFlags(PolicyFromUrl, CachePolicy::QueryLocal); - // TODO: Propagation for rejected PUTs - ZenCacheStore::PutResult PutResult = m_CacheStore.Put(RequestContext, - Ref.Namespace, - Ref.BucketSegment, - Ref.HashKey, - {.Value = Body, .RawSize = RawSize, .RawHash = RawHash}, - {}, - Overwrite, - nullptr); - if (PutResult.Status != zen::PutStatus::Success) - { - return WriteFailureResponse(PutResult); - } - m_CacheStats.WriteCount++; - - if (HasUpstream && EnumHasAllFlags(PolicyFromUrl, CachePolicy::StoreRemote)) - { - m_UpstreamCache.EnqueueUpstream({.Type = ContentType, .Namespace = Ref.Namespace, .Key = {Ref.BucketSegment, Ref.HashKey}}); - } - - ZEN_DEBUG("PUTCACHERECORD - '{}/{}/{}' {} '{}' in {}", - Ref.Namespace, - Ref.BucketSegment, - Ref.HashKey, - NiceBytes(Body.Size()), - ToString(ContentType), - NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); - Request.WriteResponse(HttpResponseCode::Created); - } - else if (ContentType == HttpContentType::kCbObject) - { - const CbValidateError ValidationResult = ValidateCompactBinary(MemoryView(Body.GetData(), Body.GetSize()), CbValidateMode::All); - - if (ValidationResult != CbValidateError::None) - { - ZEN_WARN("PUTCACHERECORD - '{}/{}/{}' '{}' FAILED, invalid compact binary", - Ref.Namespace, - Ref.BucketSegment, - Ref.HashKey, - ToString(ContentType)); - m_CacheStats.BadRequestCount++; - return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Compact binary validation failed"sv); - } - - Body.SetContentType(ZenContentType::kCbObject); - - CbObjectView CacheRecord(Body.Data()); - std::vector ValidAttachments; - std::vector ReferencedAttachments; - int32_t TotalCount = 0; - - CacheRecord.IterateAttachments([this, &TotalCount, &ValidAttachments, &ReferencedAttachments](CbFieldView AttachmentHash) { - const IoHash Hash = AttachmentHash.AsHash(); - ReferencedAttachments.push_back(Hash); - if (m_CidStore.ContainsChunk(Hash)) - { - ValidAttachments.emplace_back(Hash); - } - TotalCount++; - }); - - const bool Overwrite = !EnumHasAllFlags(PolicyFromUrl, CachePolicy::QueryLocal); - - // TODO: Propagation for rejected PUTs - ZenCacheStore::PutResult PutResult = m_CacheStore.Put(RequestContext, - Ref.Namespace, - Ref.BucketSegment, - Ref.HashKey, - {.Value = Body}, - ReferencedAttachments, - Overwrite, - nullptr); - if (PutResult.Status != zen::PutStatus::Success) - { - return WriteFailureResponse(PutResult); - } - m_CacheStats.WriteCount++; - - ZEN_DEBUG("PUTCACHERECORD - '{}/{}/{}' {} '{}' attachments '{}/{}' (valid/total) in {}", - Ref.Namespace, - Ref.BucketSegment, - Ref.HashKey, - NiceBytes(Body.Size()), - ToString(ContentType), - TotalCount, - ValidAttachments.size(), - NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); - - const bool IsPartialRecord = TotalCount != static_cast(ValidAttachments.size()); - - CachePolicy Policy = PolicyFromUrl; - if (HasUpstream && EnumHasAllFlags(Policy, CachePolicy::StoreRemote) && !IsPartialRecord) - { - m_UpstreamCache.EnqueueUpstream({.Type = ZenContentType::kCbObject, - .Namespace = Ref.Namespace, - .Key = {Ref.BucketSegment, Ref.HashKey}, - .ValueContentIds = std::move(ValidAttachments)}); - } - - Request.WriteResponse(HttpResponseCode::Created); - } - else if (ContentType == HttpContentType::kCbPackage) - { - CbPackage Package; - - if (!Package.TryLoad(Body)) - { - ZEN_WARN("PUTCACHERECORD - '{}/{}/{}' '{}' FAILED, invalid package", - Ref.Namespace, - Ref.BucketSegment, - Ref.HashKey, - ToString(ContentType)); - m_CacheStats.BadRequestCount++; - return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid package"sv); - } - CachePolicy Policy = PolicyFromUrl; - - CbObject CacheRecord = Package.GetObject(); - - AttachmentCount Count; - size_t NumAttachments = Package.GetAttachments().size(); - std::vector ValidAttachments; - std::vector ReferencedAttachments; - ValidAttachments.reserve(NumAttachments); - std::vector WriteAttachmentBuffers; - std::vector WriteRawHashes; - WriteAttachmentBuffers.reserve(NumAttachments); - WriteRawHashes.reserve(NumAttachments); - - CacheRecord.IterateAttachments( - [this, &Ref, &Package, &WriteAttachmentBuffers, &WriteRawHashes, &ValidAttachments, &ReferencedAttachments, &Count]( - CbFieldView HashView) { - const IoHash Hash = HashView.AsHash(); - ReferencedAttachments.push_back(Hash); - if (const CbAttachment* Attachment = Package.FindAttachment(Hash)) - { - if (Attachment->IsCompressedBinary()) - { - WriteAttachmentBuffers.emplace_back(Attachment->AsCompressedBinary().GetCompressed().Flatten().AsIoBuffer()); - WriteRawHashes.push_back(Hash); - ValidAttachments.emplace_back(Hash); - Count.Valid++; - } - else - { - ZEN_WARN("PUTCACHERECORD - '{}/{}/{}' '{}' FAILED, attachment '{}' is not compressed", - Ref.Namespace, - Ref.BucketSegment, - Ref.HashKey, - ToString(HttpContentType::kCbPackage), - Hash); - Count.Invalid++; - } - } - else if (m_CidStore.ContainsChunk(Hash)) - { - ValidAttachments.emplace_back(Hash); - Count.Valid++; - } - Count.Total++; - }); - - if (Count.Invalid > 0) - { - m_CacheStats.BadRequestCount++; - return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid attachment(s)"sv); - } - - const bool Overwrite = !EnumHasAllFlags(Policy, CachePolicy::QueryLocal); - - ZenCacheValue CacheValue; - CacheValue.Value = CacheRecord.GetBuffer().AsIoBuffer(); - CacheValue.Value.SetContentType(ZenContentType::kCbObject); - // TODO: Propagation for rejected PUTs - ZenCacheStore::PutResult PutResult = - m_CacheStore.Put(RequestContext, Ref.Namespace, Ref.BucketSegment, Ref.HashKey, CacheValue, ReferencedAttachments, Overwrite); - if (PutResult.Status != zen::PutStatus::Success) - { - return WriteFailureResponse(PutResult); - } - m_CacheStats.WriteCount++; - - if (!WriteAttachmentBuffers.empty()) - { - std::vector InsertResults = m_CidStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes); - for (const CidStore::InsertResult& InsertResult : InsertResults) - { - if (InsertResult.New) - { - Count.New++; - } - } - WriteAttachmentBuffers = {}; - WriteRawHashes = {}; - } - - ZEN_DEBUG("PUTCACHERECORD - '{}/{}/{}' {} '{}', attachments '{}/{}/{}' (new/valid/total) in {}", - Ref.Namespace, - Ref.BucketSegment, - Ref.HashKey, - NiceBytes(Body.GetSize()), - ToString(ContentType), - Count.New, - Count.Valid, - Count.Total, - NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); - - const bool IsPartialRecord = Count.Valid != Count.Total; - - if (HasUpstream && EnumHasAllFlags(Policy, CachePolicy::StoreRemote) && !IsPartialRecord) - { - m_UpstreamCache.EnqueueUpstream({.Type = ZenContentType::kCbPackage, - .Namespace = Ref.Namespace, - .Key = {Ref.BucketSegment, Ref.HashKey}, - .ValueContentIds = std::move(ValidAttachments)}); - } - - Request.WriteResponse(HttpResponseCode::Created); - } - else - { - m_CacheStats.BadRequestCount++; - return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Content-Type invalid"sv); - } -} - -void -HttpStructuredCacheService::HandleCacheChunkRequest(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl) -{ - switch (Request.RequestVerb()) - { - case HttpVerb::kHead: - case HttpVerb::kGet: - HandleGetCacheChunk(Request, Ref, PolicyFromUrl); - break; - case HttpVerb::kPut: - HandlePutCacheChunk(Request, Ref, PolicyFromUrl); - break; - default: - break; - } -} - -void -HttpStructuredCacheService::HandleGetCacheChunk(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl) -{ - Stopwatch Timer; - - IoBuffer Value = m_CidStore.FindChunkByCid(Ref.ValueContentId); - const UpstreamEndpointInfo* Source = nullptr; - CachePolicy Policy = PolicyFromUrl; - - const bool HasUpstream = m_UpstreamCache.IsActive(); - { - const bool QueryUpstream = HasUpstream && !Value && EnumHasAllFlags(Policy, CachePolicy::QueryRemote); - - if (QueryUpstream) - { - if (GetUpstreamCacheSingleResult UpstreamResult = - m_UpstreamCache.GetCacheChunk(Ref.Namespace, {Ref.BucketSegment, Ref.HashKey}, Ref.ValueContentId); - UpstreamResult.Status.Success) - { - IoHash RawHash; - uint64_t RawSize; - if (CompressedBuffer::ValidateCompressedHeader(UpstreamResult.Value, RawHash, RawSize)) - { - if (RawHash == Ref.ValueContentId) - { - if (AreDiskWritesAllowed()) - { - m_CidStore.AddChunk(UpstreamResult.Value, RawHash); - } - Source = UpstreamResult.Source; - } - else - { - ZEN_WARN("got missmatching upstream cache value"); - } - } - else - { - ZEN_WARN("got uncompressed upstream cache value"); - } - } - } - } - - if (!Value) - { - ZEN_DEBUG("GETCACHECHUNK MISS - '{}/{}/{}/{}' '{}' in {}", - Ref.Namespace, - Ref.BucketSegment, - Ref.HashKey, - Ref.ValueContentId, - ToString(Request.AcceptContentType()), - NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); - m_CacheStats.MissCount++; - return Request.WriteResponse(HttpResponseCode::NotFound); - } - - ZEN_DEBUG("GETCACHECHUNK HIT - '{}/{}/{}/{}' {} '{}' ({}) in {}", - Ref.Namespace, - Ref.BucketSegment, - Ref.HashKey, - Ref.ValueContentId, - NiceBytes(Value.Size()), - ToString(Value.GetContentType()), - Source ? Source->Url : "LOCAL"sv, - NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); - - m_CacheStats.HitCount++; - if (Source) - { - m_CacheStats.UpstreamHitCount++; - } - - if (EnumHasAllFlags(Policy, CachePolicy::SkipData)) - { - Request.WriteResponse(HttpResponseCode::OK); - } - else - { - Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, Value); - } -} - -void -HttpStructuredCacheService::HandlePutCacheChunk(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl) -{ - // Note: Individual cacherecord values are not propagated upstream until a valid cache record has been stored - ZEN_UNUSED(PolicyFromUrl); - - Stopwatch Timer; - - IoBuffer Body = Request.ReadPayload(); - - if (!Body || Body.Size() == 0) - { - m_CacheStats.BadRequestCount++; - return Request.WriteResponse(HttpResponseCode::BadRequest); - } - if (!AreDiskWritesAllowed()) - { - return Request.WriteResponse(HttpResponseCode::InsufficientStorage); - } - - Body.SetContentType(Request.RequestContentType()); - - IoHash RawHash; - uint64_t RawSize; - if (!CompressedBuffer::ValidateCompressedHeader(Body, RawHash, RawSize)) - { - m_CacheStats.BadRequestCount++; - return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Attachments must be compressed"sv); - } - - if (RawHash != Ref.ValueContentId) - { - m_CacheStats.BadRequestCount++; - return Request.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - "ValueContentId does not match attachment hash"sv); - } - - CidStore::InsertResult Result = m_CidStore.AddChunk(Body, RawHash); - - ZEN_DEBUG("PUTCACHECHUNK - '{}/{}/{}/{}' {} '{}' ({}) in {}", - Ref.Namespace, - Ref.BucketSegment, - Ref.HashKey, - Ref.ValueContentId, - NiceBytes(Body.Size()), - ToString(Body.GetContentType()), - Result.New ? "NEW" : "OLD", - NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); - - const HttpResponseCode ResponseCode = Result.New ? HttpResponseCode::Created : HttpResponseCode::OK; - - Request.WriteResponse(ResponseCode); -} - -void -HttpStructuredCacheService::ReplayRequestRecorder(const CacheRequestContext& Context, - cache::IRpcRequestReplayer& Replayer, - uint32_t ThreadCount) -{ - WorkerThreadPool WorkerPool(ThreadCount); - uint64_t RequestCount = Replayer.GetRequestCount(); - Stopwatch Timer; - auto _ = MakeGuard([&]() { ZEN_INFO("Replayed {} requests in {}", RequestCount, NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); }); - std::atomic AbortFlag; - std::atomic PauseFlag; - ParallelWork Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::EnableBacklog); - ZEN_INFO("Replaying {} requests", RequestCount); - for (uint64_t RequestIndex = 0; RequestIndex < RequestCount; ++RequestIndex) - { - if (AbortFlag) - { - break; - } - Work.ScheduleWork(WorkerPool, [this, &Context, &Replayer, RequestIndex](std::atomic& AbortFlag) { - IoBuffer Body; - zen::cache::RecordedRequestInfo RequestInfo = Replayer.GetRequest(RequestIndex, /* out */ Body); - - if (AbortFlag) - { - return; - } - - if (Body) - { - uint32_t AcceptMagic = 0; - RpcAcceptOptions AcceptFlags = RpcAcceptOptions::kNone; - int TargetPid = 0; - CbPackage RpcResult; - if (m_RpcHandler.HandleRpcRequest(Context, - /* UriNamespace */ {}, - RequestInfo.ContentType, - std::move(Body), - AcceptMagic, - AcceptFlags, - TargetPid, - RpcResult) == CacheRpcHandler::RpcResponseCode::OK) - { - if (AcceptMagic == kCbPkgMagic) - { - void* TargetProcessHandle = nullptr; - FormatFlags Flags = FormatFlags::kDefault; - if (EnumHasAllFlags(AcceptFlags, RpcAcceptOptions::kAllowLocalReferences)) - { - Flags |= FormatFlags::kAllowLocalReferences; - if (!EnumHasAnyFlags(AcceptFlags, RpcAcceptOptions::kAllowPartialLocalReferences)) - { - Flags |= FormatFlags::kDenyPartialLocalReferences; - } - TargetProcessHandle = m_OpenProcessCache.GetProcessHandle(Context.SessionId, TargetPid); - } - CompositeBuffer RpcResponseBuffer = FormatPackageMessageBuffer(RpcResult, Flags, TargetProcessHandle); - ZEN_ASSERT(RpcResponseBuffer.GetSize() > 0); - } - else - { - BinaryWriter MemStream; - RpcResult.Save(MemStream); - IoBuffer RpcResponseBuffer(IoBuffer::Wrap, MemStream.GetData(), MemStream.GetSize()); - ZEN_ASSERT(RpcResponseBuffer.Size() > 0); - } - } - } - }); - } - Work.Wait(10000, [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) { - ZEN_UNUSED(IsAborted, IsPaused); - ZEN_INFO("Replayed {} of {} requests, elapsed {}", - RequestCount - PendingWork, - RequestCount, - NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); - }); -} - -void -HttpStructuredCacheService::HandleRpcRequest(HttpServerRequest& Request, std::string_view UriNamespace) -{ - ZEN_MEMSCOPE(GetCacheRpcTag()); - - ZEN_TRACE_CPU("z$::Http::HandleRpcRequest"); - - const bool HasUpstream = m_UpstreamCache.IsActive(); - - switch (Request.RequestVerb()) - { - case HttpVerb::kPost: - { - CacheRequestContext RequestContext = {.SessionId = Request.SessionId(), .RequestId = Request.RequestId()}; - - const HttpContentType ContentType = Request.RequestContentType(); - const HttpContentType AcceptType = Request.AcceptContentType(); - - if ((ContentType != HttpContentType::kCbObject && ContentType != HttpContentType::kCbPackage) || - AcceptType != HttpContentType::kCbPackage) - { - m_CacheStats.BadRequestCount++; - return Request.WriteResponse(HttpResponseCode::BadRequest); - } - - auto HandleRpc = [this, - RequestContext, - Body = Request.ReadPayload(), - ContentType, - AcceptType, - UriNamespaceString = std::string{UriNamespace}](HttpServerRequest& AsyncRequest) mutable { - if (m_RequestRecordingEnabled) - { - RwLock::SharedLockScope _(m_RequestRecordingLock); - if (m_RequestRecorder) - { - m_RequestRecorder->RecordRequest( - {.ContentType = ContentType, .AcceptType = AcceptType, .SessionId = RequestContext.SessionId}, - Body); - } - } - - uint32_t AcceptMagic = 0; - RpcAcceptOptions AcceptFlags = RpcAcceptOptions::kNone; - int TargetProcessId = 0; - CbPackage RpcResult; - - CacheRpcHandler::RpcResponseCode ResultCode = m_RpcHandler.HandleRpcRequest(RequestContext, - UriNamespaceString, - ContentType, - std::move(Body), - /* out */ AcceptMagic, - /* out */ AcceptFlags, - /* out */ TargetProcessId, - /* out */ RpcResult); - - HttpResponseCode HttpResultCode = HttpResponseCode(int(ResultCode)); - - if (!IsHttpSuccessCode(HttpResultCode)) - { - return AsyncRequest.WriteResponse(HttpResultCode); - } - - if (AcceptMagic == kCbPkgMagic) - { - void* TargetProcessHandle = nullptr; - FormatFlags Flags = FormatFlags::kDefault; - if (EnumHasAllFlags(AcceptFlags, RpcAcceptOptions::kAllowLocalReferences)) - { - Flags |= FormatFlags::kAllowLocalReferences; - if (!EnumHasAnyFlags(AcceptFlags, RpcAcceptOptions::kAllowPartialLocalReferences)) - { - Flags |= FormatFlags::kDenyPartialLocalReferences; - } - TargetProcessHandle = m_OpenProcessCache.GetProcessHandle(RequestContext.SessionId, TargetProcessId); - } - CompositeBuffer RpcResponseBuffer = FormatPackageMessageBuffer(RpcResult, Flags, TargetProcessHandle); - AsyncRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, RpcResponseBuffer); - } - else - { - BinaryWriter MemStream; - RpcResult.Save(MemStream); - - AsyncRequest.WriteResponse(HttpResponseCode::OK, - HttpContentType::kCbPackage, - IoBuffer(IoBuffer::Wrap, MemStream.GetData(), MemStream.GetSize())); - } - }; - - if (HasUpstream) - { - ZEN_TRACE_CPU("z$::Http::HandleRpcRequest::WriteResponseAsync"); - Request.WriteResponseAsync(std::move(HandleRpc)); - } - else - { - ZEN_TRACE_CPU("z$::Http::HandleRpcRequest::WriteResponse"); - HandleRpc(Request); - } - } - break; - - default: - m_CacheStats.BadRequestCount++; - Request.WriteResponse(HttpResponseCode::BadRequest); - break; - } -} - -void -HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request) -{ - ZEN_MEMSCOPE(GetCacheHttpTag()); - - CbObjectWriter Cbo; - - EmitSnapshot("requests", m_HttpRequests, Cbo); - - const uint64_t HitCount = m_CacheStats.HitCount; - const uint64_t UpstreamHitCount = m_CacheStats.UpstreamHitCount; - const uint64_t MissCount = m_CacheStats.MissCount; - const uint64_t WriteCount = m_CacheStats.WriteCount; - const uint64_t BadRequestCount = m_CacheStats.BadRequestCount; - struct CidStoreStats StoreStats = m_CidStore.Stats(); - const uint64_t ChunkHitCount = StoreStats.HitCount; - const uint64_t ChunkMissCount = StoreStats.MissCount; - const uint64_t ChunkWriteCount = StoreStats.WriteCount; - const uint64_t TotalCount = HitCount + MissCount; - - const uint64_t RpcRequests = m_CacheStats.RpcRequests; - const uint64_t RpcRecordRequests = m_CacheStats.RpcRecordRequests; - const uint64_t RpcRecordBatchRequests = m_CacheStats.RpcRecordBatchRequests; - const uint64_t RpcValueRequests = m_CacheStats.RpcValueRequests; - const uint64_t RpcValueBatchRequests = m_CacheStats.RpcValueBatchRequests; - const uint64_t RpcChunkRequests = m_CacheStats.RpcChunkRequests; - const uint64_t RpcChunkBatchRequests = m_CacheStats.RpcChunkBatchRequests; - - const CidStoreSize CidSize = m_CidStore.TotalSize(); - const CacheStoreSize CacheSize = m_CacheStore.TotalSize(); - - bool ShowCidStoreStats = Request.GetQueryParams().GetValue("cidstorestats") == "true"; - bool ShowCacheStoreStats = Request.GetQueryParams().GetValue("cachestorestats") == "true"; - - CidStoreStats CidStoreStats = {}; - if (ShowCidStoreStats) - { - CidStoreStats = m_CidStore.Stats(); - } - ZenCacheStore::CacheStoreStats CacheStoreStats = {}; - if (ShowCacheStoreStats) - { - CacheStoreStats = m_CacheStore.Stats(); - } - - Cbo.BeginObject("cache"); - { - Cbo << "badrequestcount" << BadRequestCount; - Cbo.BeginObject("rpc"); - Cbo << "count" << RpcRequests; - Cbo << "ops" << RpcRecordBatchRequests + RpcValueBatchRequests + RpcChunkBatchRequests; - Cbo.BeginObject("records"); - Cbo << "count" << RpcRecordRequests; - Cbo << "ops" << RpcRecordBatchRequests; - Cbo.EndObject(); - Cbo.BeginObject("values"); - Cbo << "count" << RpcValueRequests; - Cbo << "ops" << RpcValueBatchRequests; - Cbo.EndObject(); - Cbo.BeginObject("chunks"); - Cbo << "count" << RpcChunkRequests; - Cbo << "ops" << RpcChunkBatchRequests; - Cbo.EndObject(); - Cbo.EndObject(); - - Cbo.BeginObject("size"); - { - Cbo << "disk" << CacheSize.DiskSize; - Cbo << "memory" << CacheSize.MemorySize; - } - Cbo.EndObject(); - - Cbo << "hits" << HitCount << "misses" << MissCount << "writes" << WriteCount; - Cbo << "hit_ratio" << (TotalCount > 0 ? (double(HitCount) / double(TotalCount)) : 0.0); - - if (m_UpstreamCache.IsActive()) - { - Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0); - Cbo << "upstream_hits" << m_CacheStats.UpstreamHitCount; - Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0); - Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0); - } - - Cbo << "cidhits" << ChunkHitCount << "cidmisses" << ChunkMissCount << "cidwrites" << ChunkWriteCount; - - if (ShowCacheStoreStats) - { - Cbo.BeginObject("store"); - Cbo << "hits" << CacheStoreStats.HitCount << "misses" << CacheStoreStats.MissCount << "writes" << CacheStoreStats.WriteCount - << "rejected_writes" << CacheStoreStats.RejectedWriteCount << "rejected_reads" << CacheStoreStats.RejectedReadCount; - const uint64_t StoreTotal = CacheStoreStats.HitCount + CacheStoreStats.MissCount; - Cbo << "hit_ratio" << (StoreTotal > 0 ? (double(CacheStoreStats.HitCount) / double(StoreTotal)) : 0.0); - EmitSnapshot("read", CacheStoreStats.GetOps, Cbo); - EmitSnapshot("write", CacheStoreStats.PutOps, Cbo); - if (!CacheStoreStats.NamespaceStats.empty()) - { - Cbo.BeginArray("namespaces"); - for (const ZenCacheStore::NamedNamespaceStats& NamespaceStats : CacheStoreStats.NamespaceStats) - { - Cbo.BeginObject(); - Cbo.AddString("namespace", NamespaceStats.NamespaceName); - Cbo << "hits" << NamespaceStats.Stats.HitCount << "misses" << NamespaceStats.Stats.MissCount << "writes" - << NamespaceStats.Stats.WriteCount; - const uint64_t NamespaceTotal = NamespaceStats.Stats.HitCount + NamespaceStats.Stats.MissCount; - Cbo << "hit_ratio" << (NamespaceTotal > 0 ? (double(NamespaceStats.Stats.HitCount) / double(NamespaceTotal)) : 0.0); - EmitSnapshot("read", NamespaceStats.Stats.GetOps, Cbo); - EmitSnapshot("write", NamespaceStats.Stats.PutOps, Cbo); - Cbo.BeginObject("size"); - { - Cbo << "disk" << NamespaceStats.Stats.DiskStats.DiskSize; - Cbo << "memory" << NamespaceStats.Stats.DiskStats.MemorySize; - } - Cbo.EndObject(); - if (!NamespaceStats.Stats.DiskStats.BucketStats.empty()) - { - Cbo.BeginArray("buckets"); - for (const ZenCacheDiskLayer::NamedBucketStats& BucketStats : NamespaceStats.Stats.DiskStats.BucketStats) - { - Cbo.BeginObject(); - Cbo.AddString("bucket", BucketStats.BucketName); - if (BucketStats.Stats.DiskSize != 0 || BucketStats.Stats.MemorySize != 0) - { - Cbo.BeginObject("size"); - { - Cbo << "disk" << BucketStats.Stats.DiskSize; - Cbo << "memory" << BucketStats.Stats.MemorySize; - } - Cbo.EndObject(); - } - - if (BucketStats.Stats.DiskSize == 0 && BucketStats.Stats.DiskHitCount == 0 && - BucketStats.Stats.DiskMissCount == 0 && BucketStats.Stats.DiskWriteCount == 0 && - BucketStats.Stats.MemoryHitCount == 0 && BucketStats.Stats.MemoryMissCount == 0 && - BucketStats.Stats.MemoryWriteCount == 0) - { - Cbo.EndObject(); - continue; - } - - const uint64_t BucketDiskTotal = BucketStats.Stats.DiskHitCount + BucketStats.Stats.DiskMissCount; - if (BucketDiskTotal != 0 || BucketStats.Stats.DiskWriteCount != 0) - { - Cbo << "hits" << BucketStats.Stats.DiskHitCount << "misses" << BucketStats.Stats.DiskMissCount << "writes" - << BucketStats.Stats.DiskWriteCount; - Cbo << "hit_ratio" - << (BucketDiskTotal > 0 ? (double(BucketStats.Stats.DiskHitCount) / double(BucketDiskTotal)) : 0.0); - } - - const uint64_t BucketMemoryTotal = BucketStats.Stats.MemoryHitCount + BucketStats.Stats.MemoryMissCount; - if (BucketMemoryTotal != 0 || BucketStats.Stats.MemoryWriteCount != 0) - { - Cbo << "mem_hits" << BucketStats.Stats.MemoryHitCount << "mem_misses" << BucketStats.Stats.MemoryMissCount - << "mem_writes" << BucketStats.Stats.MemoryWriteCount; - Cbo << "mem_hit_ratio" - << (BucketMemoryTotal > 0 ? (double(BucketStats.Stats.MemoryHitCount) / double(BucketMemoryTotal)) - : 0.0); - } - - if (BucketDiskTotal != 0 || BucketStats.Stats.DiskWriteCount != 0 || BucketMemoryTotal != 0 || - BucketStats.Stats.MemoryWriteCount != 0) - { - EmitSnapshot("read", BucketStats.Stats.GetOps, Cbo); - EmitSnapshot("write", BucketStats.Stats.PutOps, Cbo); - } - - Cbo.EndObject(); - } - Cbo.EndArray(); - } - Cbo.EndObject(); - } - Cbo.EndArray(); - } - Cbo.EndObject(); - } - Cbo.EndObject(); - } - - if (m_UpstreamCache.IsActive()) - { - EmitSnapshot("upstream_gets", m_UpstreamGetRequestTiming, Cbo); - Cbo.BeginObject("upstream"); - { - m_UpstreamCache.GetStatus(Cbo); - } - Cbo.EndObject(); - } - - Cbo.BeginObject("cid"); - { - Cbo.BeginObject("size"); - { - Cbo << "tiny" << CidSize.TinySize; - Cbo << "small" << CidSize.SmallSize; - Cbo << "large" << CidSize.LargeSize; - Cbo << "total" << CidSize.TotalSize; - } - Cbo.EndObject(); - - if (ShowCidStoreStats) - { - Cbo.BeginObject("store"); - Cbo << "hits" << CidStoreStats.HitCount << "misses" << CidStoreStats.MissCount << "writes" << CidStoreStats.WriteCount; - EmitSnapshot("read", CidStoreStats.FindChunkOps, Cbo); - EmitSnapshot("write", CidStoreStats.AddChunkOps, Cbo); - // EmitSnapshot("exists", CidStoreStats.ContainChunkOps, Cbo); - Cbo.EndObject(); - } - } - Cbo.EndObject(); - - Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); -} - -void -HttpStructuredCacheService::HandleStatusRequest(HttpServerRequest& Request) -{ - CbObjectWriter Cbo; - Cbo << "ok" << true; - Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); -} - -bool -HttpStructuredCacheService::AreDiskWritesAllowed() const -{ - return (m_DiskWriteBlocker == nullptr || m_DiskWriteBlocker->AreDiskWritesAllowed()); -} - -} // namespace zen diff --git a/src/zenserver/cache/httpstructuredcache.h b/src/zenserver/cache/httpstructuredcache.h deleted file mode 100644 index a157148c9..000000000 --- a/src/zenserver/cache/httpstructuredcache.h +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -namespace zen { - -struct CacheChunkRequest; -struct CacheKeyRequest; -struct PutRequestData; - -class CidStore; -class CbObjectView; -class DiskWriteBlocker; -class HttpStructuredCacheService; -class ScrubContext; -class UpstreamCache; -class ZenCacheStore; - -enum class CachePolicy : uint32_t; -enum class RpcAcceptOptions : uint16_t; - -namespace cache { - class IRpcRequestReplayer; - class IRpcRequestRecorder; - namespace detail { - struct RecordBody; - struct ChunkRequest; - } // namespace detail -} // namespace cache - -/** - * Structured cache service. Imposes constraints on keys, supports blobs and - * structured values - * - * Keys are structured as: - * - * {BucketId}/{KeyHash} - * - * Where BucketId is a lower-case alphanumeric string, and KeyHash is a 40-character - * hexadecimal sequence. The hash value may be derived in any number of ways, it's - * up to the application to pick an approach. - * - * Values may be structured or unstructured. Structured values are encoded using Unreal - * Engine's compact binary encoding (see CbObject) - * - * Additionally, attachments may be addressed as: - * - * {BucketId}/{KeyHash}/{ValueHash} - * - * Where the two initial components are the same as for the main endpoint - * - * The storage strategy is as follows: - * - * - Structured values are stored in a dedicated backing store per bucket - * - Unstructured values and attachments are stored in the CAS pool - * - */ - -class HttpStructuredCacheService : public HttpService, public IHttpStatsProvider, public IHttpStatusProvider -{ -public: - HttpStructuredCacheService(ZenCacheStore& InCacheStore, - CidStore& InCidStore, - HttpStatsService& StatsService, - HttpStatusService& StatusService, - UpstreamCache& UpstreamCache, - const DiskWriteBlocker* InDiskWriteBlocker, - OpenProcessCache& InOpenProcessCache); - ~HttpStructuredCacheService(); - - virtual const char* BaseUri() const override; - virtual void HandleRequest(HttpServerRequest& Request) override; - - void Flush(); - -private: - struct CacheRef - { - std::string Namespace; - std::string BucketSegment; - IoHash HashKey; - IoHash ValueContentId; - }; - - void HandleCacheRecordRequest(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl); - void HandleGetCacheRecord(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl); - void HandlePutCacheRecord(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl); - void HandleCacheChunkRequest(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl); - void HandleGetCacheChunk(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl); - void HandlePutCacheChunk(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl); - void HandleRpcRequest(HttpServerRequest& Request, std::string_view UriNamespace); - void HandleDetailsRequest(HttpServerRequest& Request); - - void HandleCacheRequest(HttpServerRequest& Request); - void HandleCacheNamespaceRequest(HttpServerRequest& Request, std::string_view Namespace); - void HandleCacheBucketRequest(HttpServerRequest& Request, std::string_view Namespace, std::string_view Bucket); - virtual void HandleStatsRequest(HttpServerRequest& Request) override; - virtual void HandleStatusRequest(HttpServerRequest& Request) override; - - bool AreDiskWritesAllowed() const; - - LoggerRef Log() { return m_Log; } - LoggerRef m_Log; - ZenCacheStore& m_CacheStore; - HttpStatsService& m_StatsService; - HttpStatusService& m_StatusService; - CidStore& m_CidStore; - UpstreamCache& m_UpstreamCache; - metrics::OperationTiming m_HttpRequests; - metrics::OperationTiming m_UpstreamGetRequestTiming; - CacheStats m_CacheStats; - const DiskWriteBlocker* m_DiskWriteBlocker = nullptr; - OpenProcessCache& m_OpenProcessCache; - CacheRpcHandler m_RpcHandler; - - void ReplayRequestRecorder(const CacheRequestContext& Context, cache::IRpcRequestReplayer& Replayer, uint32_t ThreadCount); - - // This exists to avoid taking locks when recording is not enabled - std::atomic_bool m_RequestRecordingEnabled{false}; - - // This lock should be taken in SHARED mode when calling into the recorder, - // and taken in EXCLUSIVE mode whenever the recorder is created or destroyed - RwLock m_RequestRecordingLock; - std::unique_ptr m_RequestRecorder; -}; - -} // namespace zen diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp deleted file mode 100644 index 8ee50cee2..000000000 --- a/src/zenserver/config.cpp +++ /dev/null @@ -1,512 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "config.h" - -#include "storageconfig.h" - -#include "config/luaconfig.h" -#include "diag/logging.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -ZEN_THIRD_PARTY_INCLUDES_START -#include -#include -ZEN_THIRD_PARTY_INCLUDES_END - -#if ZEN_PLATFORM_WINDOWS -# include -#else -# include -#endif - -namespace zen { - -std::filesystem::path -PickDefaultStateDirectory(std::filesystem::path SystemRoot) -{ - if (SystemRoot.empty()) - return SystemRoot; - - return SystemRoot / "Data"; -} - -void -EmitCentralManifest(const std::filesystem::path& SystemRoot, Oid Identifier, CbObject Manifest, std::filesystem::path ManifestPath) -{ - CbObjectWriter Cbo; - Cbo << "path" << ManifestPath.generic_wstring(); - Cbo << "manifest" << Manifest; - - const std::filesystem::path StatesPath = SystemRoot / "States"; - - CreateDirectories(StatesPath); - WriteFile(StatesPath / fmt::format("{}", Identifier), Cbo.Save().GetBuffer().AsIoBuffer()); -} - -std::vector -ReadAllCentralManifests(const std::filesystem::path& SystemRoot) -{ - std::vector Manifests; - - DirectoryContent Content; - GetDirectoryContent(SystemRoot / "States", DirectoryContentFlags::IncludeFiles, Content); - - for (std::filesystem::path& File : Content.Files) - { - try - { - FileContents FileData = ReadFile(File); - CbValidateError ValidateError; - if (CbObject Manifest = ValidateAndReadCompactBinaryObject(FileData.Flatten(), ValidateError); - ValidateError == CbValidateError::None) - { - Manifests.emplace_back(std::move(Manifest)); - } - else - { - ZEN_WARN("failed to load manifest '{}': {}", File, ToString(ValidateError)); - } - } - catch (const std::exception& Ex) - { - ZEN_WARN("failed to load manifest '{}': {}", File, Ex.what()); - } - } - - return Manifests; -} - -void -ParseEnvVariables(ZenServerOptions& ServerOptions, const cxxopts::ParseResult& CmdLineResult) -{ - using namespace std::literals; - - EnvironmentOptions Options; - Options.AddOption("UE_ZEN_SENTRY_ALLOWPERSONALINFO"sv, ServerOptions.SentryConfig.AllowPII, "sentry-allow-personal-info"sv); - Options.AddOption("UE_ZEN_SENTRY_DSN"sv, ServerOptions.SentryConfig.Dsn, "sentry-dsn"sv); - Options.AddOption("UE_ZEN_SENTRY_ENVIRONMENT"sv, ServerOptions.SentryConfig.Environment, "sentry-environment"sv); - - bool EnvEnableSentry = !ServerOptions.SentryConfig.Disable; - Options.AddOption("UE_ZEN_SENTRY_ENABLED"sv, EnvEnableSentry, "no-sentry"sv); - - Options.AddOption("UE_ZEN_SENTRY_DEBUG"sv, ServerOptions.SentryConfig.Debug, "sentry-debug"sv); - - Options.Parse(CmdLineResult); - - if (EnvEnableSentry != !ServerOptions.SentryConfig.Disable) - { - ServerOptions.SentryConfig.Disable = !EnvEnableSentry; - } -} - -void -AddServerConfigOptions(LuaConfig::Options& LuaOptions, ZenServerOptions& ServerOptions) -{ - using namespace std::literals; - - // server - - LuaOptions.AddOption("server.dedicated"sv, ServerOptions.IsDedicated, "dedicated"sv); - LuaOptions.AddOption("server.logid"sv, ServerOptions.LogId, "log-id"sv); - LuaOptions.AddOption("server.sentry.disable"sv, ServerOptions.SentryConfig.Disable, "no-sentry"sv); - LuaOptions.AddOption("server.sentry.allowpersonalinfo"sv, ServerOptions.SentryConfig.AllowPII, "sentry-allow-personal-info"sv); - LuaOptions.AddOption("server.sentry.dsn"sv, ServerOptions.SentryConfig.Dsn, "sentry-dsn"sv); - LuaOptions.AddOption("server.sentry.environment"sv, ServerOptions.SentryConfig.Environment, "sentry-environment"sv); - LuaOptions.AddOption("server.sentry.debug"sv, ServerOptions.SentryConfig.Debug, "sentry-debug"sv); - LuaOptions.AddOption("server.systemrootdir"sv, ServerOptions.SystemRootDir, "system-dir"sv); - LuaOptions.AddOption("server.datadir"sv, ServerOptions.DataDir, "data-dir"sv); - LuaOptions.AddOption("server.contentdir"sv, ServerOptions.ContentDir, "content-dir"sv); - LuaOptions.AddOption("server.abslog"sv, ServerOptions.AbsLogFile, "abslog"sv); - LuaOptions.AddOption("server.debug"sv, ServerOptions.IsDebug, "debug"sv); - LuaOptions.AddOption("server.clean"sv, ServerOptions.IsCleanStart, "clean"sv); - LuaOptions.AddOption("server.quiet"sv, ServerOptions.QuietConsole, "quiet"sv); - LuaOptions.AddOption("server.noconsole"sv, ServerOptions.NoConsoleOutput, "noconsole"sv); - - ////// network - LuaOptions.AddOption("network.httpserverclass"sv, ServerOptions.HttpServerConfig.ServerClass, "http"sv); - LuaOptions.AddOption("network.httpserverthreads"sv, ServerOptions.HttpServerConfig.ThreadCount, "http-threads"sv); - LuaOptions.AddOption("network.port"sv, ServerOptions.BasePort, "port"sv); - LuaOptions.AddOption("network.forceloopback"sv, ServerOptions.HttpServerConfig.ForceLoopback, "http-forceloopback"sv); - -#if ZEN_WITH_HTTPSYS - LuaOptions.AddOption("network.httpsys.async.workthreads"sv, - ServerOptions.HttpServerConfig.HttpSys.AsyncWorkThreadCount, - "httpsys-async-work-threads"sv); - LuaOptions.AddOption("network.httpsys.async.response"sv, - ServerOptions.HttpServerConfig.HttpSys.IsAsyncResponseEnabled, - "httpsys-enable-async-response"sv); - LuaOptions.AddOption("network.httpsys.requestlogging"sv, - ServerOptions.HttpServerConfig.HttpSys.IsRequestLoggingEnabled, - "httpsys-enable-request-logging"sv); -#endif - -#if ZEN_WITH_TRACE - ////// trace - LuaOptions.AddOption("trace.channels"sv, ServerOptions.TraceOptions.Channels, "trace"sv); - LuaOptions.AddOption("trace.host"sv, ServerOptions.TraceOptions.Host, "tracehost"sv); - LuaOptions.AddOption("trace.file"sv, ServerOptions.TraceOptions.File, "tracefile"sv); -#endif - - ////// stats - LuaOptions.AddOption("stats.enable"sv, ServerOptions.StatsConfig.Enabled, "statsd"sv); - LuaOptions.AddOption("stats.host"sv, ServerOptions.StatsConfig.StatsdHost); - LuaOptions.AddOption("stats.port"sv, ServerOptions.StatsConfig.StatsdPort); -} - -struct ZenServerCmdLineOptions -{ - // Note to those adding future options; std::filesystem::path-type options - // must be read into a std::string first. As of cxxopts-3.0.0 it uses a >> - // stream operator to convert argv value into the options type. std::fs::path - // expects paths in streams to be quoted but argv paths are unquoted. By - // going into a std::string first, paths with whitespace parse correctly. - std::string ConfigFile; - std::string OutputConfigFile; - std::string SystemRootDir; - std::string ContentDir; - std::string DataDir; - std::string AbsLogFile; - - void AddCliOptions(cxxopts::Options& options, ZenServerOptions& ServerOptions); - void ApplyOptions(ZenServerOptions& ServerOptions); -}; - -void -ZenServerCmdLineOptions::AddCliOptions(cxxopts::Options& options, ZenServerOptions& ServerOptions) -{ - const char* DefaultHttp = "asio"; - -#if ZEN_WITH_HTTPSYS - if (!windows::IsRunningOnWine()) - { - DefaultHttp = "httpsys"; - } -#endif - - options.add_options()("dedicated", - "Enable dedicated server mode", - cxxopts::value(ServerOptions.IsDedicated)->default_value("false")); - options.add_options()("d, debug", "Enable debugging", cxxopts::value(ServerOptions.IsDebug)->default_value("false")); - options.add_options()("clean", - "Clean out all state at startup", - cxxopts::value(ServerOptions.IsCleanStart)->default_value("false")); - options.add_options()("help", "Show command line help"); - options.add_options()("t, test", "Enable test mode", cxxopts::value(ServerOptions.IsTest)->default_value("false")); - - options.add_options()("data-dir", "Specify persistence root", cxxopts::value(DataDir)); - options.add_options()("system-dir", "Specify system root", cxxopts::value(SystemRootDir)); - options.add_options()("content-dir", "Frontend content directory", cxxopts::value(ContentDir)); - options.add_options()("config", "Path to Lua config file", cxxopts::value(ConfigFile)); - options.add_options()("write-config", "Path to output Lua config file", cxxopts::value(OutputConfigFile)); - - options.add_options()("no-sentry", - "Disable Sentry crash handler", - cxxopts::value(ServerOptions.SentryConfig.Disable)->default_value("false")); - options.add_options()("sentry-allow-personal-info", - "Allow personally identifiable information in sentry crash reports", - cxxopts::value(ServerOptions.SentryConfig.AllowPII)->default_value("false")); - options.add_options()("sentry-dsn", "Sentry DSN to send events to", cxxopts::value(ServerOptions.SentryConfig.Dsn)); - options.add_options()("sentry-environment", "Sentry environment", cxxopts::value(ServerOptions.SentryConfig.Environment)); - options.add_options()("sentry-debug", - "Enable debug mode for Sentry", - cxxopts::value(ServerOptions.SentryConfig.Debug)->default_value("false")); - options.add_options()("detach", - "Indicate whether zenserver should detach from parent process group", - cxxopts::value(ServerOptions.Detach)->default_value("true")); - options.add_options()("malloc", - "Configure memory allocator subsystem", - cxxopts::value(ServerOptions.MemoryOptions)->default_value("mimalloc")); - options.add_options()("powercycle", - "Exit immediately after initialization is complete", - cxxopts::value(ServerOptions.IsPowerCycle)); - - options.add_option("diagnostics", - "", - "crash", - "Simulate a crash", - cxxopts::value(ServerOptions.ShouldCrash)->default_value("false"), - ""); - - // clang-format off - options.add_options("logging") - ("abslog", "Path to log file", cxxopts::value(AbsLogFile)) - ("log-id", "Specify id for adding context to log output", cxxopts::value(ServerOptions.LogId)) - ("quiet", "Configure console logger output to level WARN", cxxopts::value(ServerOptions.QuietConsole)->default_value("false")) - ("noconsole", "Disable console logging", cxxopts::value(ServerOptions.NoConsoleOutput)->default_value("false")) - ("log-trace", "Change selected loggers to level TRACE", cxxopts::value(ServerOptions.Loggers[logging::level::Trace])) - ("log-debug", "Change selected loggers to level DEBUG", cxxopts::value(ServerOptions.Loggers[logging::level::Debug])) - ("log-info", "Change selected loggers to level INFO", cxxopts::value(ServerOptions.Loggers[logging::level::Info])) - ("log-warn", "Change selected loggers to level WARN", cxxopts::value(ServerOptions.Loggers[logging::level::Warn])) - ("log-error", "Change selected loggers to level ERROR", cxxopts::value(ServerOptions.Loggers[logging::level::Err])) - ("log-critical", "Change selected loggers to level CRITICAL", cxxopts::value(ServerOptions.Loggers[logging::level::Critical])) - ("log-off", "Change selected loggers to level OFF", cxxopts::value(ServerOptions.Loggers[logging::level::Off])) - ; - // clang-format on - - options - .add_option("lifetime", "", "owner-pid", "Specify owning process id", cxxopts::value(ServerOptions.OwnerPid), ""); - options.add_option("lifetime", - "", - "child-id", - "Specify id which can be used to signal parent", - cxxopts::value(ServerOptions.ChildId), - ""); - -#if ZEN_PLATFORM_WINDOWS - options.add_option("lifetime", - "", - "install", - "Install zenserver as a Windows service", - cxxopts::value(ServerOptions.InstallService), - ""); - options.add_option("lifetime", - "", - "uninstall", - "Uninstall zenserver as a Windows service", - cxxopts::value(ServerOptions.UninstallService), - ""); -#endif - - options.add_option("network", - "", - "http-threads", - "Number of http server connection threads", - cxxopts::value(ServerOptions.HttpServerConfig.ThreadCount), - ""); - - options.add_option("network", - "p", - "port", - "Select HTTP port", - cxxopts::value(ServerOptions.BasePort)->default_value("8558"), - ""); - - options.add_option("network", - "", - "http-forceloopback", - "Force using local loopback interface", - cxxopts::value(ServerOptions.HttpServerConfig.ForceLoopback)->default_value("false"), - ""); - -#if ZEN_WITH_HTTPSYS - options.add_option("httpsys", - "", - "httpsys-async-work-threads", - "Number of HttpSys async worker threads", - cxxopts::value(ServerOptions.HttpServerConfig.HttpSys.AsyncWorkThreadCount), - ""); - - options.add_option("httpsys", - "", - "httpsys-enable-async-response", - "Enables Httpsys async response", - cxxopts::value(ServerOptions.HttpServerConfig.HttpSys.IsAsyncResponseEnabled)->default_value("true"), - ""); - - options.add_option("httpsys", - "", - "httpsys-enable-request-logging", - "Enables Httpsys request logging", - cxxopts::value(ServerOptions.HttpServerConfig.HttpSys.IsRequestLoggingEnabled), - ""); -#endif - - options.add_option("network", - "", - "http", - "Select HTTP server implementation (asio|" -#if ZEN_WITH_HTTPSYS - "httpsys|" -#endif - "null)", - cxxopts::value(ServerOptions.HttpServerConfig.ServerClass)->default_value(DefaultHttp), - ""); - -#if ZEN_WITH_TRACE - // We only have this in options for command line help purposes - we parse these argument separately earlier using - // GetTraceOptionsFromCommandline() - - options.add_option("ue-trace", - "", - "trace", - "Specify which trace channels should be enabled", - cxxopts::value(ServerOptions.TraceOptions.Channels)->default_value(""), - ""); - - options.add_option("ue-trace", - "", - "tracehost", - "Hostname to send the trace to", - cxxopts::value(ServerOptions.TraceOptions.Host)->default_value(""), - ""); - - options.add_option("ue-trace", - "", - "tracefile", - "Path to write a trace to", - cxxopts::value(ServerOptions.TraceOptions.File)->default_value(""), - ""); -#endif // ZEN_WITH_TRACE - - options.add_option("stats", - "", - "statsd", - "", - cxxopts::value(ServerOptions.StatsConfig.Enabled)->default_value("false"), - "Enable statsd reporter (localhost:8125)"); -} - -void -ZenServerCmdLineOptions::ApplyOptions(ZenServerOptions& ServerOptions) -{ - ServerOptions.SystemRootDir = MakeSafeAbsolutePath(SystemRootDir); - ServerOptions.DataDir = MakeSafeAbsolutePath(DataDir); - ServerOptions.ContentDir = MakeSafeAbsolutePath(ContentDir); - ServerOptions.AbsLogFile = MakeSafeAbsolutePath(AbsLogFile); - ServerOptions.ConfigFile = MakeSafeAbsolutePath(ConfigFile); -} - -void -ParseCliOptions(int argc, char* argv[], ZenStorageServerOptions& ServerOptions) -{ - for (int i = 0; i < argc; ++i) - { - if (i) - { - ServerOptions.CommandLine.push_back(' '); - } - - ServerOptions.CommandLine += argv[i]; - } - - cxxopts::Options options("zenserver", "Zen Storage Server"); - - ZenServerCmdLineOptions BaseOptions; - BaseOptions.AddCliOptions(options, ServerOptions); - - ZenStorageServerCmdLineOptions StorageOptions; - StorageOptions.AddCliOptions(options, ServerOptions); - - try - { - cxxopts::ParseResult Result; - - try - { - Result = options.parse(argc, argv); - } - catch (const std::exception& Ex) - { - throw OptionParseException(Ex.what(), options.help()); - } - - if (Result.count("help")) - { - ZEN_CONSOLE("{}", options.help()); - -#if ZEN_PLATFORM_WINDOWS - ZEN_CONSOLE("Press any key to exit!"); - _getch(); -#else - // Assume the user's in a terminal on all other platforms and that - // they'll use less/more/etc. if need be. -#endif - - exit(0); - } - - if (!ServerOptions.HasTraceCommandlineOptions) - { - // Apply any Lua settings if we don't have them set from the command line - TraceConfigure(ServerOptions.TraceOptions); - } - - if (ServerOptions.QuietConsole) - { - bool HasExplicitConsoleLevel = false; - for (int i = 0; i < logging::level::LogLevelCount; ++i) - { - if (ServerOptions.Loggers[i].find("console") != std::string::npos) - { - HasExplicitConsoleLevel = true; - break; - } - } - - if (!HasExplicitConsoleLevel) - { - std::string& WarnLoggers = ServerOptions.Loggers[logging::level::Warn]; - if (!WarnLoggers.empty()) - { - WarnLoggers += ","; - } - WarnLoggers += "console"; - } - } - - for (int i = 0; i < logging::level::LogLevelCount; ++i) - { - logging::ConfigureLogLevels(logging::level::LogLevel(i), ServerOptions.Loggers[i]); - } - logging::RefreshLogLevels(); - - BaseOptions.ApplyOptions(ServerOptions); - StorageOptions.ApplyOptions(options, ServerOptions); - - ParseEnvVariables(ServerOptions, Result); - - ZEN_TRACE_CPU("ConfigParse"); - - if (!ServerOptions.ConfigFile.empty()) - { - ParseConfigFile(ServerOptions.ConfigFile, ServerOptions, Result, BaseOptions.OutputConfigFile); - } - else - { - ParseConfigFile(ServerOptions.DataDir / "zen_cfg.lua", ServerOptions, Result, BaseOptions.OutputConfigFile); - } - - if (!ServerOptions.PluginsConfigFile.empty()) - { - ParsePluginsConfigFile(ServerOptions.PluginsConfigFile, ServerOptions, ServerOptions.BasePort); - } - - ValidateOptions(ServerOptions); - } - catch (const OptionParseException& e) - { - ZEN_CONSOLE("{}\n", options.help()); - ZEN_CONSOLE_ERROR("Invalid zenserver arguments: {}", e.what()); - throw; - } - - if (ServerOptions.SystemRootDir.empty()) - { - ServerOptions.SystemRootDir = PickDefaultSystemRootDirectory(); - } - - if (ServerOptions.DataDir.empty()) - { - ServerOptions.DataDir = PickDefaultStateDirectory(ServerOptions.SystemRootDir); - } - - if (ServerOptions.AbsLogFile.empty()) - { - ServerOptions.AbsLogFile = ServerOptions.DataDir / "logs" / "zenserver.log"; - } - - ServerOptions.HttpServerConfig.IsDedicatedServer = ServerOptions.IsDedicated; -} - -} // namespace zen diff --git a/src/zenserver/config.h b/src/zenserver/config.h deleted file mode 100644 index 8471ee89b..000000000 --- a/src/zenserver/config.h +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include -#include -#include -#include -#include -#include -#include - -namespace zen::LuaConfig { -struct Options; -} -namespace cxxopts { -class Options; -class ParseResult; -} // namespace cxxopts -namespace zen { - -struct ZenStorageServerOptions; - -struct ZenStatsConfig -{ - bool Enabled = false; - std::string StatsdHost = "localhost"; - int StatsdPort = 8125; -}; - -struct ZenSentryConfig -{ - bool Disable = false; - bool AllowPII = false; // Allow personally identifiable information in sentry crash reports - std::string Dsn; - std::string Environment; - bool Debug = false; // Enable debug mode for Sentry -}; - -struct ZenServerOptions -{ - HttpServerConfig HttpServerConfig; - ZenSentryConfig SentryConfig; - int BasePort = 8558; // Service listen port (used for both UDP and TCP) - int OwnerPid = 0; // Parent process id (zero for standalone) - bool IsDebug = false; - bool IsCleanStart = false; // Indicates whether all state should be wiped on startup or not - bool IsPowerCycle = false; // When true, the process shuts down immediately after initialization - bool IsTest = false; - bool Detach = true; // Whether zenserver should detach from existing process group (Mac/Linux) - bool NoConsoleOutput = false; // Control default use of stdout for diagnostics - bool QuietConsole = false; // Configure console logger output to level WARN - int CoreLimit = 0; // If set, hardware concurrency queries are capped at this number - bool IsDedicated = false; // Indicates a dedicated/shared instance, with larger resource requirements - bool ShouldCrash = false; // Option for testing crash handling - bool IsFirstRun = false; - std::filesystem::path ConfigFile; // Path to Lua config file - std::filesystem::path SystemRootDir; // System root directory (used for machine level config) - std::filesystem::path ContentDir; // Root directory for serving frontend content (experimental) - std::filesystem::path DataDir; // Root directory for state (used for testing) - std::filesystem::path AbsLogFile; // Absolute path to main log file - std::string ChildId; // Id assigned by parent process (used for lifetime management) - std::string LogId; // Id for tagging log output - std::string Loggers[zen::logging::level::LogLevelCount]; -#if ZEN_WITH_TRACE - bool HasTraceCommandlineOptions = false; - TraceOptions TraceOptions; -#endif - std::string MemoryOptions; // Memory allocation options - std::string CommandLine; - std::string EncryptionKey; // 256 bit AES encryption key - std::string EncryptionIV; // 128 bit AES initialization vector - - ZenStatsConfig StatsConfig; - - bool InstallService = false; // Flag used to initiate service install (temporary) - bool UninstallService = false; // Flag used to initiate service uninstall (temporary) -}; - -void ParseCliOptions(int argc, char* argv[], ZenStorageServerOptions& ServerOptions); - -void EmitCentralManifest(const std::filesystem::path& SystemRoot, Oid Identifier, CbObject Manifest, std::filesystem::path ManifestPath); -std::vector ReadAllCentralManifests(const std::filesystem::path& SystemRoot); - -void AddServerConfigOptions(LuaConfig::Options& LuaOptions, ZenServerOptions& ServerOptions); - -} // namespace zen diff --git a/src/zenserver/config/config.cpp b/src/zenserver/config/config.cpp new file mode 100644 index 000000000..d044c61d5 --- /dev/null +++ b/src/zenserver/config/config.cpp @@ -0,0 +1,512 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "config.h" + +#include "storage/storageconfig.h" + +#include "config/luaconfig.h" +#include "diag/logging.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +ZEN_THIRD_PARTY_INCLUDES_START +#include +#include +ZEN_THIRD_PARTY_INCLUDES_END + +#if ZEN_PLATFORM_WINDOWS +# include +#else +# include +#endif + +namespace zen { + +std::filesystem::path +PickDefaultStateDirectory(std::filesystem::path SystemRoot) +{ + if (SystemRoot.empty()) + return SystemRoot; + + return SystemRoot / "Data"; +} + +void +EmitCentralManifest(const std::filesystem::path& SystemRoot, Oid Identifier, CbObject Manifest, std::filesystem::path ManifestPath) +{ + CbObjectWriter Cbo; + Cbo << "path" << ManifestPath.generic_wstring(); + Cbo << "manifest" << Manifest; + + const std::filesystem::path StatesPath = SystemRoot / "States"; + + CreateDirectories(StatesPath); + WriteFile(StatesPath / fmt::format("{}", Identifier), Cbo.Save().GetBuffer().AsIoBuffer()); +} + +std::vector +ReadAllCentralManifests(const std::filesystem::path& SystemRoot) +{ + std::vector Manifests; + + DirectoryContent Content; + GetDirectoryContent(SystemRoot / "States", DirectoryContentFlags::IncludeFiles, Content); + + for (std::filesystem::path& File : Content.Files) + { + try + { + FileContents FileData = ReadFile(File); + CbValidateError ValidateError; + if (CbObject Manifest = ValidateAndReadCompactBinaryObject(FileData.Flatten(), ValidateError); + ValidateError == CbValidateError::None) + { + Manifests.emplace_back(std::move(Manifest)); + } + else + { + ZEN_WARN("failed to load manifest '{}': {}", File, ToString(ValidateError)); + } + } + catch (const std::exception& Ex) + { + ZEN_WARN("failed to load manifest '{}': {}", File, Ex.what()); + } + } + + return Manifests; +} + +void +ParseEnvVariables(ZenServerOptions& ServerOptions, const cxxopts::ParseResult& CmdLineResult) +{ + using namespace std::literals; + + EnvironmentOptions Options; + Options.AddOption("UE_ZEN_SENTRY_ALLOWPERSONALINFO"sv, ServerOptions.SentryConfig.AllowPII, "sentry-allow-personal-info"sv); + Options.AddOption("UE_ZEN_SENTRY_DSN"sv, ServerOptions.SentryConfig.Dsn, "sentry-dsn"sv); + Options.AddOption("UE_ZEN_SENTRY_ENVIRONMENT"sv, ServerOptions.SentryConfig.Environment, "sentry-environment"sv); + + bool EnvEnableSentry = !ServerOptions.SentryConfig.Disable; + Options.AddOption("UE_ZEN_SENTRY_ENABLED"sv, EnvEnableSentry, "no-sentry"sv); + + Options.AddOption("UE_ZEN_SENTRY_DEBUG"sv, ServerOptions.SentryConfig.Debug, "sentry-debug"sv); + + Options.Parse(CmdLineResult); + + if (EnvEnableSentry != !ServerOptions.SentryConfig.Disable) + { + ServerOptions.SentryConfig.Disable = !EnvEnableSentry; + } +} + +void +AddServerConfigOptions(LuaConfig::Options& LuaOptions, ZenServerOptions& ServerOptions) +{ + using namespace std::literals; + + // server + + LuaOptions.AddOption("server.dedicated"sv, ServerOptions.IsDedicated, "dedicated"sv); + LuaOptions.AddOption("server.logid"sv, ServerOptions.LogId, "log-id"sv); + LuaOptions.AddOption("server.sentry.disable"sv, ServerOptions.SentryConfig.Disable, "no-sentry"sv); + LuaOptions.AddOption("server.sentry.allowpersonalinfo"sv, ServerOptions.SentryConfig.AllowPII, "sentry-allow-personal-info"sv); + LuaOptions.AddOption("server.sentry.dsn"sv, ServerOptions.SentryConfig.Dsn, "sentry-dsn"sv); + LuaOptions.AddOption("server.sentry.environment"sv, ServerOptions.SentryConfig.Environment, "sentry-environment"sv); + LuaOptions.AddOption("server.sentry.debug"sv, ServerOptions.SentryConfig.Debug, "sentry-debug"sv); + LuaOptions.AddOption("server.systemrootdir"sv, ServerOptions.SystemRootDir, "system-dir"sv); + LuaOptions.AddOption("server.datadir"sv, ServerOptions.DataDir, "data-dir"sv); + LuaOptions.AddOption("server.contentdir"sv, ServerOptions.ContentDir, "content-dir"sv); + LuaOptions.AddOption("server.abslog"sv, ServerOptions.AbsLogFile, "abslog"sv); + LuaOptions.AddOption("server.debug"sv, ServerOptions.IsDebug, "debug"sv); + LuaOptions.AddOption("server.clean"sv, ServerOptions.IsCleanStart, "clean"sv); + LuaOptions.AddOption("server.quiet"sv, ServerOptions.QuietConsole, "quiet"sv); + LuaOptions.AddOption("server.noconsole"sv, ServerOptions.NoConsoleOutput, "noconsole"sv); + + ////// network + LuaOptions.AddOption("network.httpserverclass"sv, ServerOptions.HttpServerConfig.ServerClass, "http"sv); + LuaOptions.AddOption("network.httpserverthreads"sv, ServerOptions.HttpServerConfig.ThreadCount, "http-threads"sv); + LuaOptions.AddOption("network.port"sv, ServerOptions.BasePort, "port"sv); + LuaOptions.AddOption("network.forceloopback"sv, ServerOptions.HttpServerConfig.ForceLoopback, "http-forceloopback"sv); + +#if ZEN_WITH_HTTPSYS + LuaOptions.AddOption("network.httpsys.async.workthreads"sv, + ServerOptions.HttpServerConfig.HttpSys.AsyncWorkThreadCount, + "httpsys-async-work-threads"sv); + LuaOptions.AddOption("network.httpsys.async.response"sv, + ServerOptions.HttpServerConfig.HttpSys.IsAsyncResponseEnabled, + "httpsys-enable-async-response"sv); + LuaOptions.AddOption("network.httpsys.requestlogging"sv, + ServerOptions.HttpServerConfig.HttpSys.IsRequestLoggingEnabled, + "httpsys-enable-request-logging"sv); +#endif + +#if ZEN_WITH_TRACE + ////// trace + LuaOptions.AddOption("trace.channels"sv, ServerOptions.TraceOptions.Channels, "trace"sv); + LuaOptions.AddOption("trace.host"sv, ServerOptions.TraceOptions.Host, "tracehost"sv); + LuaOptions.AddOption("trace.file"sv, ServerOptions.TraceOptions.File, "tracefile"sv); +#endif + + ////// stats + LuaOptions.AddOption("stats.enable"sv, ServerOptions.StatsConfig.Enabled, "statsd"sv); + LuaOptions.AddOption("stats.host"sv, ServerOptions.StatsConfig.StatsdHost); + LuaOptions.AddOption("stats.port"sv, ServerOptions.StatsConfig.StatsdPort); +} + +struct ZenServerCmdLineOptions +{ + // Note to those adding future options; std::filesystem::path-type options + // must be read into a std::string first. As of cxxopts-3.0.0 it uses a >> + // stream operator to convert argv value into the options type. std::fs::path + // expects paths in streams to be quoted but argv paths are unquoted. By + // going into a std::string first, paths with whitespace parse correctly. + std::string ConfigFile; + std::string OutputConfigFile; + std::string SystemRootDir; + std::string ContentDir; + std::string DataDir; + std::string AbsLogFile; + + void AddCliOptions(cxxopts::Options& options, ZenServerOptions& ServerOptions); + void ApplyOptions(ZenServerOptions& ServerOptions); +}; + +void +ZenServerCmdLineOptions::AddCliOptions(cxxopts::Options& options, ZenServerOptions& ServerOptions) +{ + const char* DefaultHttp = "asio"; + +#if ZEN_WITH_HTTPSYS + if (!windows::IsRunningOnWine()) + { + DefaultHttp = "httpsys"; + } +#endif + + options.add_options()("dedicated", + "Enable dedicated server mode", + cxxopts::value(ServerOptions.IsDedicated)->default_value("false")); + options.add_options()("d, debug", "Enable debugging", cxxopts::value(ServerOptions.IsDebug)->default_value("false")); + options.add_options()("clean", + "Clean out all state at startup", + cxxopts::value(ServerOptions.IsCleanStart)->default_value("false")); + options.add_options()("help", "Show command line help"); + options.add_options()("t, test", "Enable test mode", cxxopts::value(ServerOptions.IsTest)->default_value("false")); + + options.add_options()("data-dir", "Specify persistence root", cxxopts::value(DataDir)); + options.add_options()("system-dir", "Specify system root", cxxopts::value(SystemRootDir)); + options.add_options()("content-dir", "Frontend content directory", cxxopts::value(ContentDir)); + options.add_options()("config", "Path to Lua config file", cxxopts::value(ConfigFile)); + options.add_options()("write-config", "Path to output Lua config file", cxxopts::value(OutputConfigFile)); + + options.add_options()("no-sentry", + "Disable Sentry crash handler", + cxxopts::value(ServerOptions.SentryConfig.Disable)->default_value("false")); + options.add_options()("sentry-allow-personal-info", + "Allow personally identifiable information in sentry crash reports", + cxxopts::value(ServerOptions.SentryConfig.AllowPII)->default_value("false")); + options.add_options()("sentry-dsn", "Sentry DSN to send events to", cxxopts::value(ServerOptions.SentryConfig.Dsn)); + options.add_options()("sentry-environment", "Sentry environment", cxxopts::value(ServerOptions.SentryConfig.Environment)); + options.add_options()("sentry-debug", + "Enable debug mode for Sentry", + cxxopts::value(ServerOptions.SentryConfig.Debug)->default_value("false")); + options.add_options()("detach", + "Indicate whether zenserver should detach from parent process group", + cxxopts::value(ServerOptions.Detach)->default_value("true")); + options.add_options()("malloc", + "Configure memory allocator subsystem", + cxxopts::value(ServerOptions.MemoryOptions)->default_value("mimalloc")); + options.add_options()("powercycle", + "Exit immediately after initialization is complete", + cxxopts::value(ServerOptions.IsPowerCycle)); + + options.add_option("diagnostics", + "", + "crash", + "Simulate a crash", + cxxopts::value(ServerOptions.ShouldCrash)->default_value("false"), + ""); + + // clang-format off + options.add_options("logging") + ("abslog", "Path to log file", cxxopts::value(AbsLogFile)) + ("log-id", "Specify id for adding context to log output", cxxopts::value(ServerOptions.LogId)) + ("quiet", "Configure console logger output to level WARN", cxxopts::value(ServerOptions.QuietConsole)->default_value("false")) + ("noconsole", "Disable console logging", cxxopts::value(ServerOptions.NoConsoleOutput)->default_value("false")) + ("log-trace", "Change selected loggers to level TRACE", cxxopts::value(ServerOptions.Loggers[logging::level::Trace])) + ("log-debug", "Change selected loggers to level DEBUG", cxxopts::value(ServerOptions.Loggers[logging::level::Debug])) + ("log-info", "Change selected loggers to level INFO", cxxopts::value(ServerOptions.Loggers[logging::level::Info])) + ("log-warn", "Change selected loggers to level WARN", cxxopts::value(ServerOptions.Loggers[logging::level::Warn])) + ("log-error", "Change selected loggers to level ERROR", cxxopts::value(ServerOptions.Loggers[logging::level::Err])) + ("log-critical", "Change selected loggers to level CRITICAL", cxxopts::value(ServerOptions.Loggers[logging::level::Critical])) + ("log-off", "Change selected loggers to level OFF", cxxopts::value(ServerOptions.Loggers[logging::level::Off])) + ; + // clang-format on + + options + .add_option("lifetime", "", "owner-pid", "Specify owning process id", cxxopts::value(ServerOptions.OwnerPid), ""); + options.add_option("lifetime", + "", + "child-id", + "Specify id which can be used to signal parent", + cxxopts::value(ServerOptions.ChildId), + ""); + +#if ZEN_PLATFORM_WINDOWS + options.add_option("lifetime", + "", + "install", + "Install zenserver as a Windows service", + cxxopts::value(ServerOptions.InstallService), + ""); + options.add_option("lifetime", + "", + "uninstall", + "Uninstall zenserver as a Windows service", + cxxopts::value(ServerOptions.UninstallService), + ""); +#endif + + options.add_option("network", + "", + "http-threads", + "Number of http server connection threads", + cxxopts::value(ServerOptions.HttpServerConfig.ThreadCount), + ""); + + options.add_option("network", + "p", + "port", + "Select HTTP port", + cxxopts::value(ServerOptions.BasePort)->default_value("8558"), + ""); + + options.add_option("network", + "", + "http-forceloopback", + "Force using local loopback interface", + cxxopts::value(ServerOptions.HttpServerConfig.ForceLoopback)->default_value("false"), + ""); + +#if ZEN_WITH_HTTPSYS + options.add_option("httpsys", + "", + "httpsys-async-work-threads", + "Number of HttpSys async worker threads", + cxxopts::value(ServerOptions.HttpServerConfig.HttpSys.AsyncWorkThreadCount), + ""); + + options.add_option("httpsys", + "", + "httpsys-enable-async-response", + "Enables Httpsys async response", + cxxopts::value(ServerOptions.HttpServerConfig.HttpSys.IsAsyncResponseEnabled)->default_value("true"), + ""); + + options.add_option("httpsys", + "", + "httpsys-enable-request-logging", + "Enables Httpsys request logging", + cxxopts::value(ServerOptions.HttpServerConfig.HttpSys.IsRequestLoggingEnabled), + ""); +#endif + + options.add_option("network", + "", + "http", + "Select HTTP server implementation (asio|" +#if ZEN_WITH_HTTPSYS + "httpsys|" +#endif + "null)", + cxxopts::value(ServerOptions.HttpServerConfig.ServerClass)->default_value(DefaultHttp), + ""); + +#if ZEN_WITH_TRACE + // We only have this in options for command line help purposes - we parse these argument separately earlier using + // GetTraceOptionsFromCommandline() + + options.add_option("ue-trace", + "", + "trace", + "Specify which trace channels should be enabled", + cxxopts::value(ServerOptions.TraceOptions.Channels)->default_value(""), + ""); + + options.add_option("ue-trace", + "", + "tracehost", + "Hostname to send the trace to", + cxxopts::value(ServerOptions.TraceOptions.Host)->default_value(""), + ""); + + options.add_option("ue-trace", + "", + "tracefile", + "Path to write a trace to", + cxxopts::value(ServerOptions.TraceOptions.File)->default_value(""), + ""); +#endif // ZEN_WITH_TRACE + + options.add_option("stats", + "", + "statsd", + "", + cxxopts::value(ServerOptions.StatsConfig.Enabled)->default_value("false"), + "Enable statsd reporter (localhost:8125)"); +} + +void +ZenServerCmdLineOptions::ApplyOptions(ZenServerOptions& ServerOptions) +{ + ServerOptions.SystemRootDir = MakeSafeAbsolutePath(SystemRootDir); + ServerOptions.DataDir = MakeSafeAbsolutePath(DataDir); + ServerOptions.ContentDir = MakeSafeAbsolutePath(ContentDir); + ServerOptions.AbsLogFile = MakeSafeAbsolutePath(AbsLogFile); + ServerOptions.ConfigFile = MakeSafeAbsolutePath(ConfigFile); +} + +void +ParseCliOptions(int argc, char* argv[], ZenStorageServerOptions& ServerOptions) +{ + for (int i = 0; i < argc; ++i) + { + if (i) + { + ServerOptions.CommandLine.push_back(' '); + } + + ServerOptions.CommandLine += argv[i]; + } + + cxxopts::Options options("zenserver", "Zen Storage Server"); + + ZenServerCmdLineOptions BaseOptions; + BaseOptions.AddCliOptions(options, ServerOptions); + + ZenStorageServerCmdLineOptions StorageOptions; + StorageOptions.AddCliOptions(options, ServerOptions); + + try + { + cxxopts::ParseResult Result; + + try + { + Result = options.parse(argc, argv); + } + catch (const std::exception& Ex) + { + throw OptionParseException(Ex.what(), options.help()); + } + + if (Result.count("help")) + { + ZEN_CONSOLE("{}", options.help()); + +#if ZEN_PLATFORM_WINDOWS + ZEN_CONSOLE("Press any key to exit!"); + _getch(); +#else + // Assume the user's in a terminal on all other platforms and that + // they'll use less/more/etc. if need be. +#endif + + exit(0); + } + + if (!ServerOptions.HasTraceCommandlineOptions) + { + // Apply any Lua settings if we don't have them set from the command line + TraceConfigure(ServerOptions.TraceOptions); + } + + if (ServerOptions.QuietConsole) + { + bool HasExplicitConsoleLevel = false; + for (int i = 0; i < logging::level::LogLevelCount; ++i) + { + if (ServerOptions.Loggers[i].find("console") != std::string::npos) + { + HasExplicitConsoleLevel = true; + break; + } + } + + if (!HasExplicitConsoleLevel) + { + std::string& WarnLoggers = ServerOptions.Loggers[logging::level::Warn]; + if (!WarnLoggers.empty()) + { + WarnLoggers += ","; + } + WarnLoggers += "console"; + } + } + + for (int i = 0; i < logging::level::LogLevelCount; ++i) + { + logging::ConfigureLogLevels(logging::level::LogLevel(i), ServerOptions.Loggers[i]); + } + logging::RefreshLogLevels(); + + BaseOptions.ApplyOptions(ServerOptions); + StorageOptions.ApplyOptions(options, ServerOptions); + + ParseEnvVariables(ServerOptions, Result); + + ZEN_TRACE_CPU("ConfigParse"); + + if (!ServerOptions.ConfigFile.empty()) + { + ParseConfigFile(ServerOptions.ConfigFile, ServerOptions, Result, BaseOptions.OutputConfigFile); + } + else + { + ParseConfigFile(ServerOptions.DataDir / "zen_cfg.lua", ServerOptions, Result, BaseOptions.OutputConfigFile); + } + + if (!ServerOptions.PluginsConfigFile.empty()) + { + ParsePluginsConfigFile(ServerOptions.PluginsConfigFile, ServerOptions, ServerOptions.BasePort); + } + + ValidateOptions(ServerOptions); + } + catch (const OptionParseException& e) + { + ZEN_CONSOLE("{}\n", options.help()); + ZEN_CONSOLE_ERROR("Invalid zenserver arguments: {}", e.what()); + throw; + } + + if (ServerOptions.SystemRootDir.empty()) + { + ServerOptions.SystemRootDir = PickDefaultSystemRootDirectory(); + } + + if (ServerOptions.DataDir.empty()) + { + ServerOptions.DataDir = PickDefaultStateDirectory(ServerOptions.SystemRootDir); + } + + if (ServerOptions.AbsLogFile.empty()) + { + ServerOptions.AbsLogFile = ServerOptions.DataDir / "logs" / "zenserver.log"; + } + + ServerOptions.HttpServerConfig.IsDedicatedServer = ServerOptions.IsDedicated; +} + +} // namespace zen diff --git a/src/zenserver/config/config.h b/src/zenserver/config/config.h new file mode 100644 index 000000000..8471ee89b --- /dev/null +++ b/src/zenserver/config/config.h @@ -0,0 +1,87 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace zen::LuaConfig { +struct Options; +} +namespace cxxopts { +class Options; +class ParseResult; +} // namespace cxxopts +namespace zen { + +struct ZenStorageServerOptions; + +struct ZenStatsConfig +{ + bool Enabled = false; + std::string StatsdHost = "localhost"; + int StatsdPort = 8125; +}; + +struct ZenSentryConfig +{ + bool Disable = false; + bool AllowPII = false; // Allow personally identifiable information in sentry crash reports + std::string Dsn; + std::string Environment; + bool Debug = false; // Enable debug mode for Sentry +}; + +struct ZenServerOptions +{ + HttpServerConfig HttpServerConfig; + ZenSentryConfig SentryConfig; + int BasePort = 8558; // Service listen port (used for both UDP and TCP) + int OwnerPid = 0; // Parent process id (zero for standalone) + bool IsDebug = false; + bool IsCleanStart = false; // Indicates whether all state should be wiped on startup or not + bool IsPowerCycle = false; // When true, the process shuts down immediately after initialization + bool IsTest = false; + bool Detach = true; // Whether zenserver should detach from existing process group (Mac/Linux) + bool NoConsoleOutput = false; // Control default use of stdout for diagnostics + bool QuietConsole = false; // Configure console logger output to level WARN + int CoreLimit = 0; // If set, hardware concurrency queries are capped at this number + bool IsDedicated = false; // Indicates a dedicated/shared instance, with larger resource requirements + bool ShouldCrash = false; // Option for testing crash handling + bool IsFirstRun = false; + std::filesystem::path ConfigFile; // Path to Lua config file + std::filesystem::path SystemRootDir; // System root directory (used for machine level config) + std::filesystem::path ContentDir; // Root directory for serving frontend content (experimental) + std::filesystem::path DataDir; // Root directory for state (used for testing) + std::filesystem::path AbsLogFile; // Absolute path to main log file + std::string ChildId; // Id assigned by parent process (used for lifetime management) + std::string LogId; // Id for tagging log output + std::string Loggers[zen::logging::level::LogLevelCount]; +#if ZEN_WITH_TRACE + bool HasTraceCommandlineOptions = false; + TraceOptions TraceOptions; +#endif + std::string MemoryOptions; // Memory allocation options + std::string CommandLine; + std::string EncryptionKey; // 256 bit AES encryption key + std::string EncryptionIV; // 128 bit AES initialization vector + + ZenStatsConfig StatsConfig; + + bool InstallService = false; // Flag used to initiate service install (temporary) + bool UninstallService = false; // Flag used to initiate service uninstall (temporary) +}; + +void ParseCliOptions(int argc, char* argv[], ZenStorageServerOptions& ServerOptions); + +void EmitCentralManifest(const std::filesystem::path& SystemRoot, Oid Identifier, CbObject Manifest, std::filesystem::path ManifestPath); +std::vector ReadAllCentralManifests(const std::filesystem::path& SystemRoot); + +void AddServerConfigOptions(LuaConfig::Options& LuaOptions, ZenServerOptions& ServerOptions); + +} // namespace zen diff --git a/src/zenserver/diag/logging.cpp b/src/zenserver/diag/logging.cpp index 34d9b05b7..ee36b2ca9 100644 --- a/src/zenserver/diag/logging.cpp +++ b/src/zenserver/diag/logging.cpp @@ -2,7 +2,7 @@ #include "logging.h" -#include "config.h" +#include "config/config.h" #include #include diff --git a/src/zenserver/main.cpp b/src/zenserver/main.cpp index 97847b65d..3119b37c6 100644 --- a/src/zenserver/main.cpp +++ b/src/zenserver/main.cpp @@ -1,7 +1,5 @@ // Copyright Epic Games, Inc. All Rights Reserved. -#include "zenstorageserver.h" - #include #include #include @@ -9,23 +7,22 @@ #include #include #include +#include +#include +#include +#include +#include #include #include #include #include #include #include - -#include -#include -#include -#include -#include - #include #include "diag/logging.h" -#include "storageconfig.h" +#include "storage/storageconfig.h" +#include "storage/zenstorageserver.h" #if ZEN_PLATFORM_WINDOWS # include @@ -39,11 +36,8 @@ #if ZEN_WITH_TESTS # define ZEN_TEST_WITH_RUNNER 1 # include -# include #endif -#include - namespace zen::utils { std::atomic_uint32_t SignalCounter[NSIG] = {0}; diff --git a/src/zenserver/objectstore/objectstore.cpp b/src/zenserver/objectstore/objectstore.cpp deleted file mode 100644 index b1e73c7df..000000000 --- a/src/zenserver/objectstore/objectstore.cpp +++ /dev/null @@ -1,618 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include "zencore/compactbinary.h" -#include "zencore/compactbinarybuilder.h" -#include "zenhttp/httpcommon.h" -#include "zenhttp/httpserver.h" - -#include -#include - -ZEN_THIRD_PARTY_INCLUDES_START -#include -ZEN_THIRD_PARTY_INCLUDES_END - -namespace zen { - -using namespace std::literals; - -ZEN_DEFINE_LOG_CATEGORY_STATIC(LogObj, "obj"sv); - -class CbXmlWriter -{ -public: - explicit CbXmlWriter(StringBuilderBase& InBuilder) : Builder(InBuilder) - { - Builder.Append(""); - Builder << LINE_TERMINATOR_ANSI; - } - - void WriteField(CbFieldView Field) - { - using namespace std::literals; - - bool SkipEndTag = false; - const std::u8string_view Tag = Field.GetU8Name(); - - AppendBeginTag(Tag); - - switch (CbValue Accessor = Field.GetValue(); Accessor.GetType()) - { - case CbFieldType::Null: - Builder << "Null"sv; - break; - case CbFieldType::Object: - case CbFieldType::UniformObject: - { - for (CbFieldView It : Field) - { - WriteField(It); - } - } - break; - case CbFieldType::Array: - case CbFieldType::UniformArray: - { - bool FirstField = true; - for (CbFieldView It : Field) - { - if (!FirstField) - AppendBeginTag(Tag); - - WriteField(It); - AppendEndTag(Tag); - FirstField = false; - } - SkipEndTag = true; - } - break; - case CbFieldType::Binary: - AppendBase64String(Accessor.AsBinary()); - break; - case CbFieldType::String: - Builder << Accessor.AsU8String(); - break; - case CbFieldType::IntegerPositive: - Builder << Accessor.AsIntegerPositive(); - break; - case CbFieldType::IntegerNegative: - Builder << Accessor.AsIntegerNegative(); - break; - case CbFieldType::Float32: - { - const float Value = Accessor.AsFloat32(); - if (std::isfinite(Value)) - { - Builder.Append(fmt::format("{:.9g}", Value)); - } - else - { - Builder << "Null"sv; - } - } - break; - case CbFieldType::Float64: - { - const double Value = Accessor.AsFloat64(); - if (std::isfinite(Value)) - { - Builder.Append(fmt::format("{:.17g}", Value)); - } - else - { - Builder << "null"sv; - } - } - break; - case CbFieldType::BoolFalse: - Builder << "False"sv; - break; - case CbFieldType::BoolTrue: - Builder << "True"sv; - break; - case CbFieldType::ObjectAttachment: - case CbFieldType::BinaryAttachment: - { - Accessor.AsAttachment().ToHexString(Builder); - } - break; - case CbFieldType::Hash: - { - Accessor.AsHash().ToHexString(Builder); - } - break; - case CbFieldType::Uuid: - { - Accessor.AsUuid().ToString(Builder); - } - break; - case CbFieldType::DateTime: - Builder << DateTime(Accessor.AsDateTimeTicks()).ToIso8601(); - break; - case CbFieldType::TimeSpan: - { - const TimeSpan Span(Accessor.AsTimeSpanTicks()); - if (Span.GetDays() == 0) - { - Builder << Span.ToString("%h:%m:%s.%n"); - } - else - { - Builder << Span.ToString("%d.%h:%m:%s.%n"); - } - break; - } - case CbFieldType::ObjectId: - Accessor.AsObjectId().ToString(Builder); - break; - case CbFieldType::CustomById: - { - CbCustomById Custom = Accessor.AsCustomById(); - - AppendBeginTag(u8"Id"sv); - Builder << Custom.Id; - AppendEndTag(u8"Id"sv); - - AppendBeginTag(u8"Data"sv); - AppendBase64String(Custom.Data); - AppendEndTag(u8"Data"sv); - break; - } - case CbFieldType::CustomByName: - { - CbCustomByName Custom = Accessor.AsCustomByName(); - - AppendBeginTag(u8"Name"sv); - Builder << Custom.Name; - AppendEndTag(u8"Name"sv); - - AppendBeginTag(u8"Data"sv); - AppendBase64String(Custom.Data); - AppendEndTag(u8"Data"sv); - break; - } - default: - ZEN_ASSERT(false); - break; - } - - if (!SkipEndTag) - AppendEndTag(Tag); - } - -private: - void AppendBeginTag(std::u8string_view Tag) - { - if (!Tag.empty()) - { - Builder << '<' << Tag << '>'; - } - } - - void AppendEndTag(std::u8string_view Tag) - { - if (!Tag.empty()) - { - Builder << "'; - } - } - - void AppendBase64String(MemoryView Value) - { - Builder << '"'; - ZEN_ASSERT(Value.GetSize() <= 512 * 1024 * 1024); - const uint32_t EncodedSize = Base64::GetEncodedDataSize(uint32_t(Value.GetSize())); - const size_t EncodedIndex = Builder.AddUninitialized(size_t(EncodedSize)); - Base64::Encode(static_cast(Value.GetData()), uint32_t(Value.GetSize()), Builder.Data() + EncodedIndex); - } - -private: - StringBuilderBase& Builder; -}; - -HttpObjectStoreService::HttpObjectStoreService(HttpStatusService& StatusService, ObjectStoreConfig Cfg) -: m_StatusService(StatusService) -, m_Cfg(std::move(Cfg)) -{ - Inititalize(); - m_StatusService.RegisterHandler("obj", *this); -} - -HttpObjectStoreService::~HttpObjectStoreService() -{ - m_StatusService.UnregisterHandler("obj", *this); -} - -const char* -HttpObjectStoreService::BaseUri() const -{ - return "/obj/"; -} - -void -HttpObjectStoreService::HandleRequest(zen::HttpServerRequest& Request) -{ - if (m_Router.HandleRequest(Request) == false) - { - ZEN_LOG_WARN(LogObj, "No route found for {0}", Request.RelativeUri()); - return Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Not found"sv); - } -} - -void -HttpObjectStoreService::HandleStatusRequest(HttpServerRequest& Request) -{ - CbObjectWriter Cbo; - Cbo << "ok" << true; - Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); -} - -void -HttpObjectStoreService::Inititalize() -{ - ZEN_TRACE_CPU("HttpObjectStoreService::Inititalize"); - - namespace fs = std::filesystem; - ZEN_LOG_INFO(LogObj, "Initialzing Object Store in '{}'", m_Cfg.RootDirectory); - - const fs::path BucketsPath = m_Cfg.RootDirectory / "buckets"; - if (!IsDir(BucketsPath)) - { - CreateDirectories(BucketsPath); - } - - m_Router.RegisterRoute( - "bucket", - [this](zen::HttpRouterRequest& Request) { CreateBucket(Request); }, - HttpVerb::kPost | HttpVerb::kPut); - - m_Router.RegisterRoute( - "bucket", - [this](zen::HttpRouterRequest& Request) { DeleteBucket(Request); }, - HttpVerb::kDelete); - - m_Router.RegisterRoute( - "bucket/{path}", - [this](zen::HttpRouterRequest& Request) { - const std::string_view Path = Request.GetCapture(1); - const auto Sep = Path.find_last_of('.'); - const bool IsObject = Sep != std::string::npos && Path.size() - Sep > 0; - - if (IsObject) - { - GetObject(Request, Path); - } - else - { - ListBucket(Request, Path); - } - }, - HttpVerb::kHead | HttpVerb::kGet); - - m_Router.RegisterRoute( - "bucket/{bucket}/{path}", - [this](zen::HttpRouterRequest& Request) { PutObject(Request); }, - HttpVerb::kPost | HttpVerb::kPut); -} - -std::filesystem::path -HttpObjectStoreService::GetBucketDirectory(std::string_view BucketName) -{ - { - std::lock_guard _(BucketsMutex); - - if (const auto It = std::find_if(std::begin(m_Cfg.Buckets), - std::end(m_Cfg.Buckets), - [&BucketName](const auto& Bucket) -> bool { return Bucket.Name == BucketName; }); - It != std::end(m_Cfg.Buckets)) - { - return It->Directory.make_preferred(); - } - } - - return (m_Cfg.RootDirectory / "buckets" / BucketName).make_preferred(); -} - -void -HttpObjectStoreService::CreateBucket(zen::HttpRouterRequest& Request) -{ - namespace fs = std::filesystem; - - const CbObject Params = Request.ServerRequest().ReadPayloadObject(); - const std::string_view BucketName = Params["bucketname"].AsString(); - - if (BucketName.empty()) - { - return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); - } - - const fs::path BucketPath = m_Cfg.RootDirectory / "buckets" / BucketName; - { - std::lock_guard _(BucketsMutex); - if (!IsDir(BucketPath)) - { - CreateDirectories(BucketPath); - ZEN_LOG_INFO(LogObj, "CREATE - new bucket '{}' OK", BucketName); - return Request.ServerRequest().WriteResponse(HttpResponseCode::Created); - } - } - - ZEN_LOG_INFO(LogObj, "CREATE - existing bucket '{}' OK", BucketName); - Request.ServerRequest().WriteResponse(HttpResponseCode::OK); -} - -void -HttpObjectStoreService::ListBucket(zen::HttpRouterRequest& Request, const std::string_view Path) -{ - namespace fs = std::filesystem; - - const auto Sep = Path.find_first_of('/'); - const std::string BucketName{Sep == std::string::npos ? Path : Path.substr(0, Sep)}; - if (BucketName.empty()) - { - return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); - } - - std::string BucketPrefix{Sep == std::string::npos || Sep == Path.size() - 1 ? std::string() : Path.substr(BucketName.size() + 1)}; - if (BucketPrefix.empty()) - { - const auto QueryParms = Request.ServerRequest().GetQueryParams(); - if (auto PrefixParam = QueryParms.GetValue("prefix"); PrefixParam.empty() == false) - { - BucketPrefix = PrefixParam; - } - } - BucketPrefix.erase(0, BucketPrefix.find_first_not_of('/')); - BucketPrefix.erase(0, BucketPrefix.find_first_not_of('\\')); - - const fs::path BucketRoot = GetBucketDirectory(BucketName); - const fs::path RelativeBucketPath = fs::path(BucketPrefix).make_preferred(); - const fs::path FullPath = BucketRoot / RelativeBucketPath; - - struct Visitor : FileSystemTraversal::TreeVisitor - { - Visitor(const std::string_view BucketName, const fs::path& Path, const fs::path& Prefix) : BucketPath(Path) - { - Writer.BeginObject("ListBucketResult"sv); - Writer << "Name"sv << BucketName; - std::string Tmp = Prefix.string(); - std::replace(Tmp.begin(), Tmp.end(), '\\', '/'); - Writer << "Prefix"sv << Tmp; - Writer.BeginArray("Contents"sv); - } - - void VisitFile(const fs::path& Parent, const path_view& File, uint64_t FileSize, uint32_t, uint64_t) override - { - const fs::path FullPath = Parent / fs::path(File); - fs::path RelativePath = fs::relative(FullPath, BucketPath); - - std::string Key = RelativePath.string(); - std::replace(Key.begin(), Key.end(), '\\', '/'); - - Writer.BeginObject(); - Writer << "Key"sv << Key; - Writer << "Size"sv << FileSize; - Writer.EndObject(); - } - - bool VisitDirectory(const std::filesystem::path&, const path_view&, uint32_t) override { return false; } - - CbObject GetResult() - { - Writer.EndArray(); - Writer.EndObject(); - return Writer.Save(); - } - - CbObjectWriter Writer; - fs::path BucketPath; - }; - - Visitor FileVisitor(BucketName, BucketRoot, RelativeBucketPath); - FileSystemTraversal Traversal; - - if (IsDir(FullPath)) - { - std::lock_guard _(BucketsMutex); - Traversal.TraverseFileSystem(FullPath, FileVisitor); - } - CbObject Result = FileVisitor.GetResult(); - - if (Request.ServerRequest().AcceptContentType() == HttpContentType::kJSON) - { - ExtendableStringBuilder<1024> Sb; - return Request.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kJSON, Result.ToJson(Sb).ToView()); - } - - ExtendableStringBuilder<1024> Xml; - CbXmlWriter XmlWriter(Xml); - XmlWriter.WriteField(Result.AsFieldView()); - - Request.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kXML, Xml.ToView()); -} - -void -HttpObjectStoreService::DeleteBucket(zen::HttpRouterRequest& Request) -{ - namespace fs = std::filesystem; - - const CbObject Params = Request.ServerRequest().ReadPayloadObject(); - const std::string_view BucketName = Params["bucketname"].AsString(); - - if (BucketName.empty()) - { - return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); - } - - const fs::path BucketPath = m_Cfg.RootDirectory / "buckets" / BucketName; - { - std::lock_guard _(BucketsMutex); - DeleteDirectories(BucketPath); - } - - ZEN_LOG_INFO(LogObj, "DELETE - bucket '{}' OK", BucketName); - Request.ServerRequest().WriteResponse(HttpResponseCode::OK); -} - -void -HttpObjectStoreService::GetObject(zen::HttpRouterRequest& Request, const std::string_view Path) -{ - namespace fs = std::filesystem; - - const auto Sep = Path.find_first_of('/'); - const std::string BucketName{Sep == std::string::npos ? Path : Path.substr(0, Sep)}; - const std::string BucketPrefix{Sep == std::string::npos || Sep == Path.size() - 1 ? std::string() : Path.substr(BucketName.size() + 1)}; - - const fs::path BucketDir = GetBucketDirectory(BucketName); - - if (BucketDir.empty()) - { - ZEN_LOG_DEBUG(LogObj, "GET - [FAILED], unknown bucket '{}'", BucketName); - return Request.ServerRequest().WriteResponse(HttpResponseCode::NotFound); - } - - const fs::path RelativeBucketPath = fs::path(BucketPrefix).make_preferred(); - - if (RelativeBucketPath.is_absolute() || RelativeBucketPath.string().starts_with("..")) - { - ZEN_LOG_DEBUG(LogObj, "GET - from bucket '{}' [FAILED], invalid file path", BucketName); - return Request.ServerRequest().WriteResponse(HttpResponseCode::Forbidden); - } - - const fs::path FilePath = BucketDir / RelativeBucketPath; - if (!IsFile(FilePath)) - { - ZEN_LOG_DEBUG(LogObj, "GET - '{}/{}' [FAILED], doesn't exist", BucketName, FilePath); - return Request.ServerRequest().WriteResponse(HttpResponseCode::NotFound); - } - - zen::HttpRanges Ranges; - if (Request.ServerRequest().TryGetRanges(Ranges); Ranges.size() > 1) - { - // Only a single range is supported - return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); - } - - FileContents File; - { - std::lock_guard _(BucketsMutex); - File = ReadFile(FilePath); - } - - if (File.ErrorCode) - { - ZEN_LOG_WARN(LogObj, - "GET - '{}/{}' [FAILED] ('{}': {})", - BucketName, - FilePath, - File.ErrorCode.category().name(), - File.ErrorCode.value()); - - return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); - } - - const IoBuffer& FileBuf = File.Data[0]; - - if (Ranges.empty()) - { - const uint64_t TotalServed = TotalBytesServed.fetch_add(FileBuf.Size()) + FileBuf.Size(); - - ZEN_LOG_DEBUG(LogObj, - "GET - '{}/{}' ({}) [OK] (Served: {})", - BucketName, - RelativeBucketPath, - NiceBytes(FileBuf.Size()), - NiceBytes(TotalServed)); - - Request.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, FileBuf); - } - else - { - const auto Range = Ranges[0]; - const uint64_t RangeSize = 1 + (Range.End - Range.Start); - const uint64_t TotalServed = TotalBytesServed.fetch_add(RangeSize) + RangeSize; - - ZEN_LOG_DEBUG(LogObj, - "GET - '{}/{}' (Range: {}-{}) ({}/{}) [OK] (Served: {})", - BucketName, - RelativeBucketPath, - Range.Start, - Range.End, - NiceBytes(RangeSize), - NiceBytes(FileBuf.Size()), - NiceBytes(TotalServed)); - - MemoryView RangeView = FileBuf.GetView().Mid(Range.Start, RangeSize); - if (RangeView.GetSize() != RangeSize) - { - return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); - } - - IoBuffer RangeBuf = IoBuffer(IoBuffer::Wrap, RangeView.GetData(), RangeView.GetSize()); - Request.ServerRequest().WriteResponse(HttpResponseCode::PartialContent, HttpContentType::kBinary, RangeBuf); - } -} - -void -HttpObjectStoreService::PutObject(zen::HttpRouterRequest& Request) -{ - namespace fs = std::filesystem; - - const std::string_view BucketName = Request.GetCapture(1); - const fs::path BucketDir = GetBucketDirectory(BucketName); - - if (BucketDir.empty()) - { - ZEN_LOG_DEBUG(LogObj, "PUT - [FAILED], unknown bucket '{}'", BucketName); - return Request.ServerRequest().WriteResponse(HttpResponseCode::NotFound); - } - - const fs::path RelativeBucketPath = fs::path(Request.GetCapture(2)).make_preferred(); - - if (RelativeBucketPath.is_absolute() || RelativeBucketPath.string().starts_with("..")) - { - ZEN_LOG_DEBUG(LogObj, "PUT - bucket '{}' [FAILED], invalid file path", BucketName); - return Request.ServerRequest().WriteResponse(HttpResponseCode::Forbidden); - } - - const fs::path FilePath = BucketDir / RelativeBucketPath; - const fs::path FileDirectory = FilePath.parent_path(); - - { - std::lock_guard _(BucketsMutex); - - if (!IsDir(FileDirectory)) - { - CreateDirectories(FileDirectory); - } - - const IoBuffer FileBuf = Request.ServerRequest().ReadPayload(); - - if (FileBuf.Size() == 0) - { - ZEN_LOG_DEBUG(LogObj, "PUT - '{}' [FAILED], empty file", FilePath); - return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); - } - - TemporaryFile::SafeWriteFile(FilePath, FileBuf.GetView()); - - ZEN_LOG_DEBUG(LogObj, - "PUT - '{}' [OK] ({})", - (fs::path(BucketName) / RelativeBucketPath).make_preferred(), - NiceBytes(FileBuf.Size())); - } - - Request.ServerRequest().WriteResponse(HttpResponseCode::OK); -} - -} // namespace zen diff --git a/src/zenserver/objectstore/objectstore.h b/src/zenserver/objectstore/objectstore.h deleted file mode 100644 index 44e50e208..000000000 --- a/src/zenserver/objectstore/objectstore.h +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include -#include -#include -#include -#include - -namespace zen { - -class HttpRouterRequest; - -struct ObjectStoreConfig -{ - struct BucketConfig - { - std::string Name; - std::filesystem::path Directory; - }; - - std::filesystem::path RootDirectory; - std::vector Buckets; -}; - -class HttpObjectStoreService final : public zen::HttpService, public IHttpStatusProvider -{ -public: - HttpObjectStoreService(HttpStatusService& StatusService, ObjectStoreConfig Cfg); - virtual ~HttpObjectStoreService(); - - virtual const char* BaseUri() const override; - virtual void HandleRequest(zen::HttpServerRequest& Request) override; - virtual void HandleStatusRequest(HttpServerRequest& Request) override; - -private: - void Inititalize(); - std::filesystem::path GetBucketDirectory(std::string_view BucketName); - void CreateBucket(zen::HttpRouterRequest& Request); - void ListBucket(zen::HttpRouterRequest& Request, const std::string_view Path); - void DeleteBucket(zen::HttpRouterRequest& Request); - void GetObject(zen::HttpRouterRequest& Request, const std::string_view Path); - void PutObject(zen::HttpRouterRequest& Request); - - HttpStatusService& m_StatusService; - ObjectStoreConfig m_Cfg; - std::mutex BucketsMutex; - HttpRequestRouter m_Router; - std::atomic_uint64_t TotalBytesServed{0}; -}; - -} // namespace zen diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp deleted file mode 100644 index 1c6b5d6b0..000000000 --- a/src/zenserver/projectstore/httpprojectstore.cpp +++ /dev/null @@ -1,3307 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "httpprojectstore.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace zen { - -const FLLMTag& -GetProjectHttpTag() -{ - static FLLMTag _("http", FLLMTag("project")); - - return _; -} - -void -CSVHeader(bool Details, bool AttachmentDetails, StringBuilderBase& CSVWriter) -{ - if (AttachmentDetails) - { - CSVWriter << "Project, Oplog, LSN, Key, Cid, Size"; - } - else if (Details) - { - CSVWriter << "Project, Oplog, LSN, Key, Size, AttachmentCount, AttachmentsSize"; - } - else - { - CSVWriter << "Project, Oplog, Key"; - } -} - -void -CSVWriteOp(CidStore& CidStore, - std::string_view ProjectId, - std::string_view OplogId, - bool Details, - bool AttachmentDetails, - ProjectStore::LogSequenceNumber LSN, - const Oid& Key, - CbObjectView Op, - StringBuilderBase& CSVWriter) -{ - StringBuilder<32> KeyStringBuilder; - Key.ToString(KeyStringBuilder); - const std::string_view KeyString = KeyStringBuilder.ToView(); - - if (AttachmentDetails) - { - Op.IterateAttachments([&CidStore, &CSVWriter, &ProjectId, &OplogId, LSN, &KeyString](CbFieldView FieldView) { - const IoHash AttachmentHash = FieldView.AsAttachment(); - IoBuffer Attachment = CidStore.FindChunkByCid(AttachmentHash); - CSVWriter << "\r\n" - << ProjectId << ", " << OplogId << ", " << LSN.Number << ", " << KeyString << ", " << AttachmentHash.ToHexString() - << ", " << gsl::narrow(Attachment.GetSize()); - }); - } - else if (Details) - { - uint64_t AttachmentCount = 0; - size_t AttachmentsSize = 0; - Op.IterateAttachments([&CidStore, &AttachmentCount, &AttachmentsSize](CbFieldView FieldView) { - const IoHash AttachmentHash = FieldView.AsAttachment(); - AttachmentCount++; - IoBuffer Attachment = CidStore.FindChunkByCid(AttachmentHash); - AttachmentsSize += Attachment.GetSize(); - }); - CSVWriter << "\r\n" - << ProjectId << ", " << OplogId << ", " << LSN.Number << ", " << KeyString << ", " << gsl::narrow(Op.GetSize()) - << ", " << AttachmentCount << ", " << gsl::narrow(AttachmentsSize); - } - else - { - CSVWriter << "\r\n" << ProjectId << ", " << OplogId << ", " << KeyString; - } -}; - -////////////////////////////////////////////////////////////////////////// - -namespace { - - void CbWriteOp(CidStore& CidStore, - bool Details, - bool OpDetails, - bool AttachmentDetails, - ProjectStore::LogSequenceNumber LSN, - const Oid& Key, - CbObjectView Op, - CbObjectWriter& CbWriter) - { - CbWriter.BeginObject(); - { - CbWriter.AddObjectId("key", Key); - if (Details) - { - CbWriter.AddInteger("lsn", LSN.Number); - CbWriter.AddInteger("size", gsl::narrow(Op.GetSize())); - } - if (AttachmentDetails) - { - CbWriter.BeginArray("attachments"); - Op.IterateAttachments([&CidStore, &CbWriter](CbFieldView FieldView) { - const IoHash AttachmentHash = FieldView.AsAttachment(); - CbWriter.BeginObject(); - { - IoBuffer Attachment = CidStore.FindChunkByCid(AttachmentHash); - CbWriter.AddString("cid", AttachmentHash.ToHexString()); - CbWriter.AddInteger("size", gsl::narrow(Attachment.GetSize())); - } - CbWriter.EndObject(); - }); - CbWriter.EndArray(); - } - else if (Details) - { - uint64_t AttachmentCount = 0; - size_t AttachmentsSize = 0; - Op.IterateAttachments([&CidStore, &AttachmentCount, &AttachmentsSize](CbFieldView FieldView) { - const IoHash AttachmentHash = FieldView.AsAttachment(); - AttachmentCount++; - IoBuffer Attachment = CidStore.FindChunkByCid(AttachmentHash); - AttachmentsSize += Attachment.GetSize(); - }); - if (AttachmentCount > 0) - { - CbWriter.AddInteger("attachments", AttachmentCount); - CbWriter.AddInteger("attachmentssize", gsl::narrow(AttachmentsSize)); - } - } - if (OpDetails) - { - CbWriter.BeginObject("op"); - for (const CbFieldView& Field : Op) - { - if (!Field.HasName()) - { - CbWriter.AddField(Field); - continue; - } - std::string_view FieldName = Field.GetName(); - CbWriter.AddField(FieldName, Field); - } - CbWriter.EndObject(); - } - } - CbWriter.EndObject(); - }; - - void CbWriteOplogOps(CidStore& CidStore, - ProjectStore::Oplog& Oplog, - bool Details, - bool OpDetails, - bool AttachmentDetails, - CbObjectWriter& Cbo) - { - Cbo.BeginArray("ops"); - { - Oplog.IterateOplogWithKey([&Cbo, &CidStore, Details, OpDetails, AttachmentDetails](ProjectStore::LogSequenceNumber LSN, - const Oid& Key, - CbObjectView Op) { - CbWriteOp(CidStore, Details, OpDetails, AttachmentDetails, LSN, Key, Op, Cbo); - }); - } - Cbo.EndArray(); - } - - void CbWriteOplog(CidStore& CidStore, - ProjectStore::Oplog& Oplog, - bool Details, - bool OpDetails, - bool AttachmentDetails, - CbObjectWriter& Cbo) - { - Cbo.BeginObject(); - { - Cbo.AddString("name", Oplog.OplogId()); - CbWriteOplogOps(CidStore, Oplog, Details, OpDetails, AttachmentDetails, Cbo); - } - Cbo.EndObject(); - } - - void CbWriteOplogs(CidStore& CidStore, - ProjectStore::Project& Project, - std::vector OpLogs, - bool Details, - bool OpDetails, - bool AttachmentDetails, - CbObjectWriter& Cbo) - { - Cbo.BeginArray("oplogs"); - { - for (const std::string& OpLogId : OpLogs) - { - Ref Oplog = Project.OpenOplog(OpLogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ true); - if (Oplog) - { - CbWriteOplog(CidStore, *Oplog, Details, OpDetails, AttachmentDetails, Cbo); - } - } - } - Cbo.EndArray(); - } - - void CbWriteProject(CidStore& CidStore, - ProjectStore::Project& Project, - std::vector OpLogs, - bool Details, - bool OpDetails, - bool AttachmentDetails, - CbObjectWriter& Cbo) - { - Cbo.BeginObject(); - { - Cbo.AddString("name", Project.Identifier); - CbWriteOplogs(CidStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo); - } - Cbo.EndObject(); - } - - struct CreateRemoteStoreResult - { - std::shared_ptr Store; - std::string Description; - }; - - CreateRemoteStoreResult CreateRemoteStore(CbObjectView Params, - AuthMgr& AuthManager, - size_t MaxBlockSize, - size_t MaxChunkEmbedSize, - const std::filesystem::path& TempFilePath) - { - ZEN_MEMSCOPE(GetProjectHttpTag()); - - using namespace std::literals; - - std::shared_ptr RemoteStore; - - if (CbObjectView File = Params["file"sv].AsObjectView(); File) - { - std::filesystem::path FolderPath(File["path"sv].AsString()); - if (FolderPath.empty()) - { - return {nullptr, "Missing file path"}; - } - std::string_view Name(File["name"sv].AsString()); - if (Name.empty()) - { - return {nullptr, "Missing file name"}; - } - std::string_view OptionalBaseName(File["basename"sv].AsString()); - bool ForceDisableBlocks = File["disableblocks"sv].AsBool(false); - bool ForceEnableTempBlocks = File["enabletempblocks"sv].AsBool(false); - - FileRemoteStoreOptions Options = { - RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize}, - FolderPath, - std::string(Name), - std::string(OptionalBaseName), - ForceDisableBlocks, - ForceEnableTempBlocks}; - RemoteStore = CreateFileRemoteStore(Options); - } - - if (CbObjectView Cloud = Params["cloud"sv].AsObjectView(); Cloud) - { - std::string_view CloudServiceUrl = Cloud["url"sv].AsString(); - if (CloudServiceUrl.empty()) - { - return {nullptr, "Missing service url"}; - } - - std::string Url = UrlDecode(CloudServiceUrl); - std::string_view Namespace = Cloud["namespace"sv].AsString(); - if (Namespace.empty()) - { - return {nullptr, "Missing namespace"}; - } - std::string_view Bucket = Cloud["bucket"sv].AsString(); - if (Bucket.empty()) - { - return {nullptr, "Missing bucket"}; - } - std::string_view OpenIdProvider = Cloud["openid-provider"sv].AsString(); - std::string AccessToken = std::string(Cloud["access-token"sv].AsString()); - if (AccessToken.empty()) - { - std::string_view AccessTokenEnvVariable = Cloud["access-token-env"].AsString(); - if (!AccessTokenEnvVariable.empty()) - { - AccessToken = GetEnvVariable(AccessTokenEnvVariable); - } - } - std::filesystem::path OidcExePath; - if (std::string_view OidcExePathString = Cloud["oidc-exe-path"].AsString(); !OidcExePathString.empty()) - { - std::filesystem::path OidcExePathMaybe(OidcExePathString); - if (IsFile(OidcExePathMaybe)) - { - OidcExePath = std::move(OidcExePathMaybe); - } - else - { - ZEN_WARN("Path to OidcToken executable '{}' can not be reached by server", OidcExePathString); - } - } - std::string_view KeyParam = Cloud["key"sv].AsString(); - if (KeyParam.empty()) - { - return {nullptr, "Missing key"}; - } - if (KeyParam.length() != IoHash::StringLength) - { - return {nullptr, "Invalid key"}; - } - IoHash Key = IoHash::FromHexString(KeyParam); - if (Key == IoHash::Zero) - { - return {nullptr, "Invalid key string"}; - } - IoHash BaseKey = IoHash::Zero; - std::string_view BaseKeyParam = Cloud["basekey"sv].AsString(); - if (!BaseKeyParam.empty()) - { - if (BaseKeyParam.length() != IoHash::StringLength) - { - return {nullptr, "Invalid base key"}; - } - BaseKey = IoHash::FromHexString(BaseKeyParam); - if (BaseKey == IoHash::Zero) - { - return {nullptr, "Invalid base key string"}; - } - } - - bool ForceDisableBlocks = Cloud["disableblocks"sv].AsBool(false); - bool ForceDisableTempBlocks = Cloud["disabletempblocks"sv].AsBool(false); - bool AssumeHttp2 = Cloud["assumehttp2"sv].AsBool(false); - - JupiterRemoteStoreOptions Options = { - RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize}, - Url, - std::string(Namespace), - std::string(Bucket), - Key, - BaseKey, - std::string(OpenIdProvider), - AccessToken, - AuthManager, - OidcExePath, - ForceDisableBlocks, - ForceDisableTempBlocks, - AssumeHttp2}; - RemoteStore = CreateJupiterRemoteStore(Options, TempFilePath, /*Quiet*/ false, /*Unattended*/ false, /*Hidden*/ true); - } - - if (CbObjectView Zen = Params["zen"sv].AsObjectView(); Zen) - { - std::string_view Url = Zen["url"sv].AsString(); - std::string_view Project = Zen["project"sv].AsString(); - if (Project.empty()) - { - return {nullptr, "Missing project"}; - } - std::string_view Oplog = Zen["oplog"sv].AsString(); - if (Oplog.empty()) - { - return {nullptr, "Missing oplog"}; - } - ZenRemoteStoreOptions Options = { - RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize}, - std::string(Url), - std::string(Project), - std::string(Oplog)}; - RemoteStore = CreateZenRemoteStore(Options, TempFilePath); - } - - if (CbObjectView Builds = Params["builds"sv].AsObjectView(); Builds) - { - std::string_view BuildsServiceUrl = Builds["url"sv].AsString(); - if (BuildsServiceUrl.empty()) - { - return {nullptr, "Missing service url"}; - } - - std::string Url = UrlDecode(BuildsServiceUrl); - std::string_view Namespace = Builds["namespace"sv].AsString(); - if (Namespace.empty()) - { - return {nullptr, "Missing namespace"}; - } - std::string_view Bucket = Builds["bucket"sv].AsString(); - if (Bucket.empty()) - { - return {nullptr, "Missing bucket"}; - } - std::string_view OpenIdProvider = Builds["openid-provider"sv].AsString(); - std::string AccessToken = std::string(Builds["access-token"sv].AsString()); - if (AccessToken.empty()) - { - std::string_view AccessTokenEnvVariable = Builds["access-token-env"].AsString(); - if (!AccessTokenEnvVariable.empty()) - { - AccessToken = GetEnvVariable(AccessTokenEnvVariable); - } - } - std::filesystem::path OidcExePath; - if (std::string_view OidcExePathString = Builds["oidc-exe-path"].AsString(); !OidcExePathString.empty()) - { - std::filesystem::path OidcExePathMaybe(OidcExePathString); - if (IsFile(OidcExePathMaybe)) - { - OidcExePath = std::move(OidcExePathMaybe); - } - else - { - ZEN_WARN("Path to OidcToken executable '{}' can not be reached by server", OidcExePathString); - } - } - std::string_view BuildIdParam = Builds["buildsid"sv].AsString(); - if (BuildIdParam.empty()) - { - return {nullptr, "Missing build id"}; - } - if (BuildIdParam.length() != Oid::StringLength) - { - return {nullptr, "Invalid build id"}; - } - Oid BuildId = Oid::FromHexString(BuildIdParam); - if (BuildId == Oid::Zero) - { - return {nullptr, "Invalid build id string"}; - } - - bool ForceDisableBlocks = Builds["disableblocks"sv].AsBool(false); - bool ForceDisableTempBlocks = Builds["disabletempblocks"sv].AsBool(false); - bool AssumeHttp2 = Builds["assumehttp2"sv].AsBool(false); - - MemoryView MetaDataSection = Builds["metadata"sv].AsBinaryView(); - IoBuffer MetaData(IoBuffer::Wrap, MetaDataSection.GetData(), MetaDataSection.GetSize()); - - BuildsRemoteStoreOptions Options = { - RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize}, - Url, - std::string(Namespace), - std::string(Bucket), - BuildId, - std::string(OpenIdProvider), - AccessToken, - AuthManager, - OidcExePath, - ForceDisableBlocks, - ForceDisableTempBlocks, - AssumeHttp2, - MetaData}; - RemoteStore = CreateJupiterBuildsRemoteStore(Options, TempFilePath, /*Quiet*/ false, /*Unattended*/ false, /*Hidden*/ true); - } - - if (!RemoteStore) - { - return {nullptr, "Unknown remote store type"}; - } - - return {std::move(RemoteStore), ""}; - } - - std::pair ConvertResult(const RemoteProjectStore::Result& Result) - { - if (Result.ErrorCode == 0) - { - return {HttpResponseCode::OK, Result.Text}; - } - return {static_cast(Result.ErrorCode), - Result.Reason.empty() ? Result.Text - : Result.Text.empty() ? Result.Reason - : fmt::format("{}: {}", Result.Reason, Result.Text)}; - } - -} // namespace - -////////////////////////////////////////////////////////////////////////// - -HttpProjectService::HttpProjectService(CidStore& Store, - ProjectStore* Projects, - HttpStatusService& StatusService, - HttpStatsService& StatsService, - AuthMgr& AuthMgr, - OpenProcessCache& InOpenProcessCache, - JobQueue& InJobQueue) -: m_Log(logging::Get("project")) -, m_CidStore(Store) -, m_ProjectStore(Projects) -, m_StatusService(StatusService) -, m_StatsService(StatsService) -, m_AuthMgr(AuthMgr) -, m_OpenProcessCache(InOpenProcessCache) -, m_JobQueue(InJobQueue) -{ - ZEN_MEMSCOPE(GetProjectHttpTag()); - - using namespace std::literals; - - m_Router.AddPattern("project", "([[:alnum:]_.]+)"); - m_Router.AddPattern("log", "([[:alnum:]_.]+)"); - m_Router.AddPattern("op", "([[:digit:]]+?)"); - m_Router.AddPattern("chunk", "([[:xdigit:]]{24})"); - m_Router.AddPattern("hash", "([[:xdigit:]]{40})"); - - m_Router.RegisterRoute( - "", - [this](HttpRouterRequest& Req) { HandleProjectListRequest(Req); }, - HttpVerb::kGet); - - m_Router.RegisterRoute( - "list", - [this](HttpRouterRequest& Req) { HandleProjectListRequest(Req); }, - HttpVerb::kGet); - - m_Router.RegisterRoute( - "{project}/oplog/{log}/batch", - [this](HttpRouterRequest& Req) { HandleChunkBatchRequest(Req); }, - HttpVerb::kPost); - - m_Router.RegisterRoute( - "{project}/oplog/{log}/files", - [this](HttpRouterRequest& Req) { HandleFilesRequest(Req); }, - HttpVerb::kGet); - - m_Router.RegisterRoute( - "{project}/oplog/{log}/chunkinfos", - [this](HttpRouterRequest& Req) { HandleChunkInfosRequest(Req); }, - HttpVerb::kGet); - - m_Router.RegisterRoute( - "{project}/oplog/{log}/{chunk}/info", - [this](HttpRouterRequest& Req) { HandleChunkInfoRequest(Req); }, - HttpVerb::kGet); - - m_Router.RegisterRoute( - "{project}/oplog/{log}/{chunk}", - [this](HttpRouterRequest& Req) { HandleChunkByIdRequest(Req); }, - HttpVerb::kGet | HttpVerb::kHead); - - m_Router.RegisterRoute( - "{project}/oplog/{log}/{hash}", - [this](HttpRouterRequest& Req) { HandleChunkByCidRequest(Req); }, - HttpVerb::kGet | HttpVerb::kPost); - - m_Router.RegisterRoute( - "{project}/oplog/{log}/prep", - [this](HttpRouterRequest& Req) { HandleOplogOpPrepRequest(Req); }, - HttpVerb::kPost); - - m_Router.RegisterRoute( - "{project}/oplog/{log}/new", - [this](HttpRouterRequest& Req) { HandleOplogOpNewRequest(Req); }, - HttpVerb::kPost); - - m_Router.RegisterRoute( - "{project}/oplog/{log}/validate", - [this](HttpRouterRequest& Req) { HandleOplogValidateRequest(Req); }, - HttpVerb::kPost); - - m_Router.RegisterRoute( - "{project}/oplog/{log}/{op}", - [this](HttpRouterRequest& Req) { HandleOpLogOpRequest(Req); }, - HttpVerb::kGet); - - m_Router.RegisterRoute( - "{project}/oplog/{log}", - [this](HttpRouterRequest& Req) { HandleOpLogRequest(Req); }, - HttpVerb::kGet | HttpVerb::kPut | HttpVerb::kPost | HttpVerb::kDelete); - - m_Router.RegisterRoute( - "{project}/oplog/{log}/entries", - [this](HttpRouterRequest& Req) { HandleOpLogEntriesRequest(Req); }, - HttpVerb::kGet); - - m_Router.RegisterRoute( - "{project}", - [this](HttpRouterRequest& Req) { HandleProjectRequest(Req); }, - HttpVerb::kGet | HttpVerb::kPut | HttpVerb::kPost | HttpVerb::kDelete); - - // Push a oplog container - m_Router.RegisterRoute( - "{project}/oplog/{log}/save", - [this](HttpRouterRequest& Req) { HandleOplogSaveRequest(Req); }, - HttpVerb::kPost); - - // Pull a oplog container - m_Router.RegisterRoute( - "{project}/oplog/{log}/load", - [this](HttpRouterRequest& Req) { HandleOplogLoadRequest(Req); }, - HttpVerb::kGet); - - // Do an rpc style operation on project/oplog - m_Router.RegisterRoute( - "{project}/oplog/{log}/rpc", - [this](HttpRouterRequest& Req) { HandleRpcRequest(Req); }, - HttpVerb::kPost); - - m_Router.RegisterRoute( - "details\\$", - [this](HttpRouterRequest& Req) { HandleDetailsRequest(Req); }, - HttpVerb::kGet); - - m_Router.RegisterRoute( - "details\\$/{project}", - [this](HttpRouterRequest& Req) { HandleProjectDetailsRequest(Req); }, - HttpVerb::kGet); - - m_Router.RegisterRoute( - "details\\$/{project}/{log}", - [this](HttpRouterRequest& Req) { HandleOplogDetailsRequest(Req); }, - HttpVerb::kGet); - - m_Router.RegisterRoute( - "details\\$/{project}/{log}/{chunk}", - [this](HttpRouterRequest& Req) { HandleOplogOpDetailsRequest(Req); }, - HttpVerb::kGet); - - m_StatusService.RegisterHandler("prj", *this); - m_StatsService.RegisterHandler("prj", *this); -} - -HttpProjectService::~HttpProjectService() -{ - m_StatsService.UnregisterHandler("prj", *this); - m_StatusService.UnregisterHandler("prj", *this); -} - -const char* -HttpProjectService::BaseUri() const -{ - return "/prj/"; -} - -void -HttpProjectService::HandleRequest(HttpServerRequest& Request) -{ - m_ProjectStats.RequestCount++; - - ZEN_MEMSCOPE(GetProjectHttpTag()); - - metrics::OperationTiming::Scope $(m_HttpRequests); - - if (m_Router.HandleRequest(Request) == false) - { - m_ProjectStats.BadRequestCount++; - ZEN_WARN("No route found for {0}", Request.RelativeUri()); - } -} - -void -HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq) -{ - ZEN_TRACE_CPU("ProjectService::Stats"); - - const GcStorageSize StoreSize = m_ProjectStore->StorageSize(); - const CidStoreSize CidSize = m_CidStore.TotalSize(); - - CbObjectWriter Cbo; - - EmitSnapshot("requests", m_HttpRequests, Cbo); - - Cbo.BeginObject("store"); - { - Cbo.BeginObject("size"); - { - Cbo << "disk" << StoreSize.DiskSize; - Cbo << "memory" << StoreSize.MemorySize; - } - Cbo.EndObject(); - - Cbo.BeginObject("project"); - { - Cbo << "readcount" << m_ProjectStats.ProjectReadCount << "writecount" << m_ProjectStats.ProjectWriteCount << "deletecount" - << m_ProjectStats.ProjectDeleteCount; - } - Cbo.EndObject(); - - Cbo.BeginObject("oplog"); - { - Cbo << "readcount" << m_ProjectStats.OpLogReadCount << "writecount" << m_ProjectStats.OpLogWriteCount << "deletecount" - << m_ProjectStats.OpLogDeleteCount; - } - Cbo.EndObject(); - - Cbo.BeginObject("op"); - { - Cbo << "hitcount" << m_ProjectStats.OpHitCount << "misscount" << m_ProjectStats.OpMissCount << "writecount" - << m_ProjectStats.OpWriteCount; - } - Cbo.EndObject(); - - Cbo.BeginObject("chunk"); - { - Cbo << "hitcount" << m_ProjectStats.ChunkHitCount << "misscount" << m_ProjectStats.ChunkMissCount << "writecount" - << m_ProjectStats.ChunkWriteCount; - } - Cbo.EndObject(); - - Cbo << "requestcount" << m_ProjectStats.RequestCount; - Cbo << "badrequestcount" << m_ProjectStats.BadRequestCount; - } - Cbo.EndObject(); - - Cbo.BeginObject("cid"); - { - Cbo.BeginObject("size"); - { - Cbo << "tiny" << CidSize.TinySize; - Cbo << "small" << CidSize.SmallSize; - Cbo << "large" << CidSize.LargeSize; - Cbo << "total" << CidSize.TotalSize; - } - Cbo.EndObject(); - } - Cbo.EndObject(); - - return HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); -} - -void -HttpProjectService::HandleStatusRequest(HttpServerRequest& Request) -{ - ZEN_TRACE_CPU("HttpProjectService::Status"); - CbObjectWriter Cbo; - Cbo << "ok" << true; - Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); -} - -void -HttpProjectService::HandleProjectListRequest(HttpRouterRequest& Req) -{ - ZEN_TRACE_CPU("ProjectService::ProjectList"); - - HttpServerRequest& HttpReq = Req.ServerRequest(); - CbArray ProjectsList = m_ProjectStore->GetProjectsList(); - HttpReq.WriteResponse(HttpResponseCode::OK, ProjectsList); -} - -void -HttpProjectService::HandleChunkBatchRequest(HttpRouterRequest& Req) -{ - ZEN_TRACE_CPU("ProjectService::ChunkBatch"); - - HttpServerRequest& HttpReq = Req.ServerRequest(); - const auto& ProjectId = Req.GetCapture(1); - const auto& OplogId = Req.GetCapture(2); - - Ref Project = m_ProjectStore->OpenProject(ProjectId); - if (!Project) - { - return HttpReq.WriteResponse(HttpResponseCode::NotFound); - } - Project->TouchProject(); - - Ref FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ false); - if (!FoundLog) - { - return HttpReq.WriteResponse(HttpResponseCode::NotFound); - } - Project->TouchOplog(OplogId); - - // Parse Request - - IoBuffer Payload = HttpReq.ReadPayload(); - BinaryReader Reader(Payload); - - struct RequestHeader - { - enum - { - kMagic = 0xAAAA'77AC - }; - uint32_t Magic; - uint32_t ChunkCount; - uint32_t Reserved1; - uint32_t Reserved2; - }; - - struct RequestChunkEntry - { - Oid ChunkId; - uint32_t CorrelationId; - uint64_t Offset; - uint64_t RequestBytes; - }; - - if (Payload.Size() <= sizeof(RequestHeader)) - { - m_ProjectStats.BadRequestCount++; - HttpReq.WriteResponse(HttpResponseCode::BadRequest); - } - - RequestHeader RequestHdr; - Reader.Read(&RequestHdr, sizeof RequestHdr); - - if (RequestHdr.Magic != RequestHeader::kMagic) - { - m_ProjectStats.BadRequestCount++; - HttpReq.WriteResponse(HttpResponseCode::BadRequest); - } - - std::vector RequestedChunks; - RequestedChunks.resize(RequestHdr.ChunkCount); - Reader.Read(RequestedChunks.data(), sizeof(RequestChunkEntry) * RequestHdr.ChunkCount); - - // Make Response - - struct ResponseHeader - { - uint32_t Magic = 0xbada'b00f; - uint32_t ChunkCount; - uint32_t Reserved1 = 0; - uint32_t Reserved2 = 0; - }; - - struct ResponseChunkEntry - { - uint32_t CorrelationId; - uint32_t Flags = 0; - uint64_t ChunkSize; - }; - - std::vector OutBlobs; - OutBlobs.emplace_back(sizeof(ResponseHeader) + RequestHdr.ChunkCount * sizeof(ResponseChunkEntry)); - for (uint32_t ChunkIndex = 0; ChunkIndex < RequestHdr.ChunkCount; ++ChunkIndex) - { - const RequestChunkEntry& RequestedChunk = RequestedChunks[ChunkIndex]; - IoBuffer FoundChunk = FoundLog->FindChunk(Project->RootDir, RequestedChunk.ChunkId, nullptr); - if (FoundChunk) - { - if (RequestedChunk.Offset > 0 || RequestedChunk.RequestBytes < uint64_t(-1)) - { - uint64_t Offset = RequestedChunk.Offset; - if (Offset > FoundChunk.Size()) - { - Offset = FoundChunk.Size(); - } - uint64_t Size = RequestedChunk.RequestBytes; - if ((Offset + Size) > FoundChunk.Size()) - { - Size = FoundChunk.Size() - Offset; - } - FoundChunk = IoBuffer(FoundChunk, Offset, Size); - } - } - OutBlobs.emplace_back(std::move(FoundChunk)); - } - uint8_t* ResponsePtr = reinterpret_cast(OutBlobs[0].MutableData()); - ResponseHeader ResponseHdr; - ResponseHdr.ChunkCount = RequestHdr.ChunkCount; - memcpy(ResponsePtr, &ResponseHdr, sizeof(ResponseHdr)); - ResponsePtr += sizeof(ResponseHdr); - for (uint32_t ChunkIndex = 0; ChunkIndex < RequestHdr.ChunkCount; ++ChunkIndex) - { - const RequestChunkEntry& RequestedChunk = RequestedChunks[ChunkIndex]; - const IoBuffer& FoundChunk(OutBlobs[ChunkIndex + 1]); - ResponseChunkEntry ResponseChunk; - ResponseChunk.CorrelationId = RequestedChunk.CorrelationId; - if (FoundChunk) - { - ResponseChunk.ChunkSize = FoundChunk.Size(); - m_ProjectStats.ChunkHitCount++; - } - else - { - ResponseChunk.ChunkSize = uint64_t(-1); - m_ProjectStats.ChunkMissCount++; - } - memcpy(ResponsePtr, &ResponseChunk, sizeof(ResponseChunk)); - ResponsePtr += sizeof(ResponseChunk); - } - std::erase_if(OutBlobs, [](IoBuffer Buffer) -> bool { return !Buffer; }); - return HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, OutBlobs); -} - -void -HttpProjectService::HandleFilesRequest(HttpRouterRequest& Req) -{ - ZEN_TRACE_CPU("ProjectService::Files"); - - using namespace std::literals; - - HttpServerRequest& HttpReq = Req.ServerRequest(); - - // File manifest fetch, returns the client file list - - const auto& ProjectId = Req.GetCapture(1); - const auto& OplogId = Req.GetCapture(2); - - HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); - - std::unordered_set WantedFieldNames; - if (auto FieldFilter = HttpServerRequest::Decode(Params.GetValue("fieldnames")); !FieldFilter.empty()) - { - if (FieldFilter != "*") // Get all - empty FieldFilter equal getting all fields - { - ForEachStrTok(FieldFilter, ',', [&](std::string_view FieldName) { - WantedFieldNames.insert(std::string(FieldName)); - return true; - }); - } - } - else - { - const bool FilterClient = Params.GetValue("filter"sv) == "client"sv; - WantedFieldNames.insert("id"); - WantedFieldNames.insert("clientpath"); - if (!FilterClient) - { - WantedFieldNames.insert("serverpath"); - } - } - - Ref Project = m_ProjectStore->OpenProject(ProjectId); - if (!Project) - { - return HttpReq.WriteResponse(HttpResponseCode::NotFound, - HttpContentType::kText, - fmt::format("Project files request for unknown project '{}'", ProjectId)); - } - Project->TouchProject(); - - Ref FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ true); - if (!FoundLog) - { - return HttpReq.WriteResponse(HttpResponseCode::NotFound, - HttpContentType::kText, - fmt::format("Project files for unknown oplog '{}/{}'", ProjectId, OplogId)); - } - Project->TouchOplog(OplogId); - - CbObject ResponsePayload = ProjectStore::GetProjectFiles(Log(), *Project, *FoundLog, WantedFieldNames); - - if (HttpReq.AcceptContentType() == HttpContentType::kCompressedBinary) - { - CompositeBuffer Payload = CompressedBuffer::Compress(ResponsePayload.GetBuffer()).GetCompressed(); - return HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kCompressedBinary, Payload); - } - else - { - return HttpReq.WriteResponse(HttpResponseCode::OK, ResponsePayload); - } -} - -void -HttpProjectService::HandleChunkInfosRequest(HttpRouterRequest& Req) -{ - ZEN_TRACE_CPU("ProjectService::ChunkInfos"); - - HttpServerRequest& HttpReq = Req.ServerRequest(); - - const auto& ProjectId = Req.GetCapture(1); - const auto& OplogId = Req.GetCapture(2); - - HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); - - std::unordered_set WantedFieldNames; - if (auto FieldFilter = HttpServerRequest::Decode(Params.GetValue("fieldnames")); !FieldFilter.empty()) - { - if (FieldFilter != "*") // Get all - empty FieldFilter equal getting all fields - { - ForEachStrTok(FieldFilter, ',', [&](std::string_view FieldName) { - WantedFieldNames.insert(std::string(FieldName)); - return true; - }); - } - } - else - { - WantedFieldNames.insert("id"); - WantedFieldNames.insert("rawhash"); - WantedFieldNames.insert("rawsize"); - } - - Ref Project = m_ProjectStore->OpenProject(ProjectId); - if (!Project) - { - return HttpReq.WriteResponse(HttpResponseCode::NotFound, - HttpContentType::kText, - fmt::format("Chunk infos request for unknown project '{}'", ProjectId)); - } - Project->TouchProject(); - - Ref FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ true); - if (!FoundLog) - { - return HttpReq.WriteResponse(HttpResponseCode::NotFound, - HttpContentType::kText, - fmt::format("Chunk infos for unknown oplog '{}/{}'", ProjectId, OplogId)); - } - Project->TouchOplog(OplogId); - - CbObject ResponsePayload = ProjectStore::GetProjectChunkInfos(Log(), *Project, *FoundLog, WantedFieldNames); - if (HttpReq.AcceptContentType() == HttpContentType::kCompressedBinary) - { - CompositeBuffer Payload = CompressedBuffer::Compress(ResponsePayload.GetBuffer()).GetCompressed(); - return HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kCompressedBinary, Payload); - } - else - { - return HttpReq.WriteResponse(HttpResponseCode::OK, ResponsePayload); - } -} - -void -HttpProjectService::HandleChunkInfoRequest(HttpRouterRequest& Req) -{ - ZEN_TRACE_CPU("ProjectService::ChunkInfo"); - - HttpServerRequest& HttpReq = Req.ServerRequest(); - - const auto& ProjectId = Req.GetCapture(1); - const auto& OplogId = Req.GetCapture(2); - const auto& ChunkId = Req.GetCapture(3); - - Ref Project = m_ProjectStore->OpenProject(ProjectId); - if (!Project) - { - return HttpReq.WriteResponse(HttpResponseCode::NotFound, - HttpContentType::kText, - fmt::format("Chunk info request for unknown project '{}'", ProjectId)); - } - Project->TouchProject(); - - Ref FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ true); - if (!FoundLog) - { - return HttpReq.WriteResponse(HttpResponseCode::NotFound, - HttpContentType::kText, - fmt::format("Chunk info for unknown oplog '{}/{}'", ProjectId, OplogId)); - } - Project->TouchOplog(OplogId); - - if (ChunkId.size() != 2 * sizeof(Oid::OidBits)) - { - m_ProjectStats.BadRequestCount++; - return HttpReq.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Chunk info request for invalid chunk id '{}/{}'/'{}'", ProjectId, OplogId, ChunkId)); - } - - const Oid Obj = Oid::FromHexString(ChunkId); - - CbObject ResponsePayload = ProjectStore::GetChunkInfo(Log(), *Project, *FoundLog, Obj); - if (ResponsePayload) - { - m_ProjectStats.ChunkHitCount++; - return HttpReq.WriteResponse(HttpResponseCode::OK, ResponsePayload); - } - else - { - m_ProjectStats.ChunkMissCount++; - ZEN_DEBUG("chunk - '{}/{}/{}' MISSING", ProjectId, OplogId, ChunkId); - return HttpReq.WriteResponse(HttpResponseCode::NotFound, - HttpContentType::kText, - fmt::format("Chunk info for unknown chunk '{}/{}/{}'", ProjectId, OplogId, ChunkId)); - } -} - -void -HttpProjectService::HandleChunkByIdRequest(HttpRouterRequest& Req) -{ - ZEN_TRACE_CPU("ProjectService::ChunkById"); - - HttpServerRequest& HttpReq = Req.ServerRequest(); - - const auto& ProjectId = Req.GetCapture(1); - const auto& OplogId = Req.GetCapture(2); - const auto& ChunkId = Req.GetCapture(3); - - uint64_t Offset = 0; - uint64_t Size = ~(0ull); - - auto QueryParms = HttpReq.GetQueryParams(); - - if (auto OffsetParm = QueryParms.GetValue("offset"); OffsetParm.empty() == false) - { - if (auto OffsetVal = ParseInt(OffsetParm)) - { - Offset = OffsetVal.value(); - } - else - { - m_ProjectStats.BadRequestCount++; - return HttpReq.WriteResponse(HttpResponseCode::BadRequest); - } - } - - if (auto SizeParm = QueryParms.GetValue("size"); SizeParm.empty() == false) - { - if (auto SizeVal = ParseInt(SizeParm)) - { - Size = SizeVal.value(); - } - else - { - m_ProjectStats.BadRequestCount++; - return HttpReq.WriteResponse(HttpResponseCode::BadRequest); - } - } - - Ref Project = m_ProjectStore->OpenProject(ProjectId); - if (!Project) - { - return HttpReq.WriteResponse(HttpResponseCode::NotFound, - HttpContentType::kText, - fmt::format("Chunk request for unknown project '{}'", ProjectId)); - } - Project->TouchProject(); - - Ref FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ false); - if (!FoundLog) - { - return HttpReq.WriteResponse(HttpResponseCode::NotFound, - HttpContentType::kText, - fmt::format("Chunk request for unknown oplog '{}/{}'", ProjectId, OplogId)); - } - Project->TouchOplog(OplogId); - - if (ChunkId.size() != 2 * sizeof(Oid::OidBits)) - { - m_ProjectStats.BadRequestCount++; - return HttpReq.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Chunk request for invalid chunk id '{}/{}/{}'", ProjectId, OplogId, ChunkId)); - } - - const Oid Obj = Oid::FromHexString(ChunkId); - - HttpContentType AcceptType = HttpReq.AcceptContentType(); - - ProjectStore::GetChunkRangeResult Result = - ProjectStore::GetChunkRange(Log(), *Project, *FoundLog, Obj, Offset, Size, AcceptType, /*OptionalInOutModificationTag*/ nullptr); - - switch (Result.Error) - { - case ProjectStore::GetChunkRangeResult::EError::Ok: - m_ProjectStats.ChunkHitCount++; - ZEN_DEBUG("chunk - '{}/{}/{}' '{}'", ProjectId, OplogId, ChunkId, ToString(Result.ContentType)); - return HttpReq.WriteResponse(HttpResponseCode::OK, Result.ContentType, Result.Chunk); - case ProjectStore::GetChunkRangeResult::EError::NotFound: - m_ProjectStats.ChunkMissCount++; - ZEN_DEBUG("chunk - '{}/{}/{}' MISSING", ProjectId, OplogId, ChunkId); - return HttpReq.WriteResponse(HttpResponseCode::NotFound, Result.ContentType, Result.Chunk); - case ProjectStore::GetChunkRangeResult::EError::MalformedContent: - return HttpReq.WriteResponse( - HttpResponseCode::NotFound, - HttpContentType::kText, - fmt::format("Get chunk {}/{}/{} failed. Reason: {}", ProjectId, OplogId, ChunkId, Result.ErrorDescription)); - case ProjectStore::GetChunkRangeResult::EError::OutOfRange: - m_ProjectStats.ChunkMissCount++; - ZEN_DEBUG("chunk - '{}/{}/{}' OUT OF RANGE", ProjectId, OplogId, ChunkId); - return HttpReq.WriteResponse( - HttpResponseCode::NotFound, - HttpContentType::kText, - fmt::format("Get chunk {}/{}/{} failed. Reason: {}", ProjectId, OplogId, ChunkId, Result.ErrorDescription)); - default: - ZEN_ASSERT(false); - break; - } -} - -void -HttpProjectService::HandleChunkByCidRequest(HttpRouterRequest& Req) -{ - ZEN_TRACE_CPU("ProjectService::ChunkByCid"); - - HttpServerRequest& HttpReq = Req.ServerRequest(); - - const auto& ProjectId = Req.GetCapture(1); - const auto& OplogId = Req.GetCapture(2); - const auto& Cid = Req.GetCapture(3); - HttpContentType AcceptType = HttpReq.AcceptContentType(); - HttpContentType RequestType = HttpReq.RequestContentType(); - - Ref Project = m_ProjectStore->OpenProject(ProjectId); - if (!Project) - { - return HttpReq.WriteResponse(HttpResponseCode::NotFound, - HttpContentType::kText, - fmt::format("Chunk request for unknown project '{}'", ProjectId)); - } - Project->TouchProject(); - - Ref FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ false); - if (!FoundLog) - { - return HttpReq.WriteResponse(HttpResponseCode::NotFound, - HttpContentType::kText, - fmt::format("Chunk request for unknown oplog '{}/{}'", ProjectId, OplogId)); - } - Project->TouchOplog(OplogId); - - if (Cid.length() != IoHash::StringLength) - { - m_ProjectStats.BadRequestCount++; - return HttpReq.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Chunk request for invalid chunk id '{}/{}/{}'", ProjectId, OplogId, Cid)); - } - - const IoHash Hash = IoHash::FromHexString(Cid); - - switch (HttpReq.RequestVerb()) - { - case HttpVerb::kGet: - { - IoBuffer Value = m_ProjectStore->GetChunk(*Project, *FoundLog, Hash); - if (Value) - { - if (AcceptType == ZenContentType::kUnknownContentType || AcceptType == ZenContentType::kBinary || - AcceptType == ZenContentType::kJSON || AcceptType == ZenContentType::kYAML || - AcceptType == ZenContentType::kCbObject) - { - CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(Value)); - IoBuffer DecompressedBuffer = Compressed.Decompress().AsIoBuffer(); - - if (DecompressedBuffer) - { - if (AcceptType == ZenContentType::kJSON || AcceptType == ZenContentType::kYAML || - AcceptType == ZenContentType::kCbObject) - { - CbValidateError CbErr = ValidateCompactBinary(DecompressedBuffer.GetView(), CbValidateMode::Default); - if (!!CbErr) - { - m_ProjectStats.BadRequestCount++; - ZEN_DEBUG( - "chunk - '{}/{}/{}' WRONGTYPE. Reason: `Requested {} format, but could not convert to object`", - ProjectId, - OplogId, - Cid, - ToString(AcceptType)); - return HttpReq.WriteResponse( - HttpResponseCode::NotAcceptable, - HttpContentType::kText, - fmt::format("Content format not supported, requested {} format, but could not convert to object", - ToString(AcceptType))); - } - - m_ProjectStats.ChunkHitCount++; - CbObject ContainerObject = LoadCompactBinaryObject(DecompressedBuffer); - return HttpReq.WriteResponse(HttpResponseCode::OK, ContainerObject); - } - else - { - Value = DecompressedBuffer; - Value.SetContentType(ZenContentType::kBinary); - } - } - else - { - m_ProjectStats.BadRequestCount++; - ZEN_DEBUG("chunk - '{}/{}/{}' WRONGTYPE. Reason: `Requested {} format, but could not decompress stored data`", - ProjectId, - OplogId, - Cid, - ToString(AcceptType)); - return HttpReq.WriteResponse( - HttpResponseCode::NotAcceptable, - HttpContentType::kText, - fmt::format("Content format not supported, requested {} format, but could not decompress stored data", - ToString(AcceptType))); - } - } - m_ProjectStats.ChunkHitCount++; - return HttpReq.WriteResponse(HttpResponseCode::OK, Value.GetContentType(), Value); - } - else - { - m_ProjectStats.ChunkMissCount++; - ZEN_DEBUG("chunk - '{}/{}/{}' MISSING", ProjectId, OplogId, Cid); - return HttpReq.WriteResponse(HttpResponseCode::NotFound); - } - } - case HttpVerb::kPost: - { - if (!m_ProjectStore->AreDiskWritesAllowed()) - { - return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); - } - if (RequestType != HttpContentType::kCompressedBinary) - { - m_ProjectStats.BadRequestCount++; - return HttpReq.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Chunk request for chunk id '{}/{}'/'{}' as unexpected content type: '{}'", - ProjectId, - OplogId, - Cid, - ToString(RequestType))); - } - IoBuffer Payload = HttpReq.ReadPayload(); - Payload.SetContentType(RequestType); - bool IsNew = m_ProjectStore->PutChunk(*Project, *FoundLog, Hash, std::move(Payload)); - - m_ProjectStats.ChunkWriteCount++; - return HttpReq.WriteResponse(IsNew ? HttpResponseCode::Created : HttpResponseCode::OK); - } - break; - } -} - -void -HttpProjectService::HandleOplogOpPrepRequest(HttpRouterRequest& Req) -{ - ZEN_TRACE_CPU("ProjectService::OplogOpPrep"); - - using namespace std::literals; - - HttpServerRequest& HttpReq = Req.ServerRequest(); - - const auto& ProjectId = Req.GetCapture(1); - const auto& OplogId = Req.GetCapture(2); - - Ref Project = m_ProjectStore->OpenProject(ProjectId); - if (!Project) - { - return HttpReq.WriteResponse(HttpResponseCode::NotFound); - } - Project->TouchProject(); - - Ref FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ false); - if (!FoundLog) - { - return HttpReq.WriteResponse(HttpResponseCode::NotFound); - } - Project->TouchOplog(OplogId); - - // This operation takes a list of referenced hashes and decides which - // chunks are not present on this server. This list is then returned in - // the "need" list in the response - - CbValidateError ValidateResult; - if (CbObject RequestObject = ValidateAndReadCompactBinaryObject(HttpReq.ReadPayload(), ValidateResult); - ValidateResult == CbValidateError::None) - { - std::vector NeedList; - - { - eastl::fixed_vector ChunkList; - CbArrayView HaveList = RequestObject["have"sv].AsArrayView(); - ChunkList.reserve(HaveList.Num()); - for (auto& Entry : HaveList) - { - ChunkList.push_back(Entry.AsHash()); - } - - NeedList = FoundLog->CheckPendingChunkReferences(std::span(begin(ChunkList), end(ChunkList)), std::chrono::minutes(2)); - } - - CbObjectWriter Cbo(1 + 1 + 5 + NeedList.size() * (1 + sizeof(IoHash::Hash)) + 1); - Cbo.BeginArray("need"); - { - for (const IoHash& Hash : NeedList) - { - ZEN_DEBUG("prep - NEED: {}", Hash); - Cbo << Hash; - } - } - Cbo.EndArray(); - CbObject Response = Cbo.Save(); - - return HttpReq.WriteResponse(HttpResponseCode::OK, Response); - } - else - { - return HttpReq.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Invalid compact binary format: '{}'", ToString(ValidateResult))); - } -} - -void -HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req) -{ - ZEN_TRACE_CPU("ProjectService::OplogOpNew"); - - using namespace std::literals; - - HttpServerRequest& HttpReq = Req.ServerRequest(); - - if (!m_ProjectStore->AreDiskWritesAllowed()) - { - return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); - } - - const auto& ProjectId = Req.GetCapture(1); - const auto& OplogId = Req.GetCapture(2); - - HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); - - bool IsUsingSalt = false; - IoHash SaltHash = IoHash::Zero; - - if (std::string_view SaltParam = Params.GetValue("salt"sv); SaltParam.empty() == false) - { - const uint32_t Salt = std::stoi(std::string(SaltParam)); - SaltHash = IoHash::HashBuffer(&Salt, sizeof Salt); - IsUsingSalt = true; - } - - Ref Project = m_ProjectStore->OpenProject(ProjectId); - if (!Project) - { - return HttpReq.WriteResponse(HttpResponseCode::NotFound); - } - Project->TouchProject(); - - Ref FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ false); - if (!FoundLog) - { - return HttpReq.WriteResponse(HttpResponseCode::NotFound); - } - Project->TouchOplog(OplogId); - - ProjectStore::Oplog& Oplog = *FoundLog; - - IoBuffer Payload = HttpReq.ReadPayload(); - - // This will attempt to open files which may not exist for the case where - // the prep step rejected the chunk. This should be fixed since there's - // a performance cost associated with any file system activity - - bool IsValid = true; - std::vector MissingChunks; - - CbPackage::AttachmentResolver Resolver = [&](const IoHash& Hash) -> SharedBuffer { - if (m_CidStore.ContainsChunk(Hash)) - { - // Return null attachment as we already have it, no point in reading it and storing it again - return {}; - } - - IoHash AttachmentId; - if (IsUsingSalt) - { - IoHash AttachmentSpec[]{SaltHash, Hash}; - AttachmentId = IoHash::HashBuffer(MakeMemoryView(AttachmentSpec)); - } - else - { - AttachmentId = Hash; - } - - std::filesystem::path AttachmentPath = Oplog.TempPath() / AttachmentId.ToHexString(); - if (IoBuffer Data = IoBufferBuilder::MakeFromTemporaryFile(AttachmentPath)) - { - Data.SetDeleteOnClose(true); - return SharedBuffer(std::move(Data)); - } - else - { - IsValid = false; - MissingChunks.push_back(Hash); - - return {}; - } - }; - - CbPackage Package; - - if (!legacy::TryLoadCbPackage(Package, Payload, &UniqueBuffer::Alloc, &Resolver)) - { - CbValidateError ValidateResult; - if (CbObject Core = ValidateAndReadCompactBinaryObject(IoBuffer(Payload), ValidateResult); - ValidateResult == CbValidateError::None && Core) - { - Package.SetObject(Core); - } - else - { - std::filesystem::path BadPackagePath = - Oplog.TempPath() / "bad_packages"sv / fmt::format("session{}_request{}"sv, HttpReq.SessionId(), HttpReq.RequestId()); - - ZEN_WARN("Received malformed package ('{}')! Saving payload to '{}'", ToString(ValidateResult), BadPackagePath); - - WriteFile(BadPackagePath, Payload); - - m_ProjectStats.BadRequestCount++; - return HttpReq.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - u8"request body must be a compact binary object or package in legacy format"); - } - } - - m_ProjectStats.ChunkMissCount += MissingChunks.size(); - - if (!IsValid) - { - ExtendableStringBuilder<256> ResponseText; - ResponseText.Append("Missing chunk references: "); - - bool IsFirst = true; - for (const auto& Hash : MissingChunks) - { - if (IsFirst) - { - IsFirst = false; - } - else - { - ResponseText.Append(", "); - } - Hash.ToHexString(ResponseText); - } - - return HttpReq.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, ResponseText); - } - - CbObject Core = Package.GetObject(); - - if (!Core["key"sv]) - { - m_ProjectStats.BadRequestCount++; - return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "No oplog entry key specified"); - } - - eastl::fixed_vector ReferencedChunks; - Core.IterateAttachments([&ReferencedChunks](CbFieldView View) { ReferencedChunks.push_back(View.AsAttachment()); }); - - // Write core to oplog - - size_t AttachmentCount = Package.GetAttachments().size(); - const ProjectStore::LogSequenceNumber OpLsn = Oplog.AppendNewOplogEntry(Package); - if (!OpLsn) - { - m_ProjectStats.BadRequestCount++; - return HttpReq.WriteResponse(HttpResponseCode::BadRequest); - } - m_ProjectStats.ChunkWriteCount += AttachmentCount; - - // Once we stored the op, we no longer need to retain any chunks this op references - if (!ReferencedChunks.empty()) - { - FoundLog->RemovePendingChunkReferences(std::span(begin(ReferencedChunks), end(ReferencedChunks))); - } - - m_ProjectStats.OpWriteCount++; - ZEN_DEBUG("'{}/{}' op #{} ({}) - '{}'", ProjectId, OplogId, OpLsn.Number, NiceBytes(Payload.Size()), Core["key"sv].AsString()); - HttpReq.WriteResponse(HttpResponseCode::Created); -} - -void -HttpProjectService::HandleOplogValidateRequest(HttpRouterRequest& Req) -{ - ZEN_TRACE_CPU("ProjectService::OplogOpValidate"); - - using namespace std::literals; - - HttpServerRequest& HttpReq = Req.ServerRequest(); - - if (!m_ProjectStore->AreDiskWritesAllowed()) - { - return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); - } - - const auto& ProjectId = Req.GetCapture(1); - const auto& OplogId = Req.GetCapture(2); - - Ref Project = m_ProjectStore->OpenProject(ProjectId); - if (!Project) - { - return HttpReq.WriteResponse(HttpResponseCode::NotFound, ZenContentType::kText, fmt::format("Project '{}' not found", ProjectId)); - } - Project->TouchProject(); - - Ref FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ true); - if (!FoundLog) - { - return HttpReq.WriteResponse(HttpResponseCode::NotFound, - ZenContentType::kText, - fmt::format("Oplog '{}' not found in project '{}'", OplogId, ProjectId)); - } - Project->TouchOplog(OplogId); - - ProjectStore::Oplog& Oplog = *FoundLog; - - std::atomic_bool CancelFlag = false; - ProjectStore::Oplog::ValidationResult Result = Oplog.Validate(Project->RootDir, CancelFlag, &GetSmallWorkerPool(EWorkloadType::Burst)); - tsl::robin_map KeyNameLookup; - KeyNameLookup.reserve(Result.OpKeys.size()); - for (const auto& It : Result.OpKeys) - { - KeyNameLookup.insert_or_assign(It.first, It.second); - } - CbObjectWriter Writer; - Writer << "HasMissingData" << !Result.IsEmpty(); - Writer << "OpCount" << Result.OpCount; - Writer << "LSNLow" << Result.LSNLow.Number; - Writer << "LSNHigh" << Result.LSNHigh.Number; - if (!Result.MissingFiles.empty()) - { - Writer.BeginArray("MissingFiles"); - for (const auto& MissingFile : Result.MissingFiles) - { - Writer.BeginObject(); - { - Writer << "Key" << MissingFile.first; - Writer << "KeyName" << KeyNameLookup[MissingFile.first]; - Writer << "Id" << MissingFile.second.Id; - Writer << "Hash" << MissingFile.second.Hash; - Writer << "ServerPath" << MissingFile.second.ServerPath; - Writer << "ClientPath" << MissingFile.second.ClientPath; - } - Writer.EndObject(); - } - Writer.EndArray(); - } - if (!Result.MissingChunks.empty()) - { - Writer.BeginArray("MissingChunks"); - for (const auto& MissingChunk : Result.MissingChunks) - { - Writer.BeginObject(); - { - Writer << "Key" << MissingChunk.first; - Writer << "KeyName" << KeyNameLookup[MissingChunk.first]; - Writer << "Id" << MissingChunk.second.Id; - Writer << "Hash" << MissingChunk.second.Hash; - } - Writer.EndObject(); - } - Writer.EndArray(); - } - if (!Result.MissingMetas.empty()) - { - Writer.BeginArray("MissingMetas"); - for (const auto& MissingMeta : Result.MissingMetas) - { - Writer.BeginObject(); - { - Writer << "Key" << MissingMeta.first; - Writer << "KeyName" << KeyNameLookup[MissingMeta.first]; - Writer << "Id" << MissingMeta.second.Id; - Writer << "Hash" << MissingMeta.second.Hash; - } - Writer.EndObject(); - } - Writer.EndArray(); - } - if (!Result.MissingAttachments.empty()) - { - Writer.BeginArray("MissingAttachments"); - for (const auto& MissingMeta : Result.MissingAttachments) - { - Writer.BeginObject(); - { - Writer << "Key" << MissingMeta.first; - Writer << "KeyName" << KeyNameLookup[MissingMeta.first]; - Writer << "Hash" << MissingMeta.second; - } - Writer.EndObject(); - } - Writer.EndArray(); - } - CbObject Response = Writer.Save(); - HttpReq.WriteResponse(HttpResponseCode::OK, Response); -} - -void -HttpProjectService::HandleOpLogOpRequest(HttpRouterRequest& Req) -{ - ZEN_TRACE_CPU("ProjectService::OplogOp"); - - HttpServerRequest& HttpReq = Req.ServerRequest(); - - const std::string_view ProjectId = Req.GetCapture(1); - const std::string_view OplogId = Req.GetCapture(2); - const std::string_view OpIdString = Req.GetCapture(3); - - Ref Project = m_ProjectStore->OpenProject(ProjectId); - if (!Project) - { - return HttpReq.WriteResponse(HttpResponseCode::NotFound); - } - Project->TouchProject(); - - Ref FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ false); - if (!FoundLog) - { - return HttpReq.WriteResponse(HttpResponseCode::NotFound); - } - Project->TouchOplog(OplogId); - - ProjectStore::Oplog& Oplog = *FoundLog; - - if (const std::optional OpId = ParseInt(OpIdString)) - { - if (std::optional MaybeOp = Oplog.GetOpByIndex(ProjectStore::LogSequenceNumber(OpId.value()))) - { - CbObject& Op = MaybeOp.value(); - if (HttpReq.AcceptContentType() == ZenContentType::kCbPackage) - { - CbPackage Package; - Package.SetObject(Op); - - Op.IterateAttachments([&](CbFieldView FieldView) { - const IoHash AttachmentHash = FieldView.AsAttachment(); - IoBuffer Payload = m_CidStore.FindChunkByCid(AttachmentHash); - if (Payload) - { - switch (Payload.GetContentType()) - { - case ZenContentType::kCbObject: - { - CbValidateError ValidateResult; - if (CbObject Object = ValidateAndReadCompactBinaryObject(std::move(Payload), ValidateResult); - ValidateResult == CbValidateError::None && Object) - { - Package.AddAttachment(CbAttachment(Object)); - } - else - { - // Error - malformed object - ZEN_WARN("malformed object returned for {} ('{}')", AttachmentHash, ToString(ValidateResult)); - } - } - break; - - case ZenContentType::kCompressedBinary: - if (CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(Payload))) - { - Package.AddAttachment(CbAttachment(Compressed, AttachmentHash)); - } - else - { - // Error - not compressed! - - ZEN_WARN("invalid compressed binary returned for {}", AttachmentHash); - } - break; - - default: - Package.AddAttachment(CbAttachment(SharedBuffer(Payload))); - break; - } - } - }); - m_ProjectStats.OpHitCount++; - return HttpReq.WriteResponse(HttpResponseCode::Accepted, Package); - } - else - { - // Client cannot accept a package, so we only send the core object - m_ProjectStats.OpHitCount++; - return HttpReq.WriteResponse(HttpResponseCode::Accepted, Op); - } - } - } - m_ProjectStats.OpMissCount++; - return HttpReq.WriteResponse(HttpResponseCode::NotFound); -} - -void -HttpProjectService::HandleOpLogRequest(HttpRouterRequest& Req) -{ - ZEN_TRACE_CPU("ProjectService::Oplog"); - - HttpServerRequest& HttpReq = Req.ServerRequest(); - - using namespace std::literals; - - const auto& ProjectId = Req.GetCapture(1); - const auto& OplogId = Req.GetCapture(2); - - Ref Project = m_ProjectStore->OpenProject(ProjectId); - - if (!Project) - { - return HttpReq.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, fmt::format("project {} not found", ProjectId)); - } - Project->TouchProject(); - - switch (HttpReq.RequestVerb()) - { - case HttpVerb::kGet: - { - Ref OplogIt = Project->ReadOplog(OplogId); - if (!OplogIt) - { - return HttpReq.WriteResponse(HttpResponseCode::NotFound, - HttpContentType::kText, - fmt::format("oplog {} not found in project {}", OplogId, ProjectId)); - } - - ProjectStore::Oplog& Log = *OplogIt; - - CbObjectWriter Cb; - Cb << "id"sv << Log.OplogId() << "project"sv << Project->Identifier << "tempdir"sv << Log.TempPath().c_str() - << "markerpath"sv << Log.MarkerPath().c_str() << "totalsize"sv << Log.TotalSize() << "opcount" << Log.OplogCount() - << "expired"sv << Project->IsExpired(GcClock::TimePoint::min(), Log); - HttpReq.WriteResponse(HttpResponseCode::OK, Cb.Save()); - - m_ProjectStats.OpLogReadCount++; - } - break; - - case HttpVerb::kPost: - { - if (!m_ProjectStore->AreDiskWritesAllowed()) - { - return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); - } - std::filesystem::path OplogMarkerPath; - if (CbObject Params = HttpReq.ReadPayloadObject()) - { - OplogMarkerPath = Params["gcpath"sv].AsString(); - } - - Ref OplogIt = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ true); - if (!OplogIt) - { - if (!Project->NewOplog(OplogId, OplogMarkerPath)) - { - // TODO: indicate why the operation failed! - return HttpReq.WriteResponse(HttpResponseCode::InternalServerError); - } - Project->TouchOplog(OplogId); - - m_ProjectStats.OpLogWriteCount++; - ZEN_INFO("established oplog '{}/{}', gc marker file at '{}'", ProjectId, OplogId, OplogMarkerPath); - - return HttpReq.WriteResponse(HttpResponseCode::Created); - } - - // I guess this should ultimately be used to execute RPCs but for now, it - // does absolutely nothing - - m_ProjectStats.BadRequestCount++; - return HttpReq.WriteResponse(HttpResponseCode::BadRequest); - } - break; - - case HttpVerb::kPut: - { - if (!m_ProjectStore->AreDiskWritesAllowed()) - { - return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); - } - - std::filesystem::path OplogMarkerPath; - if (CbObject Params = HttpReq.ReadPayloadObject()) - { - OplogMarkerPath = Params["gcpath"sv].AsString(); - } - - Ref FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ true); - if (!FoundLog) - { - if (!Project->NewOplog(OplogId, OplogMarkerPath)) - { - // TODO: indicate why the operation failed! - return HttpReq.WriteResponse(HttpResponseCode::InternalServerError); - } - Project->TouchOplog(OplogId); - - m_ProjectStats.OpLogWriteCount++; - ZEN_INFO("established oplog '{}/{}', gc marker file at '{}'", ProjectId, OplogId, OplogMarkerPath); - - return HttpReq.WriteResponse(HttpResponseCode::Created); - } - Project->TouchOplog(OplogId); - - FoundLog->Update(OplogMarkerPath); - - m_ProjectStats.OpLogWriteCount++; - ZEN_INFO("updated oplog '{}/{}', gc marker file at '{}'", ProjectId, OplogId, OplogMarkerPath); - - return HttpReq.WriteResponse(HttpResponseCode::OK); - } - break; - - case HttpVerb::kDelete: - { - ZEN_INFO("deleting oplog '{}/{}'", ProjectId, OplogId); - - if (Project->DeleteOplog(OplogId)) - { - m_ProjectStats.OpLogDeleteCount++; - return HttpReq.WriteResponse(HttpResponseCode::OK); - } - else - { - return HttpReq.WriteResponse(HttpResponseCode::Locked, - HttpContentType::kText, - fmt::format("oplog {}/{} is in use", ProjectId, OplogId)); - } - } - break; - - default: - break; - } -} - -std::optional -LoadReferencedSet(ProjectStore::Project& Project, ProjectStore::Oplog& Log) -{ - using namespace std::literals; - - Oid ReferencedSetOplogId = OpKeyStringAsOid(OplogReferencedSet::ReferencedSetOplogKey); - std::optional ReferencedSetOp = Log.GetOpByKey(ReferencedSetOplogId); - if (!ReferencedSetOp) - { - return std::optional(); - } - // We expect only a single file in the "files" array; get the chunk for the first file - CbFieldView FileField = *(*ReferencedSetOp)["files"sv].AsArrayView().CreateViewIterator(); - Oid ChunkId = FileField.AsObjectView()["id"sv].AsObjectId(); - if (ChunkId == Oid::Zero) - { - return std::optional(); - } - - return OplogReferencedSet::LoadFromChunk(Log.FindChunk(Project.RootDir, ChunkId, nullptr)); -} - -void -HttpProjectService::HandleOpLogEntriesRequest(HttpRouterRequest& Req) -{ - ZEN_TRACE_CPU("ProjectService::OplogEntries"); - - using namespace std::literals; - - HttpServerRequest& HttpReq = Req.ServerRequest(); - - const auto& ProjectId = Req.GetCapture(1); - const auto& OplogId = Req.GetCapture(2); - - Ref Project = m_ProjectStore->OpenProject(ProjectId); - if (!Project) - { - return HttpReq.WriteResponse(HttpResponseCode::NotFound); - } - Project->TouchProject(); - - Ref FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ true); - if (!FoundLog) - { - return HttpReq.WriteResponse(HttpResponseCode::NotFound); - } - Project->TouchOplog(OplogId); - - CbObjectWriter Response; - - if (FoundLog->OplogCount() > 0) - { - std::unordered_set FieldNamesFilter; - auto FilterObject = [&FieldNamesFilter](CbObjectView& Object) -> CbObject { - CbObject RewrittenOp = RewriteCbObject(Object, [&FieldNamesFilter](CbObjectWriter&, CbFieldView Field) -> bool { - if (FieldNamesFilter.contains(std::string(Field.GetName()))) - { - return false; - } - - return true; - }); - - return RewrittenOp; - }; - - HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); - if (auto FieldFilter = HttpServerRequest::Decode(Params.GetValue("fieldfilter")); !FieldFilter.empty()) - { - ForEachStrTok(FieldFilter, ',', [&](std::string_view FieldName) { - FieldNamesFilter.insert(std::string(FieldName)); - return true; - }); - } - - if (auto OpKey = Params.GetValue("opkey"); !OpKey.empty()) - { - Oid OpKeyId = OpKeyStringAsOid(OpKey); - std::optional Op = FoundLog->GetOpByKey(OpKeyId); - - if (Op.has_value()) - { - if (FieldNamesFilter.empty()) - { - Response << "entry"sv << Op.value(); - } - else - { - Response << "entry"sv << FilterObject(Op.value()); - } - } - else - { - return HttpReq.WriteResponse(HttpResponseCode::NotFound); - } - } - else - { - ProjectStore::Oplog::Paging EntryPaging; - if (std::string_view Param = Params.GetValue("start"); !Param.empty()) - { - if (auto Value = ParseInt(Param)) - { - EntryPaging.Start = *Value; - } - } - if (std::string_view Param = Params.GetValue("count"); !Param.empty()) - { - if (auto Value = ParseInt(Param)) - { - EntryPaging.Count = *Value; - } - } - - std::optional MaybeReferencedSet; - if (auto TrimString = Params.GetValue("trim_by_referencedset"); TrimString == "true") - { - MaybeReferencedSet = LoadReferencedSet(*Project, *FoundLog); - } - Response.BeginArray("entries"sv); - - bool ShouldFilterFields = !FieldNamesFilter.empty(); - - if (MaybeReferencedSet) - { - const OplogReferencedSet& ReferencedSet = MaybeReferencedSet.value(); - FoundLog->IterateOplogWithKey( - [this, &Response, &FilterObject, ShouldFilterFields, &ReferencedSet](ProjectStore::LogSequenceNumber /* LSN */, - const Oid& Key, - CbObjectView Op) { - if (!ReferencedSet.Contains(Key)) - { - if (!OplogReferencedSet::IsNonPackage(Op["key"].AsString())) - { - return; - } - } - - if (ShouldFilterFields) - { - Response << FilterObject(Op); - } - else - { - Response << Op; - } - }, - EntryPaging); - } - else - { - FoundLog->IterateOplog( - [this, &Response, &FilterObject, ShouldFilterFields](CbObjectView Op) { - if (ShouldFilterFields) - { - Response << FilterObject(Op); - } - else - { - Response << Op; - } - }, - EntryPaging); - } - - Response.EndArray(); - } - } - if (HttpReq.AcceptContentType() == HttpContentType::kCompressedBinary) - { - CompositeBuffer Payload = CompressedBuffer::Compress(Response.Save().GetBuffer()).GetCompressed(); - return HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kCompressedBinary, Payload); - } - else - { - return HttpReq.WriteResponse(HttpResponseCode::OK, Response.Save()); - } -} - -void -HttpProjectService::HandleProjectRequest(HttpRouterRequest& Req) -{ - ZEN_TRACE_CPU("ProjectService::Project"); - - using namespace std::literals; - - HttpServerRequest& HttpReq = Req.ServerRequest(); - const std::string_view ProjectId = Req.GetCapture(1); - - switch (HttpReq.RequestVerb()) - { - case HttpVerb::kPost: - { - if (!m_ProjectStore->AreDiskWritesAllowed()) - { - return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); - } - - CbValidateError ValidateResult; - if (CbObject Params = ValidateAndReadCompactBinaryObject(HttpReq.ReadPayload(), ValidateResult); - ValidateResult == CbValidateError::None) - { - std::filesystem::path Root = Params["root"sv].AsU8String(); // Workspace root (i.e `D:/UE5/`) - std::filesystem::path EngineRoot = Params["engine"sv].AsU8String(); // Engine root (i.e `D:/UE5/Engine`) - std::filesystem::path ProjectRoot = - Params["project"sv].AsU8String(); // Project root directory (i.e `D:/UE5/Samples/Games/Lyra`) - std::filesystem::path ProjectFilePath = - Params["projectfile"sv].AsU8String(); // Project file path (i.e `D:/UE5/Samples/Games/Lyra/Lyra.uproject`) - - const std::filesystem::path BasePath = m_ProjectStore->BasePath() / ProjectId; - m_ProjectStore->NewProject(BasePath, ProjectId, Root, EngineRoot, ProjectRoot, ProjectFilePath); - - ZEN_INFO("established project (id: '{}', roots: '{}', '{}', '{}', '{}'{})", - ProjectId, - Root, - EngineRoot, - ProjectRoot, - ProjectFilePath, - ProjectFilePath.empty() ? ", project will not be GCd due to empty project file path" : ""); - - m_ProjectStats.ProjectWriteCount++; - HttpReq.WriteResponse(HttpResponseCode::Created); - } - else - { - HttpReq.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Malformed compact binary object: '{}'", ToString(ValidateResult))); - } - } - break; - - case HttpVerb::kPut: - { - if (!m_ProjectStore->AreDiskWritesAllowed()) - { - return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); - } - CbValidateError ValidateResult; - if (CbObject Params = ValidateAndReadCompactBinaryObject(HttpReq.ReadPayload(), ValidateResult); - ValidateResult == CbValidateError::None) - { - std::filesystem::path Root = Params["root"sv].AsU8String(); // Workspace root (i.e `D:/UE5/`) - std::filesystem::path EngineRoot = Params["engine"sv].AsU8String(); // Engine root (i.e `D:/UE5/Engine`) - std::filesystem::path ProjectRoot = - Params["project"sv].AsU8String(); // Project root directory (i.e `D:/UE5/Samples/Games/Lyra`) - std::filesystem::path ProjectFilePath = - Params["projectfile"sv].AsU8String(); // Project file path (i.e `D:/UE5/Samples/Games/Lyra/Lyra.uproject`) - - if (m_ProjectStore->UpdateProject(ProjectId, Root, EngineRoot, ProjectRoot, ProjectFilePath)) - { - m_ProjectStats.ProjectWriteCount++; - ZEN_INFO("updated project (id: '{}', roots: '{}', '{}', '{}', '{}'{})", - ProjectId, - Root, - EngineRoot, - ProjectRoot, - ProjectFilePath, - ProjectFilePath.empty() ? ", project will not be GCd due to empty project file path" : ""); - - HttpReq.WriteResponse(HttpResponseCode::OK); - } - else - { - const std::filesystem::path BasePath = m_ProjectStore->BasePath() / ProjectId; - m_ProjectStore->NewProject(BasePath, ProjectId, Root, EngineRoot, ProjectRoot, ProjectFilePath); - - m_ProjectStats.ProjectWriteCount++; - ZEN_INFO("established project (id: '{}', roots: '{}', '{}', '{}', '{}'{})", - ProjectId, - Root, - EngineRoot, - ProjectRoot, - ProjectFilePath, - ProjectFilePath.empty() ? ", project will not be GCd due to empty project file path" : ""); - - HttpReq.WriteResponse(HttpResponseCode::Created); - } - } - else - { - HttpReq.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Malformed compact binary object: '{}'", ToString(ValidateResult))); - } - } - break; - - case HttpVerb::kGet: - { - Ref Project = m_ProjectStore->OpenProject(ProjectId); - if (!Project) - { - return HttpReq.WriteResponse(HttpResponseCode::NotFound, - HttpContentType::kText, - fmt::format("project {} not found", ProjectId)); - } - Project->TouchProject(); - - std::vector OpLogs = Project->ScanForOplogs(); - - CbObjectWriter Response; - Response << "id"sv << Project->Identifier; - Response << "root"sv << PathToUtf8(Project->RootDir); - Response << "engine"sv << PathToUtf8(Project->EngineRootDir); - Response << "project"sv << PathToUtf8(Project->ProjectRootDir); - Response << "projectfile"sv << PathToUtf8(Project->ProjectFilePath); - - Response.BeginArray("oplogs"sv); - for (const std::string& OplogId : OpLogs) - { - Response.BeginObject(); - Response << "id"sv << OplogId; - Response.EndObject(); - } - Response.EndArray(); // oplogs - - HttpReq.WriteResponse(HttpResponseCode::OK, Response.Save()); - - m_ProjectStats.ProjectReadCount++; - } - break; - - case HttpVerb::kDelete: - { - Ref Project = m_ProjectStore->OpenProject(ProjectId); - if (!Project) - { - return HttpReq.WriteResponse(HttpResponseCode::NotFound, - HttpContentType::kText, - fmt::format("project {} not found", ProjectId)); - } - - ZEN_INFO("deleting project '{}'", ProjectId); - if (!m_ProjectStore->DeleteProject(ProjectId)) - { - return HttpReq.WriteResponse(HttpResponseCode::Locked, - HttpContentType::kText, - fmt::format("project {} is in use", ProjectId)); - } - - m_ProjectStats.ProjectDeleteCount++; - return HttpReq.WriteResponse(HttpResponseCode::NoContent); - } - break; - - default: - break; - } -} - -void -HttpProjectService::HandleOplogSaveRequest(HttpRouterRequest& Req) -{ - ZEN_TRACE_CPU("ProjectService::OplogSave"); - - HttpServerRequest& HttpReq = Req.ServerRequest(); - - if (!m_ProjectStore->AreDiskWritesAllowed()) - { - return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); - } - - const auto& ProjectId = Req.GetCapture(1); - const auto& OplogId = Req.GetCapture(2); - if (HttpReq.RequestContentType() != HttpContentType::kCbObject) - { - m_ProjectStats.BadRequestCount++; - return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid content type"); - } - IoBuffer Payload = HttpReq.ReadPayload(); - - Ref Project = m_ProjectStore->OpenProject(ProjectId); - if (!Project) - { - return HttpReq.WriteResponse(HttpResponseCode::NotFound, - HttpContentType::kText, - fmt::format("Write oplog request for unknown project '{}'", ProjectId)); - } - Project->TouchProject(); - - Ref Oplog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ false); - if (!Oplog) - { - return HttpReq.WriteResponse(HttpResponseCode::NotFound, - HttpContentType::kText, - fmt::format("Write oplog request for unknown oplog '{}/{}'", ProjectId, OplogId)); - } - Project->TouchOplog(OplogId); - - CbValidateError ValidateResult; - if (CbObject ContainerObject = ValidateAndReadCompactBinaryObject(std::move(Payload), ValidateResult); - ValidateResult == CbValidateError::None && ContainerObject) - { - RwLock AttachmentsLock; - tsl::robin_set Attachments; - - auto HasAttachment = [this](const IoHash& RawHash) { return m_CidStore.ContainsChunk(RawHash); }; - auto OnNeedBlock = [&AttachmentsLock, &Attachments](const IoHash& BlockHash, const std::vector&& ChunkHashes) { - RwLock::ExclusiveLockScope _(AttachmentsLock); - if (BlockHash != IoHash::Zero) - { - Attachments.insert(BlockHash); - } - else - { - Attachments.insert(ChunkHashes.begin(), ChunkHashes.end()); - } - }; - auto OnNeedAttachment = [&AttachmentsLock, &Attachments](const IoHash& RawHash) { - RwLock::ExclusiveLockScope _(AttachmentsLock); - Attachments.insert(RawHash); - }; - - auto OnChunkedAttachment = [](const ChunkedInfo&) {}; - - auto OnReferencedAttachments = [&Oplog](std::span RawHashes) { Oplog->CaptureAddedAttachments(RawHashes); }; - - // Make sure we retain any attachments we download before writing the oplog - Oplog->EnableUpdateCapture(); - auto _ = MakeGuard([&Oplog]() { Oplog->DisableUpdateCapture(); }); - - RemoteProjectStore::Result Result = SaveOplogContainer(*Oplog, - ContainerObject, - OnReferencedAttachments, - HasAttachment, - OnNeedBlock, - OnNeedAttachment, - OnChunkedAttachment, - nullptr); - - if (Result.ErrorCode == 0) - { - if (Attachments.empty()) - { - HttpReq.WriteResponse(HttpResponseCode::OK); - } - else - { - CbObjectWriter Cbo(1 + 1 + 5 + Attachments.size() * (1 + sizeof(IoHash::Hash)) + 1); - Cbo.BeginArray("need"); - { - for (const IoHash& Hash : Attachments) - { - ZEN_DEBUG("Need attachment {}", Hash); - Cbo << Hash; - } - } - Cbo.EndArray(); // "need" - - CbObject ResponsePayload = Cbo.Save(); - return HttpReq.WriteResponse(HttpResponseCode::OK, ResponsePayload); - } - } - else - { - ZEN_DEBUG("Request {}: '{}' failed with {}. Reason: `{}`", - ToString(HttpReq.RequestVerb()), - HttpReq.QueryString(), - Result.ErrorCode, - Result.Reason); - - if (Result.Reason.empty()) - { - return HttpReq.WriteResponse(HttpResponseCode(Result.ErrorCode)); - } - else - { - return HttpReq.WriteResponse(HttpResponseCode(Result.ErrorCode), HttpContentType::kText, Result.Reason); - } - } - } - else - { - m_ProjectStats.BadRequestCount++; - return HttpReq.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Invalid payload: '{}'", ToString(ValidateResult))); - } -} - -void -HttpProjectService::HandleOplogLoadRequest(HttpRouterRequest& Req) -{ - ZEN_TRACE_CPU("ProjectService::OplogLoad"); - - HttpServerRequest& HttpReq = Req.ServerRequest(); - const auto& ProjectId = Req.GetCapture(1); - const auto& OplogId = Req.GetCapture(2); - - const HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); - - if (HttpReq.AcceptContentType() != HttpContentType::kCbObject) - { - m_ProjectStats.BadRequestCount++; - return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid accept content type"); - } - - Ref Project = m_ProjectStore->OpenProject(ProjectId); - if (!Project) - { - return HttpReq.WriteResponse(HttpResponseCode::NotFound, - HttpContentType::kText, - fmt::format("Read oplog request for unknown project '{}'", ProjectId)); - } - Project->TouchProject(); - - Ref Oplog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ true); - if (!Oplog) - { - return HttpReq.WriteResponse(HttpResponseCode::NotFound, - HttpContentType::kText, - fmt::format("Read oplog request for unknown oplog '{}/{}'", ProjectId, OplogId)); - } - Project->TouchOplog(OplogId); - - size_t MaxBlockSize = RemoteStoreOptions::DefaultMaxBlockSize; - if (auto Param = Params.GetValue("maxblocksize"); Param.empty() == false) - { - if (auto Value = ParseInt(Param)) - { - MaxBlockSize = Value.value(); - } - } - size_t MaxChunkEmbedSize = RemoteStoreOptions::DefaultMaxChunkEmbedSize; - if (auto Param = Params.GetValue("maxchunkembedsize"); Param.empty() == false) - { - if (auto Value = ParseInt(Param)) - { - MaxChunkEmbedSize = Value.value(); - } - } - size_t MaxChunksPerBlock = RemoteStoreOptions::DefaultMaxChunksPerBlock; - if (auto Param = Params.GetValue("maxchunksperblock"); Param.empty() == false) - { - if (auto Value = ParseInt(Param)) - { - MaxChunksPerBlock = Value.value(); - } - } - - size_t ChunkFileSizeLimit = RemoteStoreOptions::DefaultChunkFileSizeLimit; - if (auto Param = Params.GetValue("chunkfilesizelimit"); Param.empty() == false) - { - if (auto Value = ParseInt(Param)) - { - ChunkFileSizeLimit = Value.value(); - } - } - - WorkerThreadPool& WorkerPool = GetLargeWorkerPool(EWorkloadType::Background); - - RemoteProjectStore::LoadContainerResult ContainerResult = BuildContainer( - m_CidStore, - *Project, - *Oplog, - WorkerPool, - MaxBlockSize, - MaxChunkEmbedSize, - MaxChunksPerBlock, - ChunkFileSizeLimit, - /* BuildBlocks */ false, - /* IgnoreMissingAttachments */ false, - /* AllowChunking*/ false, - [](CompressedBuffer&&, ChunkBlockDescription&&) {}, - [](const IoHash&, TGetAttachmentBufferFunc&&) {}, - [](std::vector>&&) {}, - /* EmbedLooseFiles*/ false); - - if (ContainerResult.ErrorCode == 0) - { - return HttpReq.WriteResponse(HttpResponseCode::OK, ContainerResult.ContainerObject); - } - else - { - ZEN_DEBUG("Request {}: '{}' failed with {}. Reason: `{}`", - ToString(HttpReq.RequestVerb()), - HttpReq.QueryString(), - ContainerResult.ErrorCode, - ContainerResult.Reason); - - if (ContainerResult.Reason.empty()) - { - return HttpReq.WriteResponse(HttpResponseCode(ContainerResult.ErrorCode)); - } - else - { - return HttpReq.WriteResponse(HttpResponseCode(ContainerResult.ErrorCode), HttpContentType::kText, ContainerResult.Reason); - } - } -} - -void -HttpProjectService::HandleRpcRequest(HttpRouterRequest& Req) -{ - ZEN_TRACE_CPU("ProjectService::Rpc"); - using namespace std::literals; - - HttpServerRequest& HttpReq = Req.ServerRequest(); - - const auto& ProjectId = Req.GetCapture(1); - const auto& OplogId = Req.GetCapture(2); - IoBuffer Payload = HttpReq.ReadPayload(); - - HttpContentType PayloadContentType = HttpReq.RequestContentType(); - CbPackage Package; - CbObject Cb; - switch (PayloadContentType) - { - case HttpContentType::kJSON: - case HttpContentType::kUnknownContentType: - case HttpContentType::kText: - { - std::string JsonText(reinterpret_cast(Payload.GetData()), Payload.GetSize()); - Cb = LoadCompactBinaryFromJson(JsonText).AsObject(); - if (!Cb) - { - m_ProjectStats.BadRequestCount++; - return HttpReq.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - "Content format not supported, expected JSON format"); - } - } - break; - case HttpContentType::kCbObject: - { - CbValidateError ValidateResult; - if (Cb = ValidateAndReadCompactBinaryObject(std::move(Payload), ValidateResult); - ValidateResult != CbValidateError::None || !Cb) - { - m_ProjectStats.BadRequestCount++; - return HttpReq.WriteResponse( - HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Content format not supported, expected compact binary format ('{}')", ToString(ValidateResult))); - } - break; - } - case HttpContentType::kCbPackage: - try - { - Package = ParsePackageMessage(Payload); - Cb = Package.GetObject(); - } - catch (const std::invalid_argument& ex) - { - m_ProjectStats.BadRequestCount++; - return HttpReq.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Failed to parse package request, reason: '{}'", ex.what())); - } - if (!Cb) - { - m_ProjectStats.BadRequestCount++; - return HttpReq.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - "Content format not supported, expected package message format"); - } - break; - default: - m_ProjectStats.BadRequestCount++; - return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid request content type"); - } - - Ref Project = m_ProjectStore->OpenProject(ProjectId); - if (!Project) - { - return HttpReq.WriteResponse(HttpResponseCode::NotFound, - HttpContentType::kText, - fmt::format("Rpc oplog request for unknown project '{}'", ProjectId)); - } - Project->TouchProject(); - - std::string_view Method = Cb["method"sv].AsString(); - - bool VerifyPathOnDisk = Method != "getchunks"sv; - - Ref Oplog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, VerifyPathOnDisk); - if (!Oplog) - { - return HttpReq.WriteResponse(HttpResponseCode::NotFound, - HttpContentType::kText, - fmt::format("Rpc oplog request for unknown oplog '{}/{}'", ProjectId, OplogId)); - } - Project->TouchOplog(OplogId); - - uint32_t MethodHash = HashStringDjb2(Method); - - switch (MethodHash) - { - case HashStringDjb2("import"sv): - { - if (!m_ProjectStore->AreDiskWritesAllowed()) - { - return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); - } - - CbObjectView Params = Cb["params"sv].AsObjectView(); - size_t MaxBlockSize = Params["maxblocksize"sv].AsUInt64(RemoteStoreOptions::DefaultMaxBlockSize); - size_t MaxChunkEmbedSize = Params["maxchunkembedsize"sv].AsUInt64(RemoteStoreOptions::DefaultMaxChunkEmbedSize); - bool Force = Params["force"sv].AsBool(false); - bool IgnoreMissingAttachments = Params["ignoremissingattachments"sv].AsBool(false); - bool CleanOplog = Params["clean"].AsBool(false); - - CreateRemoteStoreResult RemoteStoreResult = - CreateRemoteStore(Params, m_AuthMgr, MaxBlockSize, MaxChunkEmbedSize, Oplog->TempPath()); - - if (RemoteStoreResult.Store == nullptr) - { - return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, RemoteStoreResult.Description); - } - std::shared_ptr RemoteStore = std::move(RemoteStoreResult.Store); - RemoteProjectStore::RemoteStoreInfo StoreInfo = RemoteStore->GetInfo(); - - JobId JobId = m_JobQueue.QueueJob( - fmt::format("Import oplog '{}/{}'", Project->Identifier, Oplog->OplogId()), - [this, - ChunkStore = &m_CidStore, - ActualRemoteStore = std::move(RemoteStore), - Oplog, - Force, - IgnoreMissingAttachments, - CleanOplog](JobContext& Context) { - Context.ReportMessage(fmt::format("Loading oplog '{}/{}' from {}", - Oplog->GetOuterProjectIdentifier(), - Oplog->OplogId(), - ActualRemoteStore->GetInfo().Description)); - - WorkerThreadPool& WorkerPool = GetLargeWorkerPool(EWorkloadType::Background); - WorkerThreadPool& NetworkWorkerPool = GetMediumWorkerPool(EWorkloadType::Background); - - RemoteProjectStore::Result Result = LoadOplog(m_CidStore, - *ActualRemoteStore, - *Oplog, - NetworkWorkerPool, - WorkerPool, - Force, - IgnoreMissingAttachments, - CleanOplog, - &Context); - auto Response = ConvertResult(Result); - ZEN_INFO("LoadOplog: Status: {} '{}'", ToString(Response.first), Response.second); - if (!IsHttpSuccessCode(Response.first)) - { - throw JobError(Response.second.empty() ? fmt::format("Status: {}", ToString(Response.first)) : Response.second, - (int)Response.first); - } - }); - - return HttpReq.WriteResponse(HttpResponseCode::Accepted, HttpContentType::kText, fmt::format("{}", JobId.Id)); - } - case HashStringDjb2("export"sv): - { - CbObjectView Params = Cb["params"sv].AsObjectView(); - size_t MaxBlockSize = Params["maxblocksize"sv].AsUInt64(RemoteStoreOptions::DefaultMaxBlockSize); - size_t MaxChunkEmbedSize = Params["maxchunkembedsize"sv].AsUInt64(RemoteStoreOptions::DefaultMaxChunkEmbedSize); - size_t MaxChunksPerBlock = Params["maxchunksperblock"sv].AsUInt64(RemoteStoreOptions::DefaultMaxChunksPerBlock); - size_t ChunkFileSizeLimit = Params["chunkfilesizelimit"sv].AsUInt64(RemoteStoreOptions::DefaultChunkFileSizeLimit); - bool Force = Params["force"sv].AsBool(false); - bool IgnoreMissingAttachments = Params["ignoremissingattachments"sv].AsBool(false); - bool EmbedLooseFile = Params["embedloosefiles"sv].AsBool(false); - - CreateRemoteStoreResult RemoteStoreResult = - CreateRemoteStore(Params, m_AuthMgr, MaxBlockSize, MaxChunkEmbedSize, Oplog->TempPath()); - - if (RemoteStoreResult.Store == nullptr) - { - return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, RemoteStoreResult.Description); - } - std::shared_ptr RemoteStore = std::move(RemoteStoreResult.Store); - RemoteProjectStore::RemoteStoreInfo StoreInfo = RemoteStore->GetInfo(); - - JobId JobId = m_JobQueue.QueueJob( - fmt::format("Export oplog '{}/{}'", Project->Identifier, Oplog->OplogId()), - [this, - ActualRemoteStore = std::move(RemoteStore), - Project, - Oplog, - MaxBlockSize, - MaxChunksPerBlock, - MaxChunkEmbedSize, - ChunkFileSizeLimit, - EmbedLooseFile, - Force, - IgnoreMissingAttachments](JobContext& Context) { - Context.ReportMessage(fmt::format("Saving oplog '{}/{}' to {}, maxblocksize {}, maxchunkembedsize {}", - Project->Identifier, - Oplog->OplogId(), - ActualRemoteStore->GetInfo().Description, - NiceBytes(MaxBlockSize), - NiceBytes(MaxChunkEmbedSize))); - - WorkerThreadPool& WorkerPool = GetLargeWorkerPool(EWorkloadType::Background); - WorkerThreadPool& NetworkWorkerPool = GetMediumWorkerPool(EWorkloadType::Background); - - RemoteProjectStore::Result Result = SaveOplog(m_CidStore, - *ActualRemoteStore, - *Project, - *Oplog, - NetworkWorkerPool, - WorkerPool, - MaxBlockSize, - MaxChunksPerBlock, - MaxChunkEmbedSize, - ChunkFileSizeLimit, - EmbedLooseFile, - Force, - IgnoreMissingAttachments, - &Context); - auto Response = ConvertResult(Result); - ZEN_INFO("SaveOplog: Status: {} '{}'", ToString(Response.first), Response.second); - if (!IsHttpSuccessCode(Response.first)) - { - throw JobError(Response.second.empty() ? fmt::format("Status: {}", ToString(Response.first)) : Response.second, - (int)Response.first); - } - }); - - return HttpReq.WriteResponse(HttpResponseCode::Accepted, HttpContentType::kText, fmt::format("{}", JobId.Id)); - } - case HashStringDjb2("getchunks"sv): - { - RpcAcceptOptions AcceptFlags = static_cast(Cb["AcceptFlags"sv].AsUInt16(0u)); - int32_t TargetProcessId = Cb["Pid"sv].AsInt32(0); - - std::vector Requests = m_ProjectStore->ParseChunksRequests(*Project, *Oplog, Cb); - std::vector Results = - Requests.empty() ? std::vector{} : m_ProjectStore->GetChunks(*Project, *Oplog, Requests); - CbPackage Response = m_ProjectStore->WriteChunksRequestResponse(*Project, *Oplog, std::move(Requests), std::move(Results)); - - void* TargetProcessHandle = nullptr; - FormatFlags Flags = FormatFlags::kDefault; - if (EnumHasAllFlags(AcceptFlags, RpcAcceptOptions::kAllowLocalReferences)) - { - Flags |= FormatFlags::kAllowLocalReferences; - if (!EnumHasAnyFlags(AcceptFlags, RpcAcceptOptions::kAllowPartialLocalReferences)) - { - Flags |= FormatFlags::kDenyPartialLocalReferences; - } - TargetProcessHandle = m_OpenProcessCache.GetProcessHandle(HttpReq.SessionId(), TargetProcessId); - } - - CompositeBuffer RpcResponseBuffer = FormatPackageMessageBuffer(Response, Flags, TargetProcessHandle); - return HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, RpcResponseBuffer); - } - case HashStringDjb2("putchunks"sv): - { - ZEN_TRACE_CPU("Store::Rpc::putchunks"); - if (!m_ProjectStore->AreDiskWritesAllowed()) - { - return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); - } - - CbObject Object = Package.GetObject(); - const bool UsingTempFiles = Object["usingtmpfiles"].AsBool(false); - - std::span Attachments = Package.GetAttachments(); - if (!Attachments.empty()) - { - std::vector WriteAttachmentBuffers; - std::vector WriteRawHashes; - - WriteAttachmentBuffers.reserve(Attachments.size()); - WriteRawHashes.reserve(Attachments.size()); - - for (const CbAttachment& Attachment : Attachments) - { - IoHash RawHash = Attachment.GetHash(); - const CompressedBuffer& Compressed = Attachment.AsCompressedBinary(); - IoBuffer AttachmentPayload = Compressed.GetCompressed().Flatten().AsIoBuffer(); - if (UsingTempFiles) - { - AttachmentPayload.SetDeleteOnClose(true); - } - WriteAttachmentBuffers.push_back(std::move(AttachmentPayload)); - WriteRawHashes.push_back(RawHash); - } - - Oplog->CaptureAddedAttachments(WriteRawHashes); - m_CidStore.AddChunks(WriteAttachmentBuffers, - WriteRawHashes, - UsingTempFiles ? CidStore::InsertMode::kMayBeMovedInPlace : CidStore::InsertMode::kCopyOnly); - } - return HttpReq.WriteResponse(HttpResponseCode::OK); - } - case HashStringDjb2("snapshot"sv): - { - ZEN_TRACE_CPU("Store::Rpc::snapshot"); - if (!m_ProjectStore->AreDiskWritesAllowed()) - { - return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); - } - - // Snapshot all referenced files. This brings the content of all - // files into the CID store - - uint32_t OpCount = 0; - uint64_t InlinedBytes = 0; - uint64_t InlinedFiles = 0; - uint64_t TotalBytes = 0; - uint64_t TotalFiles = 0; - - std::vector NewOps; - struct AddedChunk - { - IoBuffer Buffer; - uint64_t RawSize = 0; - }; - tsl::robin_map AddedChunks; - - Oplog->IterateOplog( - [&](CbObjectView Op) { - bool OpRewritten = false; - bool AllOk = true; - - CbWriter FilesArrayWriter; - FilesArrayWriter.BeginArray("files"sv); - - for (CbFieldView& Field : Op["files"sv]) - { - bool CopyField = true; - - if (CbObjectView View = Field.AsObjectView()) - { - const IoHash DataHash = View["data"sv].AsHash(); - - if (DataHash == IoHash::Zero) - { - std::string_view ServerPath = View["serverpath"sv].AsString(); - std::filesystem::path FilePath = Project->RootDir / ServerPath; - BasicFile DataFile; - std::error_code Ec; - DataFile.Open(FilePath, BasicFile::Mode::kRead, Ec); - - if (Ec) - { - // Error... - - ZEN_ERROR("unable to read data from file '{}': {}", FilePath, Ec.message()); - - AllOk = false; - } - else - { - // Read file contents into memory, compress and keep in map of chunks to add to Cid store - IoBuffer FileIoBuffer = DataFile.ReadAll(); - CompressedBuffer Compressed = CompressedBuffer::Compress(SharedBuffer(std::move(FileIoBuffer))); - const uint64_t RawSize = Compressed.DecodeRawSize(); - const IoHash RawHash = Compressed.DecodeRawHash(); - if (!AddedChunks.contains(RawHash)) - { - const std::filesystem::path TempChunkPath = Oplog->TempPath() / RawHash.ToHexString(); - BasicFile ChunkTempFile; - ChunkTempFile.Open(TempChunkPath, BasicFile::Mode::kTruncateDelete); - ChunkTempFile.Write(Compressed.GetCompressed(), 0, Ec); - if (Ec) - { - Oid ChunkId = View["id"sv].AsObjectId(); - ZEN_ERROR("unable to write external file as compressed chunk '{}', id {}: {}", - FilePath, - ChunkId, - Ec.message()); - AllOk = false; - } - else - { - void* FileHandle = ChunkTempFile.Detach(); - IoBuffer ChunkBuffer(IoBuffer::File, - FileHandle, - 0, - Compressed.GetCompressed().GetSize(), - /*IsWholeFile*/ true); - ChunkBuffer.SetDeleteOnClose(true); - AddedChunks.insert_or_assign( - RawHash, - AddedChunk{.Buffer = std::move(ChunkBuffer), .RawSize = RawSize}); - } - } - - TotalBytes += RawSize; - ++TotalFiles; - - // Rewrite file array entry with new data reference - CbObjectWriter Writer(View.GetSize()); - RewriteCbObject(Writer, View, [&](CbObjectWriter&, CbFieldView Field) -> bool { - if (Field.GetName() == "data"sv) - { - // omit this field as we will write it explicitly ourselves - return true; - } - return false; - }); - Writer.AddBinaryAttachment("data"sv, RawHash); - - CbObject RewrittenOp = Writer.Save(); - FilesArrayWriter.AddObject(std::move(RewrittenOp)); - CopyField = false; - } - } - } - - if (CopyField) - { - FilesArrayWriter.AddField(Field); - } - else - { - OpRewritten = true; - } - } - - if (OpRewritten && AllOk) - { - FilesArrayWriter.EndArray(); - CbArray FilesArray = FilesArrayWriter.Save().AsArray(); - - CbObject RewrittenOp = RewriteCbObject(Op, [&](CbObjectWriter& NewWriter, CbFieldView Field) -> bool { - if (Field.GetName() == "files"sv) - { - NewWriter.AddArray("files"sv, FilesArray); - - return true; - } - - return false; - }); - - NewOps.push_back(std::move(RewrittenOp)); - } - - OpCount++; - }, - ProjectStore::Oplog::Paging{}); - - CbObjectWriter ResponseObj; - - // Persist rewritten oplog entries - if (!NewOps.empty()) - { - ResponseObj.BeginArray("rewritten_ops"); - - for (CbObject& NewOp : NewOps) - { - ProjectStore::LogSequenceNumber NewLsn = Oplog->AppendNewOplogEntry(std::move(NewOp)); - - ZEN_DEBUG("appended rewritten op at LSN: {}", NewLsn.Number); - - ResponseObj.AddInteger(NewLsn.Number); - } - - ResponseObj.EndArray(); - } - - // Ops that have moved chunks to a compressed buffer for storage in m_CidStore have been rewritten with references to the - // new chunk(s). Make sure we add the chunks to m_CidStore, and do it after we update the oplog so GC doesn't think we have - // unreferenced chunks. - for (auto It : AddedChunks) - { - const IoHash& RawHash = It.first; - AddedChunk& Chunk = It.second; - CidStore::InsertResult Result = m_CidStore.AddChunk(Chunk.Buffer, RawHash); - if (Result.New) - { - InlinedBytes += Chunk.RawSize; - ++InlinedFiles; - } - } - - ResponseObj << "inlined_bytes" << InlinedBytes << "inlined_files" << InlinedFiles; - ResponseObj << "total_bytes" << TotalBytes << "total_files" << TotalFiles; - - ZEN_INFO("oplog '{}/{}': rewrote {} oplog entries (out of {})", ProjectId, OplogId, NewOps.size(), OpCount); - - return HttpReq.WriteResponse(HttpResponseCode::OK, ResponseObj.Save()); - } - case HashStringDjb2("appendops"sv): - { - ZEN_TRACE_CPU("Store::Rpc::appendops"); - if (!m_ProjectStore->AreDiskWritesAllowed()) - { - return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); - } - - CbArrayView OpsArray = Cb["ops"sv].AsArrayView(); - - size_t OpsBufferSize = 0; - for (CbFieldView OpView : OpsArray) - { - OpsBufferSize += OpView.GetSize(); - } - UniqueBuffer OpsBuffers = UniqueBuffer::Alloc(OpsBufferSize); - MutableMemoryView OpsBuffersMemory = OpsBuffers.GetMutableView(); - - std::vector Ops; - Ops.reserve(OpsArray.Num()); - for (CbFieldView OpView : OpsArray) - { - OpView.CopyTo(OpsBuffersMemory); - Ops.push_back(CbObjectView(OpsBuffersMemory.GetData())); - OpsBuffersMemory.MidInline(OpView.GetSize()); - } - - std::vector LSNs = Oplog->AppendNewOplogEntries(Ops); - ZEN_ASSERT(LSNs.size() == Ops.size()); - - std::vector MissingAttachments; - for (size_t OpIndex = 0; OpIndex < Ops.size(); OpIndex++) - { - if (LSNs[OpIndex]) - { - CbObjectView Op = Ops[OpIndex]; - Op.IterateAttachments([this, &MissingAttachments](CbFieldView AttachmentView) { - const IoHash Cid = AttachmentView.AsAttachment(); - if (!m_CidStore.ContainsChunk(Cid)) - { - MissingAttachments.push_back(Cid); - } - }); - } - } - - CbPackage ResponsePackage; - - { - CbObjectWriter ResponseObj; - ResponseObj.BeginArray("written_ops"); - - for (ProjectStore::LogSequenceNumber NewLsn : LSNs) - { - ZEN_DEBUG("appended written op at LSN: {}", NewLsn.Number); - ResponseObj.AddInteger(NewLsn.Number); - } - ResponseObj.EndArray(); - - if (!MissingAttachments.empty()) - { - ResponseObj.BeginArray("need"); - - for (const IoHash& Cid : MissingAttachments) - { - ResponseObj.AddHash(Cid); - } - ResponseObj.EndArray(); - } - ResponsePackage.SetObject(ResponseObj.Save()); - } - - std::vector ResponseBuffers = FormatPackageMessage(ResponsePackage); - - return HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, ResponseBuffers); - } - default: - m_ProjectStats.BadRequestCount++; - return HttpReq.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Unknown rpc method '{}'", Method)); - } -} -void -HttpProjectService::HandleDetailsRequest(HttpRouterRequest& Req) -{ - ZEN_TRACE_CPU("ProjectService::Details"); - - using namespace std::literals; - - HttpServerRequest& HttpReq = Req.ServerRequest(); - - HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); - bool CSV = Params.GetValue("csv"sv) == "true"sv; - bool Details = Params.GetValue("details"sv) == "true"sv; - bool OpDetails = Params.GetValue("opdetails"sv) == "true"sv; - bool AttachmentDetails = Params.GetValue("attachmentdetails"sv) == "true"sv; - - if (CSV) - { - ExtendableStringBuilder<4096> CSVWriter; - CSVHeader(Details, AttachmentDetails, CSVWriter); - - m_ProjectStore->IterateProjects([&](ProjectStore::Project& Project) { - Project.IterateOplogs([&](const RwLock::SharedLockScope&, ProjectStore::Oplog& Oplog) { - Oplog.IterateOplogWithKey( - [this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](ProjectStore::LogSequenceNumber LSN, - const Oid& Key, - CbObjectView Op) { - CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter); - }); - }); - }); - - HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView()); - } - else - { - CbObjectWriter Cbo; - Cbo.BeginArray("projects"); - { - m_ProjectStore->DiscoverProjects(); - - m_ProjectStore->IterateProjects([&](ProjectStore::Project& Project) { - std::vector OpLogs = Project.ScanForOplogs(); - CbWriteProject(m_CidStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo); - }); - } - Cbo.EndArray(); - HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); - } -} - -void -HttpProjectService::HandleProjectDetailsRequest(HttpRouterRequest& Req) -{ - ZEN_TRACE_CPU("ProjectService::ProjectDetails"); - - using namespace std::literals; - - HttpServerRequest& HttpReq = Req.ServerRequest(); - const auto& ProjectId = Req.GetCapture(1); - - HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); - bool CSV = Params.GetValue("csv"sv) == "true"sv; - bool Details = Params.GetValue("details"sv) == "true"sv; - bool OpDetails = Params.GetValue("opdetails"sv) == "true"sv; - bool AttachmentDetails = Params.GetValue("attachmentdetails"sv) == "true"sv; - - Ref FoundProject = m_ProjectStore->OpenProject(ProjectId); - if (!FoundProject) - { - return HttpReq.WriteResponse(HttpResponseCode::NotFound); - } - ProjectStore::Project& Project = *FoundProject.Get(); - - if (CSV) - { - ExtendableStringBuilder<4096> CSVWriter; - CSVHeader(Details, AttachmentDetails, CSVWriter); - - FoundProject->IterateOplogs([&](const RwLock::SharedLockScope&, ProjectStore::Oplog& Oplog) { - Oplog.IterateOplogWithKey([this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](ProjectStore::LogSequenceNumber LSN, - const Oid& Key, - CbObjectView Op) { - CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter); - }); - }); - HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView()); - } - else - { - CbObjectWriter Cbo; - std::vector OpLogs = FoundProject->ScanForOplogs(); - Cbo.BeginArray("projects"); - { - CbWriteProject(m_CidStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo); - } - Cbo.EndArray(); - HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); - } -} - -void -HttpProjectService::HandleOplogDetailsRequest(HttpRouterRequest& Req) -{ - ZEN_TRACE_CPU("ProjectService::OplogDetails"); - - using namespace std::literals; - - HttpServerRequest& HttpReq = Req.ServerRequest(); - const auto& ProjectId = Req.GetCapture(1); - const auto& OplogId = Req.GetCapture(2); - - HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); - bool CSV = Params.GetValue("csv"sv) == "true"sv; - bool Details = Params.GetValue("details"sv) == "true"sv; - bool OpDetails = Params.GetValue("opdetails"sv) == "true"sv; - bool AttachmentDetails = Params.GetValue("attachmentdetails"sv) == "true"sv; - - Ref FoundProject = m_ProjectStore->OpenProject(ProjectId); - if (!FoundProject) - { - return HttpReq.WriteResponse(HttpResponseCode::NotFound); - } - - Ref FoundLog = FoundProject->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ true); - if (!FoundLog) - { - return HttpReq.WriteResponse(HttpResponseCode::NotFound); - } - - ProjectStore::Project& Project = *FoundProject.Get(); - ProjectStore::Oplog& Oplog = *FoundLog; - if (CSV) - { - ExtendableStringBuilder<4096> CSVWriter; - CSVHeader(Details, AttachmentDetails, CSVWriter); - - Oplog.IterateOplogWithKey([this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](ProjectStore::LogSequenceNumber LSN, - const Oid& Key, - CbObjectView Op) { - CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter); - }); - HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView()); - } - else - { - CbObjectWriter Cbo; - Cbo.BeginArray("oplogs"); - { - CbWriteOplog(m_CidStore, Oplog, Details, OpDetails, AttachmentDetails, Cbo); - } - Cbo.EndArray(); - HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); - } -} - -void -HttpProjectService::HandleOplogOpDetailsRequest(HttpRouterRequest& Req) -{ - ZEN_TRACE_CPU("ProjectService::OplogOpDetails"); - - using namespace std::literals; - - HttpServerRequest& HttpReq = Req.ServerRequest(); - const auto& ProjectId = Req.GetCapture(1); - const auto& OplogId = Req.GetCapture(2); - const auto& OpId = Req.GetCapture(3); - - HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); - bool CSV = Params.GetValue("csv"sv) == "true"sv; - bool Details = Params.GetValue("details"sv) == "true"sv; - bool OpDetails = Params.GetValue("opdetails"sv) == "true"sv; - bool AttachmentDetails = Params.GetValue("attachmentdetails"sv) == "true"sv; - - Ref FoundProject = m_ProjectStore->OpenProject(ProjectId); - if (!FoundProject) - { - return HttpReq.WriteResponse(HttpResponseCode::NotFound); - } - - Ref FoundLog = FoundProject->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ true); - if (!FoundLog) - { - return HttpReq.WriteResponse(HttpResponseCode::NotFound); - } - - if (OpId.size() != 2 * sizeof(Oid::OidBits)) - { - m_ProjectStats.BadRequestCount++; - return HttpReq.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Chunk info request for invalid chunk id '{}/{}'/'{}'", ProjectId, OplogId, OpId)); - } - - const Oid ObjId = Oid::FromHexString(OpId); - ProjectStore::Project& Project = *FoundProject.Get(); - ProjectStore::Oplog& Oplog = *FoundLog; - - std::optional Op = Oplog.GetOpByKey(ObjId); - if (!Op.has_value()) - { - return HttpReq.WriteResponse(HttpResponseCode::NotFound); - } - ProjectStore::LogSequenceNumber LSN = Oplog.GetOpIndexByKey(ObjId); - if (!LSN) - { - return HttpReq.WriteResponse(HttpResponseCode::NotFound); - } - - if (CSV) - { - ExtendableStringBuilder<4096> CSVWriter; - CSVHeader(Details, AttachmentDetails, CSVWriter); - - CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, ObjId, Op.value(), CSVWriter); - HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView()); - } - else - { - CbObjectWriter Cbo; - Cbo.BeginArray("ops"); - { - CbWriteOp(m_CidStore, Details, OpDetails, AttachmentDetails, LSN, ObjId, Op.value(), Cbo); - } - Cbo.EndArray(); - HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); - } -} - -} // namespace zen diff --git a/src/zenserver/projectstore/httpprojectstore.h b/src/zenserver/projectstore/httpprojectstore.h deleted file mode 100644 index f0a0bcfa1..000000000 --- a/src/zenserver/projectstore/httpprojectstore.h +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include -#include -#include -#include -#include - -namespace zen { - -class AuthMgr; -class JobQueue; -class OpenProcessCache; -class ProjectStore; - -////////////////////////////////////////////////////////////////////////// -// -// {project} a project identifier -// {target} a variation of the project, typically a build target -// {lsn} oplog entry sequence number -// -// /prj/{project} -// /prj/{project}/oplog/{target} -// /prj/{project}/oplog/{target}/{lsn} -// -// oplog entry -// -// id: {id} -// key: {} -// meta: {} -// data: [] -// refs: -// - -class HttpProjectService : public HttpService, public IHttpStatusProvider, public IHttpStatsProvider -{ -public: - HttpProjectService(CidStore& Store, - ProjectStore* InProjectStore, - HttpStatusService& StatusService, - HttpStatsService& StatsService, - AuthMgr& AuthMgr, - OpenProcessCache& InOpenProcessCache, - JobQueue& InJobQueue); - ~HttpProjectService(); - - virtual const char* BaseUri() const override; - virtual void HandleRequest(HttpServerRequest& Request) override; - - virtual void HandleStatsRequest(HttpServerRequest& Request) override; - virtual void HandleStatusRequest(HttpServerRequest& Request) override; - -private: - struct ProjectStats - { - std::atomic_uint64_t ProjectReadCount{}; - std::atomic_uint64_t ProjectWriteCount{}; - std::atomic_uint64_t ProjectDeleteCount{}; - std::atomic_uint64_t OpLogReadCount{}; - std::atomic_uint64_t OpLogWriteCount{}; - std::atomic_uint64_t OpLogDeleteCount{}; - std::atomic_uint64_t OpHitCount{}; - std::atomic_uint64_t OpMissCount{}; - std::atomic_uint64_t OpWriteCount{}; - std::atomic_uint64_t ChunkHitCount{}; - std::atomic_uint64_t ChunkMissCount{}; - std::atomic_uint64_t ChunkWriteCount{}; - std::atomic_uint64_t RequestCount{}; - std::atomic_uint64_t BadRequestCount{}; - }; - - void HandleProjectListRequest(HttpRouterRequest& Req); - void HandleChunkBatchRequest(HttpRouterRequest& Req); - void HandleFilesRequest(HttpRouterRequest& Req); - void HandleChunkInfosRequest(HttpRouterRequest& Req); - void HandleChunkInfoRequest(HttpRouterRequest& Req); - void HandleChunkByIdRequest(HttpRouterRequest& Req); - void HandleChunkByCidRequest(HttpRouterRequest& Req); - void HandleOplogOpPrepRequest(HttpRouterRequest& Req); - void HandleOplogOpNewRequest(HttpRouterRequest& Req); - void HandleOplogValidateRequest(HttpRouterRequest& Req); - void HandleOpLogOpRequest(HttpRouterRequest& Req); - void HandleOpLogRequest(HttpRouterRequest& Req); - void HandleOpLogEntriesRequest(HttpRouterRequest& Req); - void HandleProjectRequest(HttpRouterRequest& Req); - void HandleOplogSaveRequest(HttpRouterRequest& Req); - void HandleOplogLoadRequest(HttpRouterRequest& Req); - void HandleRpcRequest(HttpRouterRequest& Req); - void HandleDetailsRequest(HttpRouterRequest& Req); - void HandleProjectDetailsRequest(HttpRouterRequest& Req); - void HandleOplogDetailsRequest(HttpRouterRequest& Req); - void HandleOplogOpDetailsRequest(HttpRouterRequest& Req); - - inline LoggerRef Log() { return m_Log; } - - LoggerRef m_Log; - CidStore& m_CidStore; - HttpRequestRouter m_Router; - Ref m_ProjectStore; - HttpStatusService& m_StatusService; - HttpStatsService& m_StatsService; - AuthMgr& m_AuthMgr; - OpenProcessCache& m_OpenProcessCache; - JobQueue& m_JobQueue; - ProjectStats m_ProjectStats; - metrics::OperationTiming m_HttpRequests; -}; - -} // namespace zen diff --git a/src/zenserver/stats/statsreporter.h b/src/zenserver/stats/statsreporter.h index 2f93aa8bb..b4174073c 100644 --- a/src/zenserver/stats/statsreporter.h +++ b/src/zenserver/stats/statsreporter.h @@ -2,7 +2,7 @@ #pragma once -#include "config.h" +#include "config/config.h" #include #include diff --git a/src/zenserver/storage/admin/admin.cpp b/src/zenserver/storage/admin/admin.cpp new file mode 100644 index 000000000..4803063d7 --- /dev/null +++ b/src/zenserver/storage/admin/admin.cpp @@ -0,0 +1,804 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "admin.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if ZEN_WITH_TRACE +# include +#endif // ZEN_WITH_TRACE + +#if ZEN_USE_MIMALLOC +# include +#endif + +#include "config/config.h" + +#include + +namespace zen { + +struct DirStats +{ + uint64_t FileCount = 0; + uint64_t DirCount = 0; + uint64_t ByteCount = 0; +}; + +DirStats +GetStatsForDirectory(std::filesystem::path Dir) +{ + if (!IsDir(Dir)) + return {}; + + struct StatsTraversal : public GetDirectoryContentVisitor + { + virtual void AsyncVisitDirectory(const std::filesystem::path& RelativeRoot, DirectoryContent&& Content) override + { + ZEN_UNUSED(RelativeRoot); + + uint64_t FileCount = Content.FileNames.size(); + uint64_t DirCount = Content.DirectoryNames.size(); + uint64_t FilesSize = 0; + for (uint64_t FileSize : Content.FileSizes) + { + FilesSize += FileSize; + } + TotalBytes += FilesSize; + TotalFileCount += FileCount; + TotalDirCount += DirCount; + } + + std::atomic_uint64_t TotalBytes = 0; + std::atomic_uint64_t TotalFileCount = 0; + std::atomic_uint64_t TotalDirCount = 0; + + DirStats GetStats() + { + return {.FileCount = TotalFileCount.load(), .DirCount = TotalDirCount.load(), .ByteCount = TotalBytes.load()}; + } + } DirTraverser; + + Latch PendingWorkCount(1); + + GetDirectoryContent(Dir, + DirectoryContentFlags::IncludeAllEntries | DirectoryContentFlags::IncludeFileSizes, + DirTraverser, + GetSmallWorkerPool(EWorkloadType::Background), + PendingWorkCount); + PendingWorkCount.CountDown(); + PendingWorkCount.Wait(); + + return DirTraverser.GetStats(); +} + +struct StateDiskStats +{ + DirStats CacheStats; + DirStats CasStats; + DirStats ProjectStats; +}; + +StateDiskStats +GetStatsForStateDirectory(std::filesystem::path StateDir) +{ + StateDiskStats Stats; + Stats.CacheStats = GetStatsForDirectory(StateDir / "cache"); + Stats.CasStats = GetStatsForDirectory(StateDir / "cas"); + Stats.ProjectStats = GetStatsForDirectory(StateDir / "projects"); + return Stats; +} + +HttpAdminService::HttpAdminService(GcScheduler& Scheduler, + JobQueue& BackgroundJobQueue, + ZenCacheStore* CacheStore, + std::function&& FlushFunction, + const LogPaths& LogPaths, + const ZenServerOptions& ServerOptions) +: m_GcScheduler(Scheduler) +, m_BackgroundJobQueue(BackgroundJobQueue) +, m_CacheStore(CacheStore) +, m_FlushFunction(std::move(FlushFunction)) +, m_LogPaths(LogPaths) +, m_ServerOptions(ServerOptions) +{ + using namespace std::literals; + + m_Router.RegisterRoute( + "health", + [](HttpRouterRequest& Req) { + CbObjectWriter Obj; + Obj.AddBool("ok", true); + Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); + }, + HttpVerb::kGet); + + m_Router.AddPattern("jobid", "([[:digit:]]+?)"); + + m_Router.RegisterRoute( + "jobs", + [&](HttpRouterRequest& Req) { + std::vector Jobs = m_BackgroundJobQueue.GetJobs(); + CbObjectWriter Obj; + Obj.BeginArray("jobs"); + for (const auto& Job : Jobs) + { + Obj.BeginObject(); + Obj.AddInteger("Id", Job.Id.Id); + Obj.AddString("Status", JobQueue::ToString(Job.Status)); + Obj.EndObject(); + } + Obj.EndArray(); + Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); + }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "jobs/{jobid}", + [&](HttpRouterRequest& Req) { + const auto& JobIdString = Req.GetCapture(1); + std::optional JobIdArg = ParseInt(JobIdString); + if (!JobIdArg) + { + Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); + } + JobId Id{.Id = JobIdArg.value_or(0)}; + if (Id.Id == 0) + { + return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, + ZenContentType::kText, + fmt::format("Invalid Job Id: {}", Id.Id)); + } + + std::optional CurrentState = m_BackgroundJobQueue.Get(Id); + if (!CurrentState) + { + return Req.ServerRequest().WriteResponse(HttpResponseCode::NotFound); + } + + auto WriteState = [](CbObjectWriter& Obj, const JobQueue::State& State) { + if (!State.CurrentOp.empty()) + { + Obj.AddString( + "CurrentOp"sv, + State.CurrentOpDetails.empty() ? State.CurrentOp : fmt::format("{}: {}", State.CurrentOp, State.CurrentOpDetails)); + Obj.AddString("Op"sv, State.CurrentOp); + if (!State.CurrentOpDetails.empty()) + { + Obj.AddString("Details"sv, State.CurrentOpDetails); + } + Obj.AddInteger("TotalCount"sv, gsl::narrow(State.TotalCount)); + Obj.AddInteger("RemainingCount"sv, gsl::narrow(State.RemainingCount)); + Obj.AddInteger("CurrentOpPercentComplete"sv, + State.TotalCount > 0 + ? gsl::narrow((100 * (State.TotalCount - State.RemainingCount)) / State.TotalCount) + : 0); + } + if (!State.Messages.empty()) + { + Obj.BeginArray("Messages"); + for (const std::string& Message : State.Messages) + { + Obj.AddString(Message); + } + Obj.EndArray(); + } + if (!State.AbortReason.empty()) + { + Obj.AddString("AbortReason"sv, State.AbortReason); + } + }; + + auto GetAgeAsSeconds = [](std::chrono::system_clock::time_point Start, std::chrono::system_clock::time_point End) { + auto Age = End - Start; + auto Milliseconds = std::chrono::duration_cast(Age); + return Milliseconds.count() / 1000.0; + }; + + const std::chrono::system_clock::time_point Now = std::chrono::system_clock::now(); + + switch (CurrentState->Status) + { + case JobQueue::Status::Queued: + { + CbObjectWriter Obj; + Obj.AddString("Name"sv, CurrentState->Name); + Obj.AddString("Status"sv, "Queued"sv); + Obj.AddFloat("QueueTimeS", GetAgeAsSeconds(CurrentState->CreateTime, Now)); + Obj.AddInteger("WorkerThread", CurrentState->WorkerThreadId); + Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); + } + break; + case JobQueue::Status::Running: + { + CbObjectWriter Obj; + Obj.AddString("Name"sv, CurrentState->Name); + Obj.AddString("Status"sv, "Running"sv); + WriteState(Obj, CurrentState->State); + Obj.AddFloat("QueueTimeS", GetAgeAsSeconds(CurrentState->CreateTime, CurrentState->StartTime)); + Obj.AddFloat("RunTimeS", GetAgeAsSeconds(CurrentState->StartTime, Now)); + Obj.AddInteger("WorkerThread", CurrentState->WorkerThreadId); + Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); + } + break; + case JobQueue::Status::Aborted: + { + CbObjectWriter Obj; + Obj.AddString("Name"sv, CurrentState->Name); + Obj.AddString("Status"sv, "Aborted"sv); + + WriteState(Obj, CurrentState->State); + Obj.AddFloat("QueueTimeS", GetAgeAsSeconds(CurrentState->CreateTime, CurrentState->StartTime)); + Obj.AddFloat("RunTimeS", GetAgeAsSeconds(CurrentState->StartTime, CurrentState->EndTime)); + Obj.AddFloat("CompleteTimeS", GetAgeAsSeconds(CurrentState->EndTime, Now)); + Obj.AddInteger("ReturnCode", CurrentState->ReturnCode); + Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); + } + break; + case JobQueue::Status::Completed: + { + CbObjectWriter Obj; + Obj.AddString("Name"sv, CurrentState->Name); + Obj.AddString("Status"sv, "Complete"sv); + WriteState(Obj, CurrentState->State); + Obj.AddFloat("QueueTimeS", GetAgeAsSeconds(CurrentState->CreateTime, CurrentState->StartTime)); + Obj.AddFloat("RunTimeS", GetAgeAsSeconds(CurrentState->StartTime, CurrentState->EndTime)); + Obj.AddFloat("CompleteTimeS", GetAgeAsSeconds(CurrentState->EndTime, Now)); + Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); + } + break; + } + }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "jobs/{jobid}", + [&](HttpRouterRequest& Req) { + const auto& JobIdString = Req.GetCapture(1); + std::optional JobIdArg = ParseInt(JobIdString); + if (!JobIdArg) + { + Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); + } + JobId Id{.Id = JobIdArg.value_or(0)}; + if (m_BackgroundJobQueue.CancelJob(Id)) + { + Req.ServerRequest().WriteResponse(HttpResponseCode::OK); + } + else + { + Req.ServerRequest().WriteResponse(HttpResponseCode::NotFound); + } + }, + HttpVerb::kDelete); + + m_Router.RegisterRoute( + "gc", + [this](HttpRouterRequest& Req) { + const GcSchedulerState State = m_GcScheduler.GetState(); + + const HttpServerRequest::QueryParams Params = Req.ServerRequest().GetQueryParams(); + + bool Details = false; + if (auto Param = Params.GetValue("details"); Param == "true") + { + Details = true; + } + + CbObjectWriter Response; + Response << "Status"sv << (GcSchedulerStatus::kIdle == State.Status ? "Idle"sv : "Running"sv); + Response.BeginObject("Config"); + { + Response << "RootDirectory" << State.Config.RootDirectory.string(); + Response << "MonitorInterval" << ToTimeSpan(State.Config.MonitorInterval); + Response << "Interval" << ToTimeSpan(State.Config.Interval); + Response << "MaxCacheDuration" << ToTimeSpan(State.Config.MaxCacheDuration); + Response << "MaxProjectStoreDuration" << ToTimeSpan(State.Config.MaxProjectStoreDuration); + Response << "MaxBuildStoreDuration" << ToTimeSpan(State.Config.MaxBuildStoreDuration); + Response << "CollectSmallObjects" << State.Config.CollectSmallObjects; + Response << "Enabled" << State.Config.Enabled; + Response << "DiskReserveSize" << NiceBytes(State.Config.DiskReserveSize); + Response << "DiskSizeSoftLimit" << NiceBytes(State.Config.DiskSizeSoftLimit); + Response << "MinimumFreeDiskSpaceToAllowWrites" << NiceBytes(State.Config.MinimumFreeDiskSpaceToAllowWrites); + Response << "LightweightInterval" << ToTimeSpan(State.Config.LightweightInterval); + Response << "UseGCVersion" << ((State.Config.UseGCVersion == GcVersion::kV1_Deprecated) ? "1" : "2"); + Response << "CompactBlockUsageThresholdPercent" << State.Config.CompactBlockUsageThresholdPercent; + Response << "Verbose" << State.Config.Verbose; + Response << "SingleThreaded" << State.Config.SingleThreaded; + Response << "AttachmentPassCount" << State.Config.AttachmentPassCount; + } + Response.EndObject(); + Response << "AreDiskWritesBlocked" << State.AreDiskWritesBlocked; + Response << "HasDiskReserve" << State.HasDiskReserve; + Response << "DiskSize" << NiceBytes(State.DiskSize); + Response << "DiskUsed" << NiceBytes(State.DiskUsed); + Response << "DiskFree" << NiceBytes(State.DiskFree); + + Response.BeginObject("FullGC"); + { + Response << "LastTime" << ToDateTime(State.LastFullGcTime); + Response << "TimeToNext" << ToTimeSpan(State.RemainingTimeUntilFullGc); + if (State.Config.DiskSizeSoftLimit != 0) + { + Response << "SpaceToNext" << NiceBytes(State.RemainingSpaceUntilFullGC); + } + if (State.LastFullGCV2Result) + { + const bool HumanReadable = true; + WriteGCResult(Response, State.LastFullGCV2Result.value(), HumanReadable, Details); + } + else + { + Response << "LastDuration" << ToTimeSpan(State.LastFullGcDuration); + Response << "LastDiskFreed" << NiceBytes(State.LastFullGCDiff.DiskSize); + Response << "LastMemoryFreed" << NiceBytes(State.LastFullGCDiff.MemorySize); + } + if (State.LastFullAttachmentRangeMin != IoHash::Zero || State.LastFullAttachmentRangeMax != IoHash::Max) + { + Response << "AttachmentRangeMin" << State.LastFullAttachmentRangeMin; + Response << "AttachmentRangeMax" << State.LastFullAttachmentRangeMax; + } + } + Response.EndObject(); + Response.BeginObject("LightweightGC"); + { + Response << "LastTime" << ToDateTime(State.LastLightweightGcTime); + Response << "TimeToNext" << ToTimeSpan(State.RemainingTimeUntilLightweightGc); + + if (State.LastLightweightGCV2Result) + { + const bool HumanReadable = true; + WriteGCResult(Response, State.LastLightweightGCV2Result.value(), HumanReadable, Details); + } + else + { + Response << "LastDuration" << ToTimeSpan(State.LastLightweightGcDuration); + Response << "LastDiskFreed" << NiceBytes(State.LastLightweightGCDiff.DiskSize); + Response << "LastMemoryFreed" << NiceBytes(State.LastLightweightGCDiff.MemorySize); + } + } + Response.EndObject(); + Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Response.Save()); + }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "gc", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + const HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); + GcScheduler::TriggerGcParams GcParams; + + if (auto Param = Params.GetValue("smallobjects"); Param.empty() == false) + { + GcParams.CollectSmallObjects = Param == "true"sv; + } + + if (auto Param = Params.GetValue("maxcacheduration"); Param.empty() == false) + { + if (auto Value = ParseInt(Param)) + { + GcParams.MaxCacheDuration = std::chrono::seconds(Value.value()); + } + } + + if (auto Param = Params.GetValue("maxprojectstoreduration"); Param.empty() == false) + { + if (auto Value = ParseInt(Param)) + { + GcParams.MaxProjectStoreDuration = std::chrono::seconds(Value.value()); + } + } + + if (auto Param = Params.GetValue("maxbuildstoreduration"); Param.empty() == false) + { + if (auto Value = ParseInt(Param)) + { + GcParams.MaxBuildStoreDuration = std::chrono::seconds(Value.value()); + } + } + + if (auto Param = Params.GetValue("disksizesoftlimit"); Param.empty() == false) + { + if (auto Value = ParseInt(Param)) + { + GcParams.DiskSizeSoftLimit = Value.value(); + } + } + + if (auto Param = Params.GetValue("skipcid"); Param.empty() == false) + { + GcParams.SkipCid = Param == "true"sv; + } + + if (auto Param = Params.GetValue("skipdelete"); Param.empty() == false) + { + GcParams.SkipDelete = Param == "true"sv; + } + + if (auto Param = Params.GetValue("forceusegcv1"); Param.empty() == false) + { + GcParams.ForceGCVersion = GcVersion::kV1_Deprecated; + } + + if (auto Param = Params.GetValue("forceusegcv2"); Param.empty() == false) + { + GcParams.ForceGCVersion = GcVersion::kV2; + } + + if (auto Param = Params.GetValue("compactblockthreshold"); Param.empty() == false) + { + if (auto Value = ParseInt(Param)) + { + GcParams.CompactBlockUsageThresholdPercent = Value.value(); + } + } + + if (auto Param = Params.GetValue("verbose"); Param.empty() == false) + { + GcParams.Verbose = Param == "true"sv; + } + + if (auto Param = Params.GetValue("singlethreaded"); Param.empty() == false) + { + GcParams.SingleThreaded = Param == "true"sv; + } + + if (auto Param = Params.GetValue("referencehashlow"); Param.empty() == false) + { + GcParams.AttachmentRangeMin = IoHash::FromHexString(Param); + } + + if (auto Param = Params.GetValue("referencehashhigh"); Param.empty() == false) + { + GcParams.AttachmentRangeMax = IoHash::FromHexString(Param); + } + + if (auto Param = Params.GetValue("storecacheattachmentmetadata"); Param.empty() == false) + { + GcParams.StoreCacheAttachmentMetaData = Param == "true"sv; + } + + if (auto Param = Params.GetValue("storeprojectattachmentmetadata"); Param.empty() == false) + { + GcParams.StoreProjectAttachmentMetaData = Param == "true"sv; + } + + if (auto Param = Params.GetValue("enablevalidation"); Param.empty() == false) + { + GcParams.EnableValidation = Param == "true"sv; + } + + const bool Started = m_GcScheduler.TriggerGc(GcParams); + + CbObjectWriter Response; + Response << "Status"sv << (Started ? "Started"sv : "Running"sv); + HttpReq.WriteResponse(HttpResponseCode::Accepted, Response.Save()); + }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "gc-stop", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + if (m_GcScheduler.CancelGC()) + { + return HttpReq.WriteResponse(HttpResponseCode::Accepted); + } + HttpReq.WriteResponse(HttpResponseCode::OK); + }, + HttpVerb::kPost); + +#if ZEN_USE_MIMALLOC + m_Router.RegisterRoute( + "mi_collect", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + const HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); + + bool Force = false; + + if (auto Param = Params.GetValue("force"); Param.empty() == false) + { + Force = (Param == "true"sv); + } + + ExtendableStringBuilder<256> MiStats; + ExtendableStringBuilder<256> MiStatsAfter; + + auto MiOutputFun = [](const char* msg, void* arg) { + StringBuilderBase* StarsSb = reinterpret_cast(arg); + StarsSb->AppendAscii(msg); + }; + + mi_stats_print_out(MiOutputFun, static_cast(&MiStats)); + mi_collect(Force); + mi_stats_print_out(MiOutputFun, static_cast(&MiStatsAfter)); + + CbObjectWriter Response; + Response << "force"sv << Force; + Response << "stats_before"sv << MiStats; + Response << "stats_after"sv << MiStatsAfter; + HttpReq.WriteResponse(HttpResponseCode::OK, Response.Save()); + }, + HttpVerb::kPost); +#endif + + m_Router.RegisterRoute( + "scrub", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + const HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); + + GcScheduler::TriggerScrubParams ScrubParams; + ScrubParams.MaxTimeslice = std::chrono::seconds(100); + + if (auto Param = Params.GetValue("skipdelete"); Param.empty() == false) + { + ScrubParams.SkipDelete = (Param == "true"sv); + } + + if (auto Param = Params.GetValue("skipgc"); Param.empty() == false) + { + ScrubParams.SkipGc = (Param == "true"sv); + } + + if (auto Param = Params.GetValue("skipcid"); Param.empty() == false) + { + ScrubParams.SkipCas = (Param == "true"sv); + } + + m_GcScheduler.TriggerScrub(ScrubParams); + + CbObjectWriter Response; + Response << "ok"sv << true; + Response << "skip_delete" << ScrubParams.SkipDelete; + Response << "skip_gc" << ScrubParams.SkipGc; + Response << "skip_cas" << ScrubParams.SkipCas; + Response << "max_time" << TimeSpan(0, 0, gsl::narrow(ScrubParams.MaxTimeslice.count())); + HttpReq.WriteResponse(HttpResponseCode::OK, Response.Save()); + }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "", + [](HttpRouterRequest& Req) { + CbObject Payload = Req.ServerRequest().ReadPayloadObject(); + + CbObjectWriter Obj; + Obj.AddBool("ok", true); + Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); + }, + HttpVerb::kPost); + +#if ZEN_WITH_TRACE + m_Router.RegisterRoute( + "trace", + [this](HttpRouterRequest& Req) { + bool Enabled = IsTracing(); + return Req.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Enabled ? "enabled" : "disabled"); + }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "trace/start", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + const HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); + TraceOptions TraceOptions; + + if (!IsTracing()) + { + TraceInit("zenserver"); + } + + if (auto Channels = Params.GetValue("channels"); Channels.empty() == false) + { + TraceOptions.Channels = Channels; + } + + if (auto File = Params.GetValue("file"); File.empty() == false) + { + TraceOptions.File = File; + } + else if (auto Host = Params.GetValue("host"); Host.empty() == false) + { + TraceOptions.Host = Host; + } + else + { + return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Invalid trace type, use `file` or `host`"sv); + } + + TraceConfigure(TraceOptions); + + return Req.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kText, "Tracing started"); + }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "trace/stop", + [this](HttpRouterRequest& Req) { + if (!IsTracing()) + { + return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Tracing is not enabled"sv); + } + if (TraceStop()) + { + return Req.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kText, "Tracing stopped"); + } + else + { + return Req.ServerRequest().WriteResponse(HttpResponseCode::InternalServerError, + HttpContentType::kText, + "Failed stopping trace"); + } + }, + HttpVerb::kPost); +#endif // ZEN_WITH_TRACE + + m_Router.RegisterRoute( + "info", + [this](HttpRouterRequest& Req) { + CbObjectWriter Obj; + + Obj << "root" << m_ServerOptions.SystemRootDir.generic_wstring(); + Obj << "install" << (m_ServerOptions.SystemRootDir / "Install").generic_wstring(); + + Obj.BeginObject("primary"); + Obj << "data" << m_ServerOptions.DataDir.generic_wstring(); + + try + { + auto Stats = GetStatsForStateDirectory(m_ServerOptions.DataDir); + + auto EmitStats = [&](std::string_view Tag, const DirStats& Stats) { + Obj.BeginObject(Tag); + Obj << "bytes" << Stats.ByteCount; + Obj << "files" << Stats.FileCount; + Obj << "dirs" << Stats.DirCount; + Obj.EndObject(); + }; + + EmitStats("cache", Stats.CacheStats); + EmitStats("cas", Stats.CasStats); + EmitStats("project", Stats.ProjectStats); + } + catch (const std::exception& Ex) + { + ZEN_WARN("exception in disk stats gathering for '{}': {}", m_ServerOptions.DataDir, Ex.what()); + } + Obj.EndObject(); + + try + { + std::vector Manifests = ReadAllCentralManifests(m_ServerOptions.SystemRootDir); + + Obj.BeginArray("known"); + + for (const auto& Manifest : Manifests) + { + Obj.AddObject(Manifest); + } + + Obj.EndArray(); + } + catch (const std::exception& Ex) + { + ZEN_WARN("exception in state gathering for '{}': {}", m_ServerOptions.SystemRootDir, Ex.what()); + } + Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); + }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "logs", + [this](HttpRouterRequest& Req) { + CbObjectWriter Obj; + auto LogLevel = logging::level::ToStringView(logging::GetLogLevel()); + Obj.AddString("loglevel", std::string_view(LogLevel.data(), LogLevel.size())); + Obj.AddString("Logfile", PathToUtf8(m_LogPaths.AbsLogPath)); + Obj.BeginObject("cache"); + if (m_CacheStore) + { + const ZenCacheStore::Configuration& CacheConfig = m_CacheStore->GetConfiguration(); + Obj.AddString("Logfile", PathToUtf8(m_LogPaths.CacheLogPath)); + Obj.AddBool("Write", CacheConfig.Logging.EnableWriteLog); + Obj.AddBool("Access", CacheConfig.Logging.EnableAccessLog); + } + Obj.EndObject(); + Obj.BeginObject("http"); + { + Obj.AddString("Logfile", PathToUtf8(m_LogPaths.HttpLogPath)); + } + Obj.EndObject(); + Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); + }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "logs", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + const HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); + bool SetCacheLogConfig = false; + ExtendableStringBuilder<256> StringBuilder; + if (m_CacheStore) + { + ZenCacheStore::Configuration::LogConfig LoggingConfig = m_CacheStore->GetConfiguration().Logging; + if (std::string Param(Params.GetValue("cacheenablewritelog")); Param.empty() == false) + { + LoggingConfig.EnableWriteLog = StrCaseCompare(Param.c_str(), "true") == 0; + SetCacheLogConfig = true; + } + if (std::string Param(Params.GetValue("cacheenableaccesslog")); Param.empty() == false) + { + LoggingConfig.EnableAccessLog = StrCaseCompare(Param.c_str(), "true") == 0; + SetCacheLogConfig = true; + } + if (SetCacheLogConfig) + { + m_CacheStore->SetLoggingConfig(LoggingConfig); + StringBuilder.Append(fmt::format("cache write log: {}, cache access log: {}", + LoggingConfig.EnableWriteLog ? "true" : "false", + LoggingConfig.EnableAccessLog ? "true" : "false")); + } + } + if (std::string Param(Params.GetValue("loglevel")); Param.empty() == false) + { + logging::level::LogLevel NewLevel = logging::level::ParseLogLevelString(Param); + std::string_view LogLevel = logging::level::ToStringView(NewLevel); + if (LogLevel != Param) + { + return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid log level '{}'", Param)); + } + logging::SetLogLevel(NewLevel); + if (StringBuilder.Size() > 0) + { + StringBuilder.Append(", "); + } + StringBuilder.Append("loglevel: "); + StringBuilder.Append(Param); + } + return Req.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kText, StringBuilder.ToView()); + }, + HttpVerb::kPost); + m_Router.RegisterRoute( + "flush", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + m_FlushFunction(); + HttpReq.WriteResponse(HttpResponseCode::OK); + }, + HttpVerb::kPost); +} + +HttpAdminService::~HttpAdminService() +{ +} + +const char* +HttpAdminService::BaseUri() const +{ + return "/admin/"; +} + +void +HttpAdminService::HandleRequest(zen::HttpServerRequest& Request) +{ + m_Router.HandleRequest(Request); +} + +} // namespace zen diff --git a/src/zenserver/storage/admin/admin.h b/src/zenserver/storage/admin/admin.h new file mode 100644 index 000000000..9a49f5120 --- /dev/null +++ b/src/zenserver/storage/admin/admin.h @@ -0,0 +1,46 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include +#include + +namespace zen { + +class GcScheduler; +class JobQueue; +class ZenCacheStore; +struct ZenServerOptions; + +class HttpAdminService : public zen::HttpService +{ +public: + struct LogPaths + { + std::filesystem::path AbsLogPath; + std::filesystem::path HttpLogPath; + std::filesystem::path CacheLogPath; + }; + HttpAdminService(GcScheduler& Scheduler, + JobQueue& BackgroundJobQueue, + ZenCacheStore* CacheStore, + std::function&& FlushFunction, + const LogPaths& LogPaths, + const ZenServerOptions& ServerOptions); + ~HttpAdminService(); + + virtual const char* BaseUri() const override; + virtual void HandleRequest(zen::HttpServerRequest& Request) override; + +private: + HttpRequestRouter m_Router; + GcScheduler& m_GcScheduler; + JobQueue& m_BackgroundJobQueue; + ZenCacheStore* m_CacheStore; + std::function m_FlushFunction; + LogPaths m_LogPaths; + const ZenServerOptions& m_ServerOptions; +}; + +} // namespace zen diff --git a/src/zenserver/storage/buildstore/httpbuildstore.cpp b/src/zenserver/storage/buildstore/httpbuildstore.cpp new file mode 100644 index 000000000..bce993f17 --- /dev/null +++ b/src/zenserver/storage/buildstore/httpbuildstore.cpp @@ -0,0 +1,561 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "httpbuildstore.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace zen { +using namespace std::literals; + +ZEN_DEFINE_LOG_CATEGORY_STATIC(LogBuilds, "builds"sv); + +HttpBuildStoreService::HttpBuildStoreService(HttpStatusService& StatusService, HttpStatsService& StatsService, BuildStore& Store) +: m_Log(logging::Get("builds")) +, m_StatusService(StatusService) +, m_StatsService(StatsService) +, m_BuildStore(Store) +{ + Initialize(); + + m_StatusService.RegisterHandler("builds", *this); + m_StatsService.RegisterHandler("builds", *this); +} + +HttpBuildStoreService::~HttpBuildStoreService() +{ + m_StatsService.UnregisterHandler("builds", *this); + m_StatusService.UnregisterHandler("builds", *this); +} + +const char* +HttpBuildStoreService::BaseUri() const +{ + return "/builds/"; +} + +void +HttpBuildStoreService::Initialize() +{ + ZEN_LOG_INFO(LogBuilds, "Initializing Builds Service"); + + m_Router.AddPattern("namespace", "([[:alnum:]\\-_.]+)"); + m_Router.AddPattern("bucket", "([[:alnum:]\\-_.]+)"); + m_Router.AddPattern("buildid", "([[:xdigit:]]{24})"); + m_Router.AddPattern("hash", "([[:xdigit:]]{40})"); + + m_Router.RegisterRoute( + "{namespace}/{bucket}/{buildid}/blobs/{hash}", + [this](HttpRouterRequest& Req) { PutBlobRequest(Req); }, + HttpVerb::kPut); + + m_Router.RegisterRoute( + "{namespace}/{bucket}/{buildid}/blobs/{hash}", + [this](HttpRouterRequest& Req) { GetBlobRequest(Req); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "{namespace}/{bucket}/{buildid}/blobs/putBlobMetadata", + [this](HttpRouterRequest& Req) { PutMetadataRequest(Req); }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "{namespace}/{bucket}/{buildid}/blobs/getBlobMetadata", + [this](HttpRouterRequest& Req) { GetMetadatasRequest(Req); }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "{namespace}/{bucket}/{buildid}/blobs/exists", + [this](HttpRouterRequest& Req) { BlobsExistsRequest(Req); }, + HttpVerb::kPost); +} + +void +HttpBuildStoreService::HandleRequest(zen::HttpServerRequest& Request) +{ + ZEN_TRACE_CPU("HttpBuildStoreService::HandleRequest"); + metrics::OperationTiming::Scope $(m_HttpRequests); + + m_BuildStoreStats.RequestCount++; + if (m_Router.HandleRequest(Request) == false) + { + ZEN_LOG_WARN(LogBuilds, "No route found for {0}", Request.RelativeUri()); + return Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Not found"sv); + } +} + +void +HttpBuildStoreService::PutBlobRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("HttpBuildStoreService::PutBlobRequest"); + HttpServerRequest& ServerRequest = Req.ServerRequest(); + const std::string_view Namespace = Req.GetCapture(1); + const std::string_view Bucket = Req.GetCapture(2); + const std::string_view BuildId = Req.GetCapture(3); + const std::string_view Hash = Req.GetCapture(4); + ZEN_UNUSED(Namespace, Bucket, BuildId); + IoHash BlobHash; + if (!IoHash::TryParse(Hash, BlobHash)) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid blob hash '{}'", Hash)); + } + m_BuildStoreStats.BlobWriteCount++; + IoBuffer Payload = ServerRequest.ReadPayload(); + if (!Payload) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Payload blob {} is empty", Hash)); + } + if (Payload.GetContentType() != HttpContentType::kCompressedBinary) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse( + HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Payload blob {} content type {} is invalid", Hash, ToString(Payload.GetContentType()))); + } + m_BuildStore.PutBlob(BlobHash, ServerRequest.ReadPayload()); + // ZEN_INFO("Stored blob {}. Size: {}", BlobHash, ServerRequest.ReadPayload().GetSize()); + return ServerRequest.WriteResponse(HttpResponseCode::OK); +} + +void +HttpBuildStoreService::GetBlobRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("HttpBuildStoreService::GetBlobRequest"); + HttpServerRequest& ServerRequest = Req.ServerRequest(); + std::string_view Namespace = Req.GetCapture(1); + std::string_view Bucket = Req.GetCapture(2); + std::string_view BuildId = Req.GetCapture(3); + std::string_view Hash = Req.GetCapture(4); + ZEN_UNUSED(Namespace, Bucket, BuildId); + IoHash BlobHash; + if (!IoHash::TryParse(Hash, BlobHash)) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid blob hash '{}'", Hash)); + } + zen::HttpRanges Ranges; + bool HasRange = ServerRequest.TryGetRanges(Ranges); + if (Ranges.size() > 1) + { + // Only a single range is supported + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Multiple ranges in blob request is not supported"); + } + + m_BuildStoreStats.BlobReadCount++; + IoBuffer Blob = m_BuildStore.GetBlob(BlobHash); + if (!Blob) + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Blob with hash '{}' could not be found", Hash)); + } + // ZEN_INFO("Fetched blob {}. Size: {}", BlobHash, Blob.GetSize()); + m_BuildStoreStats.BlobHitCount++; + if (HasRange) + { + const HttpRange& Range = Ranges.front(); + const uint64_t BlobSize = Blob.GetSize(); + const uint64_t MaxBlobSize = Range.Start < BlobSize ? Range.Start - BlobSize : 0; + const uint64_t RangeSize = Min(Range.End - Range.Start + 1, MaxBlobSize); + if (Range.Start + RangeSize > BlobSize) + { + return ServerRequest.WriteResponse(HttpResponseCode::NoContent); + } + Blob = IoBuffer(Blob, Range.Start, RangeSize); + return ServerRequest.WriteResponse(HttpResponseCode::OK, ZenContentType::kBinary, Blob); + } + else + { + return ServerRequest.WriteResponse(HttpResponseCode::OK, Blob.GetContentType(), Blob); + } +} + +void +HttpBuildStoreService::PutMetadataRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("HttpBuildStoreService::PutMetadataRequest"); + HttpServerRequest& ServerRequest = Req.ServerRequest(); + std::string_view Namespace = Req.GetCapture(1); + std::string_view Bucket = Req.GetCapture(2); + std::string_view BuildId = Req.GetCapture(3); + + IoBuffer MetaPayload = ServerRequest.ReadPayload(); + if (MetaPayload.GetContentType() != ZenContentType::kCbPackage) + { + throw std::runtime_error(fmt::format("PutMetadataRequest payload has unexpected payload type '{}', expected '{}'", + ToString(MetaPayload.GetContentType()), + ToString(ZenContentType::kCbPackage))); + } + CbPackage Message = ParsePackageMessage(MetaPayload); + + CbObjectView MessageObject = Message.GetObject(); + if (!MessageObject) + { + throw std::runtime_error("PutMetadataRequest payload object is missing"); + } + CbArrayView BlobsArray = MessageObject["blobHashes"sv].AsArrayView(); + CbArrayView MetadataArray = MessageObject["metadatas"sv].AsArrayView(); + + const uint64_t BlobCount = BlobsArray.Num(); + if (BlobCount == 0) + { + throw std::runtime_error("PutMetadataRequest blobs array is empty"); + } + if (BlobCount != MetadataArray.Num()) + { + throw std::runtime_error( + fmt::format("PutMetadataRequest metadata array size {} does not match blobs array size {}", MetadataArray.Num(), BlobCount)); + } + + std::vector BlobHashes; + std::vector MetadataPayloads; + + BlobHashes.reserve(BlobCount); + MetadataPayloads.reserve(BlobCount); + + auto BlobsArrayIt = begin(BlobsArray); + auto MetadataArrayIt = begin(MetadataArray); + while (BlobsArrayIt != end(BlobsArray)) + { + const IoHash BlobHash = (*BlobsArrayIt).AsHash(); + const IoHash MetadataHash = (*MetadataArrayIt).AsAttachment(); + + const CbAttachment* Attachment = Message.FindAttachment(MetadataHash); + if (Attachment == nullptr) + { + throw std::runtime_error(fmt::format("Blob metadata attachment {} is missing", MetadataHash)); + } + BlobHashes.push_back(BlobHash); + if (Attachment->IsObject()) + { + MetadataPayloads.push_back(Attachment->AsObject().GetBuffer().MakeOwned().AsIoBuffer()); + MetadataPayloads.back().SetContentType(ZenContentType::kCbObject); + } + else if (Attachment->IsCompressedBinary()) + { + MetadataPayloads.push_back(Attachment->AsCompressedBinary().GetCompressed().Flatten().AsIoBuffer()); + MetadataPayloads.back().SetContentType(ZenContentType::kCompressedBinary); + } + else + { + ZEN_ASSERT(Attachment->IsBinary()); + MetadataPayloads.push_back(Attachment->AsBinary().AsIoBuffer()); + MetadataPayloads.back().SetContentType(ZenContentType::kBinary); + } + + BlobsArrayIt++; + MetadataArrayIt++; + } + m_BuildStore.PutMetadatas(BlobHashes, MetadataPayloads, &GetSmallWorkerPool(EWorkloadType::Burst)); + return ServerRequest.WriteResponse(HttpResponseCode::OK); +} + +void +HttpBuildStoreService::GetMetadatasRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("HttpBuildStoreService::GetMetadatasRequest"); + HttpServerRequest& ServerRequest = Req.ServerRequest(); + std::string_view Namespace = Req.GetCapture(1); + std::string_view Bucket = Req.GetCapture(2); + std::string_view BuildId = Req.GetCapture(3); + ZEN_UNUSED(Namespace, Bucket, BuildId); + IoBuffer RequestPayload = ServerRequest.ReadPayload(); + if (!RequestPayload) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Expected compact binary body for metadata request, body is missing"); + } + if (RequestPayload.GetContentType() != HttpContentType::kCbObject) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse( + HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Expected compact binary body for metadata request, got {}", ToString(RequestPayload.GetContentType()))); + } + if (CbValidateError ValidateError = ValidateCompactBinary(RequestPayload.GetView(), CbValidateMode::Default); + ValidateError != CbValidateError::None) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse( + HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Compact binary body for metadata request is not valid, reason: {}", ToString(ValidateError))); + } + CbObject RequestObject = LoadCompactBinaryObject(RequestPayload); + CbArrayView BlobsArray = RequestObject["blobHashes"sv].AsArrayView(); + if (!BlobsArray) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Compact binary body for metadata request is missing 'blobHashes' array"); + } + const uint64_t BlobCount = BlobsArray.Num(); + + std::vector BlobRawHashes; + BlobRawHashes.reserve(BlobCount); + for (CbFieldView BlockHashView : BlobsArray) + { + BlobRawHashes.push_back(BlockHashView.AsHash()); + if (BlobRawHashes.back() == IoHash::Zero) + { + const uint8_t Type = (uint8_t)BlockHashView.GetValue().GetType(); + return ServerRequest.WriteResponse( + HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Compact binary body for metadata 'blobHashes' array contains invalid field type: {}", Type)); + } + } + m_BuildStoreStats.BlobMetaReadCount += BlobRawHashes.size(); + std::vector BlockMetadatas = m_BuildStore.GetMetadatas(BlobRawHashes, &GetSmallWorkerPool(EWorkloadType::Burst)); + + CbPackage ResponsePackage; + std::vector Attachments; + tsl::robin_set AttachmentHashes; + Attachments.reserve(BlobCount); + AttachmentHashes.reserve(BlobCount); + { + CbObjectWriter ResponseWriter; + + ResponseWriter.BeginArray("blobHashes"); + for (size_t BlockHashIndex = 0; BlockHashIndex < BlobRawHashes.size(); BlockHashIndex++) + { + if (BlockMetadatas[BlockHashIndex]) + { + const IoHash& BlockHash = BlobRawHashes[BlockHashIndex]; + ResponseWriter.AddHash(BlockHash); + } + } + ResponseWriter.EndArray(); // blobHashes + + ResponseWriter.BeginArray("metadatas"); + + for (size_t BlockHashIndex = 0; BlockHashIndex < BlobRawHashes.size(); BlockHashIndex++) + { + if (IoBuffer Metadata = BlockMetadatas[BlockHashIndex]; Metadata) + { + switch (Metadata.GetContentType()) + { + case ZenContentType::kCbObject: + { + CbObject Object = CbObject(SharedBuffer(std::move(Metadata)).MakeOwned()); + const IoHash ObjectHash = Object.GetHash(); + ResponseWriter.AddBinaryAttachment(ObjectHash); + if (!AttachmentHashes.contains(ObjectHash)) + { + Attachments.push_back(CbAttachment(Object, ObjectHash)); + AttachmentHashes.insert(ObjectHash); + } + } + break; + case ZenContentType::kCompressedBinary: + { + IoHash RawHash; + uint64_t _; + CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(std::move(Metadata)), RawHash, _); + ResponseWriter.AddBinaryAttachment(RawHash); + if (!AttachmentHashes.contains(RawHash)) + { + Attachments.push_back(CbAttachment(Compressed, RawHash)); + AttachmentHashes.insert(RawHash); + } + } + break; + default: + { + const IoHash RawHash = IoHash::HashBuffer(Metadata); + ResponseWriter.AddBinaryAttachment(RawHash); + if (!AttachmentHashes.contains(RawHash)) + { + Attachments.push_back(CbAttachment(SharedBuffer(Metadata), RawHash)); + AttachmentHashes.insert(RawHash); + } + } + break; + } + } + } + + ResponseWriter.EndArray(); // metadatas + + ResponsePackage.SetObject(ResponseWriter.Save()); + } + ResponsePackage.AddAttachments(Attachments); + + CompositeBuffer RpcResponseBuffer = FormatPackageMessageBuffer(ResponsePackage); + ServerRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, RpcResponseBuffer); +} + +void +HttpBuildStoreService::BlobsExistsRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("HttpBuildStoreService::BlobsExistsRequest"); + HttpServerRequest& ServerRequest = Req.ServerRequest(); + std::string_view Namespace = Req.GetCapture(1); + std::string_view Bucket = Req.GetCapture(2); + std::string_view BuildId = Req.GetCapture(3); + ZEN_UNUSED(Namespace, Bucket, BuildId); + IoBuffer RequestPayload = ServerRequest.ReadPayload(); + if (!RequestPayload) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Expected compact binary body for blob exists request, body is missing"); + } + if (RequestPayload.GetContentType() != HttpContentType::kCbObject) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse( + HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Expected compact binary body for blob exists request, got {}", ToString(RequestPayload.GetContentType()))); + } + if (CbValidateError ValidateError = ValidateCompactBinary(RequestPayload.GetView(), CbValidateMode::Default); + ValidateError != CbValidateError::None) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse( + HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Compact binary body for blob exists request is not valid, reason: {}", ToString(ValidateError))); + } + CbObject RequestObject = LoadCompactBinaryObject(RequestPayload); + CbArrayView BlobsArray = RequestObject["blobHashes"sv].AsArrayView(); + if (!BlobsArray) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Compact binary body for blob exists request is missing 'blobHashes' array"); + } + + std::vector BlobRawHashes; + BlobRawHashes.reserve(BlobsArray.Num()); + for (CbFieldView BlockHashView : BlobsArray) + { + BlobRawHashes.push_back(BlockHashView.AsHash()); + if (BlobRawHashes.back() == IoHash::Zero) + { + const uint8_t Type = (uint8_t)BlockHashView.GetValue().GetType(); + return ServerRequest.WriteResponse( + HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Compact binary body for blob exists request 'blobHashes' array contains invalid field type: {}", Type)); + } + } + + m_BuildStoreStats.BlobExistsCount += BlobRawHashes.size(); + std::vector BlobsExists = m_BuildStore.BlobsExists(BlobRawHashes); + CbObjectWriter ResponseWriter(9 * BlobsExists.size()); + ResponseWriter.BeginArray("blobExists"sv); + for (const BuildStore::BlobExistsResult& BlobExists : BlobsExists) + { + ResponseWriter.AddBool(BlobExists.HasBody); + if (BlobExists.HasBody) + { + m_BuildStoreStats.BlobExistsBodyHitCount++; + } + } + ResponseWriter.EndArray(); // blobExist + ResponseWriter.BeginArray("metadataExists"sv); + for (const BuildStore::BlobExistsResult& BlobExists : BlobsExists) + { + ResponseWriter.AddBool(BlobExists.HasMetadata); + if (BlobExists.HasMetadata) + { + m_BuildStoreStats.BlobExistsMetaHitCount++; + } + } + ResponseWriter.EndArray(); // metadataExists + CbObject ResponseObject = ResponseWriter.Save(); + return ServerRequest.WriteResponse(HttpResponseCode::OK, ResponseObject); +} + +void +HttpBuildStoreService::HandleStatsRequest(HttpServerRequest& Request) +{ + ZEN_TRACE_CPU("HttpBuildStoreService::Stats"); + CbObjectWriter Cbo; + + EmitSnapshot("requests", m_HttpRequests, Cbo); + + Cbo.BeginObject("builds"); + { + Cbo.BeginObject("blobs"); + { + Cbo << "readcount" << m_BuildStoreStats.BlobReadCount << "writecount" << m_BuildStoreStats.BlobWriteCount << "hitcount" + << m_BuildStoreStats.BlobHitCount; + } + Cbo.EndObject(); + + Cbo.BeginObject("metadata"); + { + Cbo << "readcount" << m_BuildStoreStats.BlobMetaReadCount << "writecount" << m_BuildStoreStats.BlobMetaWriteCount << "hitcount" + << m_BuildStoreStats.BlobMetaHitCount; + } + Cbo.EndObject(); + + Cbo << "requestcount" << m_BuildStoreStats.RequestCount; + Cbo << "badrequestcount" << m_BuildStoreStats.BadRequestCount; + } + Cbo.EndObject(); + + Cbo.BeginObject("size"); + { + BuildStore::StorageStats StorageStats = m_BuildStore.GetStorageStats(); + + Cbo << "count" << StorageStats.EntryCount; + Cbo << "bytes" << StorageStats.BlobBytes + StorageStats.MetadataByteCount; + Cbo.BeginObject("blobs"); + { + Cbo << "count" << StorageStats.BlobCount; + Cbo << "bytes" << StorageStats.BlobBytes; + } + Cbo.EndObject(); // blobs + + Cbo.BeginObject("metadata"); + { + Cbo << "count" << StorageStats.MetadataCount; + Cbo << "bytes" << StorageStats.MetadataByteCount; + } + Cbo.EndObject(); // metadata + } + Cbo.EndObject(); // size + + return Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + +void +HttpBuildStoreService::HandleStatusRequest(HttpServerRequest& Request) +{ + ZEN_TRACE_CPU("HttpBuildStoreService::Status"); + CbObjectWriter Cbo; + Cbo << "ok" << true; + Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + +} // namespace zen diff --git a/src/zenserver/storage/buildstore/httpbuildstore.h b/src/zenserver/storage/buildstore/httpbuildstore.h new file mode 100644 index 000000000..50cb5db12 --- /dev/null +++ b/src/zenserver/storage/buildstore/httpbuildstore.h @@ -0,0 +1,68 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include +#include +#include + +#include + +namespace zen { + +class BuildStore; + +class HttpBuildStoreService final : public zen::HttpService, public IHttpStatusProvider, public IHttpStatsProvider +{ +public: + HttpBuildStoreService(HttpStatusService& StatusService, HttpStatsService& StatsService, BuildStore& Store); + virtual ~HttpBuildStoreService(); + + virtual const char* BaseUri() const override; + virtual void HandleRequest(zen::HttpServerRequest& Request) override; + + virtual void HandleStatsRequest(HttpServerRequest& Request) override; + virtual void HandleStatusRequest(HttpServerRequest& Request) override; + +private: + struct BuildStoreStats + { + std::atomic_uint64_t BlobReadCount{}; + std::atomic_uint64_t BlobHitCount{}; + std::atomic_uint64_t BlobWriteCount{}; + std::atomic_uint64_t BlobMetaReadCount{}; + std::atomic_uint64_t BlobMetaHitCount{}; + std::atomic_uint64_t BlobMetaWriteCount{}; + std::atomic_uint64_t BlobExistsCount{}; + std::atomic_uint64_t BlobExistsBodyHitCount{}; + std::atomic_uint64_t BlobExistsMetaHitCount{}; + std::atomic_uint64_t RequestCount{}; + std::atomic_uint64_t BadRequestCount{}; + }; + + void Initialize(); + + inline LoggerRef Log() { return m_Log; } + + LoggerRef m_Log; + + void PutBlobRequest(HttpRouterRequest& Req); + void GetBlobRequest(HttpRouterRequest& Req); + + void PutMetadataRequest(HttpRouterRequest& Req); + void GetMetadatasRequest(HttpRouterRequest& Req); + + void BlobsExistsRequest(HttpRouterRequest& Req); + + HttpRequestRouter m_Router; + + HttpStatusService& m_StatusService; + HttpStatsService& m_StatsService; + + BuildStore& m_BuildStore; + BuildStoreStats m_BuildStoreStats; + metrics::OperationTiming m_HttpRequests; +}; + +} // namespace zen diff --git a/src/zenserver/storage/cache/httpstructuredcache.cpp b/src/zenserver/storage/cache/httpstructuredcache.cpp new file mode 100644 index 000000000..ece1d7a46 --- /dev/null +++ b/src/zenserver/storage/cache/httpstructuredcache.cpp @@ -0,0 +1,2052 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "httpstructuredcache.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "storage/upstream/upstreamcache.h" +#include "storage/upstream/zen.h" +#include "zenstore/cidstore.h" +#include "zenstore/scrubcontext.h" + +#include +#include +#include +#include +#include + +#include + +namespace zen { + +const FLLMTag& +GetCacheHttpTag() +{ + static FLLMTag CacheHttpTag("http", FLLMTag("cache")); + + return CacheHttpTag; +} + +extern const FLLMTag& GetCacheRpcTag(); + +using namespace std::literals; + +////////////////////////////////////////////////////////////////////////// + +CachePolicy +ParseCachePolicy(const HttpServerRequest::QueryParams& QueryParams) +{ + std::string_view PolicyText = QueryParams.GetValue("Policy"sv); + return !PolicyText.empty() ? zen::ParseCachePolicy(PolicyText) : CachePolicy::Default; +} + +namespace { + static constinit std::string_view HttpZCacheRPCPrefix = "$rpc"sv; + static constinit std::string_view HttpZCacheUtilStartRecording = "exec$/start-recording"sv; + static constinit std::string_view HttpZCacheUtilStopRecording = "exec$/stop-recording"sv; + static constinit std::string_view HttpZCacheUtilReplayRecording = "exec$/replay-recording"sv; + static constinit std::string_view HttpZCacheDetailsPrefix = "details$"sv; +} // namespace + +////////////////////////////////////////////////////////////////////////// + +HttpStructuredCacheService::HttpStructuredCacheService(ZenCacheStore& InCacheStore, + CidStore& InCidStore, + HttpStatsService& StatsService, + HttpStatusService& StatusService, + UpstreamCache& UpstreamCache, + const DiskWriteBlocker* InDiskWriteBlocker, + OpenProcessCache& InOpenProcessCache) +: m_Log(logging::Get("cache")) +, m_CacheStore(InCacheStore) +, m_StatsService(StatsService) +, m_StatusService(StatusService) +, m_CidStore(InCidStore) +, m_UpstreamCache(UpstreamCache) +, m_DiskWriteBlocker(InDiskWriteBlocker) +, m_OpenProcessCache(InOpenProcessCache) +, m_RpcHandler(m_Log, m_CacheStats, UpstreamCache, InCacheStore, InCidStore, InDiskWriteBlocker) +{ + m_StatsService.RegisterHandler("z$", *this); + m_StatusService.RegisterHandler("z$", *this); +} + +HttpStructuredCacheService::~HttpStructuredCacheService() +{ + ZEN_INFO("closing structured cache"); + { + RwLock::ExclusiveLockScope _(m_RequestRecordingLock); + m_RequestRecordingEnabled.store(false); + m_RequestRecorder.reset(); + } + + m_StatsService.UnregisterHandler("z$", *this); + m_StatusService.UnregisterHandler("z$", *this); +} + +const char* +HttpStructuredCacheService::BaseUri() const +{ + return "/z$/"; +} + +void +HttpStructuredCacheService::Flush() +{ + m_CacheStore.Flush(); +} + +void +HttpStructuredCacheService::HandleDetailsRequest(HttpServerRequest& Request) +{ + std::string_view Key = Request.RelativeUri(); + std::vector Tokens; + uint32_t TokenCount = ForEachStrTok(Key, '/', [&Tokens](std::string_view Token) { + Tokens.push_back(std::string(Token)); + return true; + }); + std::string FilterNamespace; + std::string FilterBucket; + std::string FilterValue; + switch (TokenCount) + { + case 1: + break; + case 2: + { + FilterNamespace = Tokens[1]; + if (FilterNamespace.empty()) + { + m_CacheStats.BadRequestCount++; + return Request.WriteResponse(HttpResponseCode::BadRequest); // invalid URL + } + } + break; + case 3: + { + FilterNamespace = Tokens[1]; + if (FilterNamespace.empty()) + { + m_CacheStats.BadRequestCount++; + return Request.WriteResponse(HttpResponseCode::BadRequest); // invalid URL + } + FilterBucket = Tokens[2]; + if (FilterBucket.empty()) + { + m_CacheStats.BadRequestCount++; + return Request.WriteResponse(HttpResponseCode::BadRequest); // invalid URL + } + } + break; + case 4: + { + FilterNamespace = Tokens[1]; + if (FilterNamespace.empty()) + { + m_CacheStats.BadRequestCount++; + return Request.WriteResponse(HttpResponseCode::BadRequest); // invalid URL + } + FilterBucket = Tokens[2]; + if (FilterBucket.empty()) + { + m_CacheStats.BadRequestCount++; + return Request.WriteResponse(HttpResponseCode::BadRequest); // invalid URL + } + FilterValue = Tokens[3]; + if (FilterValue.empty()) + { + m_CacheStats.BadRequestCount++; + return Request.WriteResponse(HttpResponseCode::BadRequest); // invalid URL + } + } + break; + default: + m_CacheStats.BadRequestCount++; + return Request.WriteResponse(HttpResponseCode::BadRequest); // invalid URL + } + + HttpServerRequest::QueryParams Params = Request.GetQueryParams(); + bool CSV = Params.GetValue("csv") == "true"; + bool Details = Params.GetValue("details") == "true"; + bool AttachmentDetails = Params.GetValue("attachmentdetails") == "true"; + + std::chrono::seconds NowSeconds = std::chrono::duration_cast(GcClock::Now().time_since_epoch()); + CacheValueDetails ValueDetails = m_CacheStore.GetValueDetails(FilterNamespace, FilterBucket, FilterValue); + + if (CSV) + { + ExtendableStringBuilder<4096> CSVWriter; + if (AttachmentDetails) + { + CSVWriter << "Namespace, Bucket, Key, Cid, Size"; + } + else if (Details) + { + CSVWriter << "Namespace, Bucket, Key, Size, RawSize, RawHash, ContentType, Age, AttachmentsCount, AttachmentsSize"; + } + else + { + CSVWriter << "Namespace, Bucket, Key"; + } + for (const auto& NamespaceIt : ValueDetails.Namespaces) + { + const std::string& Namespace = NamespaceIt.first; + for (const auto& BucketIt : NamespaceIt.second.Buckets) + { + const std::string& Bucket = BucketIt.first; + for (const auto& ValueIt : BucketIt.second.Values) + { + if (AttachmentDetails) + { + for (const IoHash& Hash : ValueIt.second.Attachments) + { + IoBuffer Payload = m_CidStore.FindChunkByCid(Hash); + CSVWriter << "\r\n" + << Namespace << "," << Bucket << "," << ValueIt.first.ToHexString() << ", " << Hash.ToHexString() + << ", " << gsl::narrow(Payload.GetSize()); + } + } + else if (Details) + { + std::chrono::seconds LastAccessedSeconds = std::chrono::duration_cast( + GcClock::TimePointFromTick(ValueIt.second.LastAccess).time_since_epoch()); + CSVWriter << "\r\n" + << Namespace << "," << Bucket << "," << ValueIt.first.ToHexString() << ", " << ValueIt.second.Size << "," + << ValueIt.second.RawSize << "," << ValueIt.second.RawHash.ToHexString() << ", " + << ToString(ValueIt.second.ContentType) << ", " << (NowSeconds.count() - LastAccessedSeconds.count()) + << ", " << gsl::narrow(ValueIt.second.Attachments.size()); + size_t AttachmentsSize = 0; + for (const IoHash& Hash : ValueIt.second.Attachments) + { + IoBuffer Payload = m_CidStore.FindChunkByCid(Hash); + AttachmentsSize += Payload.GetSize(); + } + CSVWriter << ", " << gsl::narrow(AttachmentsSize); + } + else + { + CSVWriter << "\r\n" << Namespace << "," << Bucket << "," << ValueIt.first.ToHexString(); + } + } + } + } + return Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView()); + } + else + { + CbObjectWriter Cbo; + Cbo.BeginArray("namespaces"); + { + for (const auto& NamespaceIt : ValueDetails.Namespaces) + { + const std::string& Namespace = NamespaceIt.first; + Cbo.BeginObject(); + { + Cbo.AddString("name", Namespace); + Cbo.BeginArray("buckets"); + { + for (const auto& BucketIt : NamespaceIt.second.Buckets) + { + const std::string& Bucket = BucketIt.first; + Cbo.BeginObject(); + { + Cbo.AddString("name", Bucket); + Cbo.BeginArray("values"); + { + for (const auto& ValueIt : BucketIt.second.Values) + { + std::chrono::seconds LastAccessedSeconds = std::chrono::duration_cast( + GcClock::TimePointFromTick(ValueIt.second.LastAccess).time_since_epoch()); + Cbo.BeginObject(); + { + Cbo.AddHash("key", ValueIt.first); + if (Details) + { + Cbo.AddInteger("size", ValueIt.second.Size); + if (ValueIt.second.Size > 0 && ValueIt.second.RawSize != 0 && + ValueIt.second.RawSize != ValueIt.second.Size) + { + Cbo.AddInteger("rawsize", ValueIt.second.RawSize); + Cbo.AddHash("rawhash", ValueIt.second.RawHash); + } + Cbo.AddString("contenttype", ToString(ValueIt.second.ContentType)); + Cbo.AddInteger("age", + gsl::narrow(NowSeconds.count() - LastAccessedSeconds.count())); + if (ValueIt.second.Attachments.size() > 0) + { + if (AttachmentDetails) + { + Cbo.BeginArray("attachments"); + { + for (const IoHash& Hash : ValueIt.second.Attachments) + { + Cbo.BeginObject(); + Cbo.AddHash("cid", Hash); + IoBuffer Payload = m_CidStore.FindChunkByCid(Hash); + Cbo.AddInteger("size", gsl::narrow(Payload.GetSize())); + Cbo.EndObject(); + } + } + Cbo.EndArray(); + } + else + { + Cbo.AddInteger("attachmentcount", + gsl::narrow(ValueIt.second.Attachments.size())); + size_t AttachmentsSize = 0; + for (const IoHash& Hash : ValueIt.second.Attachments) + { + IoBuffer Payload = m_CidStore.FindChunkByCid(Hash); + AttachmentsSize += Payload.GetSize(); + } + Cbo.AddInteger("attachmentssize", gsl::narrow(AttachmentsSize)); + } + } + } + } + Cbo.EndObject(); + } + } + Cbo.EndArray(); + } + Cbo.EndObject(); + } + } + Cbo.EndArray(); + } + Cbo.EndObject(); + } + } + Cbo.EndArray(); + Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); + } +} + +void +HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request) +{ + ZEN_TRACE_CPU("z$::Http::HandleRequest"); + + ZEN_MEMSCOPE(GetCacheHttpTag()); + + metrics::OperationTiming::Scope $(m_HttpRequests); + + const std::string_view Key = Request.RelativeUri(); + + std::string_view UriNamespace; + + if (Key.ends_with(HttpZCacheRPCPrefix)) + { + const size_t RpcOffset = Key.length() - HttpZCacheRPCPrefix.length(); + + if (RpcOffset) + { + std::string_view KeyPrefix = Key.substr(0, RpcOffset); + + if (KeyPrefix.back() == '/') + { + KeyPrefix.remove_suffix(1); + + UriNamespace = KeyPrefix; + } + } + + return HandleRpcRequest(Request, UriNamespace); + } + + if (Key == HttpZCacheUtilStartRecording) + { + HttpServerRequest::QueryParams Params = Request.GetQueryParams(); + + std::string RecordPath = UrlDecode(Params.GetValue("path")); + + { + RwLock::ExclusiveLockScope _(m_RequestRecordingLock); + m_RequestRecordingEnabled.store(false); + m_RequestRecorder.reset(); + + m_RequestRecorder = cache::MakeDiskRequestRecorder(RecordPath); + m_RequestRecordingEnabled.store(true); + } + ZEN_INFO("cache RPC recording STARTED -> '{}'", RecordPath); + Request.WriteResponse(HttpResponseCode::OK); + return; + } + + if (Key == HttpZCacheUtilStopRecording) + { + { + RwLock::ExclusiveLockScope _(m_RequestRecordingLock); + m_RequestRecordingEnabled.store(false); + m_RequestRecorder.reset(); + } + ZEN_INFO("cache RPC recording STOPPED"); + Request.WriteResponse(HttpResponseCode::OK); + return; + } + + if (Key == HttpZCacheUtilReplayRecording) + { + CacheRequestContext RequestContext = {.SessionId = Request.SessionId(), .RequestId = Request.RequestId()}; + + { + RwLock::ExclusiveLockScope _(m_RequestRecordingLock); + m_RequestRecordingEnabled.store(false); + m_RequestRecorder.reset(); + } + + HttpServerRequest::QueryParams Params = Request.GetQueryParams(); + + std::string RecordPath = UrlDecode(Params.GetValue("path")); + + uint32_t ThreadCount = GetHardwareConcurrency(); + if (auto Param = Params.GetValue("thread_count"); Param.empty() == false) + { + if (auto Value = ParseInt(Param)) + { + ThreadCount = gsl::narrow(Value.value()); + } + } + + ZEN_INFO("initiating cache RPC replay using {} threads, from '{}'", ThreadCount, RecordPath); + + std::unique_ptr Replayer(cache::MakeDiskRequestReplayer(RecordPath, false)); + ReplayRequestRecorder(RequestContext, *Replayer, ThreadCount < 1 ? 1 : ThreadCount); + + ZEN_INFO("cache RPC replay COMPLETED"); + + Request.WriteResponse(HttpResponseCode::OK); + return; + } + + if (Key.starts_with(HttpZCacheDetailsPrefix)) + { + HandleDetailsRequest(Request); + return; + } + + HttpCacheRequestData RequestData; + if (!HttpCacheRequestParseRelativeUri(Key, ZenCacheStore::DefaultNamespace, RequestData)) + { + m_CacheStats.BadRequestCount++; + return Request.WriteResponse(HttpResponseCode::BadRequest); // invalid URL + } + + if (RequestData.ValueContentId.has_value()) + { + ZEN_ASSERT(RequestData.Namespace.has_value()); + ZEN_ASSERT(RequestData.Bucket.has_value()); + ZEN_ASSERT(RequestData.HashKey.has_value()); + CacheRef Ref = {.Namespace = RequestData.Namespace.value(), + .BucketSegment = RequestData.Bucket.value(), + .HashKey = RequestData.HashKey.value(), + .ValueContentId = RequestData.ValueContentId.value()}; + return HandleCacheChunkRequest(Request, Ref, ParseCachePolicy(Request.GetQueryParams())); + } + + if (RequestData.HashKey.has_value()) + { + ZEN_ASSERT(RequestData.Namespace.has_value()); + ZEN_ASSERT(RequestData.Bucket.has_value()); + CacheRef Ref = {.Namespace = RequestData.Namespace.value(), + .BucketSegment = RequestData.Bucket.value(), + .HashKey = RequestData.HashKey.value(), + .ValueContentId = IoHash::Zero}; + return HandleCacheRecordRequest(Request, Ref, ParseCachePolicy(Request.GetQueryParams())); + } + + if (RequestData.Bucket.has_value()) + { + ZEN_ASSERT(RequestData.Namespace.has_value()); + return HandleCacheBucketRequest(Request, RequestData.Namespace.value(), RequestData.Bucket.value()); + } + + if (RequestData.Namespace.has_value()) + { + return HandleCacheNamespaceRequest(Request, RequestData.Namespace.value()); + } + return HandleCacheRequest(Request); +} + +void +HttpStructuredCacheService::HandleCacheRequest(HttpServerRequest& Request) +{ + switch (Request.RequestVerb()) + { + case HttpVerb::kHead: + case HttpVerb::kGet: + { + ZenCacheStore::Info Info = m_CacheStore.GetInfo(); + + CbObjectWriter ResponseWriter; + + ResponseWriter.BeginObject("Configuration"); + { + ExtendableStringBuilder<128> BasePathString; + BasePathString << Info.BasePath.u8string(); + ResponseWriter.AddString("BasePath"sv, BasePathString.ToView()); + ResponseWriter.AddBool("AllowAutomaticCreationOfNamespaces", Info.Config.AllowAutomaticCreationOfNamespaces); + ResponseWriter.BeginObject("Logging"); + { + ResponseWriter.AddBool("EnableWriteLog", Info.Config.Logging.EnableWriteLog); + ResponseWriter.AddBool("EnableAccessLog", Info.Config.Logging.EnableAccessLog); + } + ResponseWriter.EndObject(); + } + ResponseWriter.EndObject(); + + std::sort(begin(Info.NamespaceNames), end(Info.NamespaceNames), [](std::string_view L, std::string_view R) { + return L.compare(R) < 0; + }); + ResponseWriter.BeginArray("Namespaces"); + for (const std::string& NamespaceName : Info.NamespaceNames) + { + ResponseWriter.AddString(NamespaceName); + } + ResponseWriter.EndArray(); + ResponseWriter.BeginObject("StorageSize"); + { + ResponseWriter.AddInteger("DiskSize", Info.StorageSize.DiskSize); + ResponseWriter.AddInteger("MemorySize", Info.StorageSize.MemorySize); + } + + ResponseWriter.EndObject(); + + ResponseWriter.AddInteger("DiskEntryCount", Info.DiskEntryCount); + + return Request.WriteResponse(HttpResponseCode::OK, ResponseWriter.Save()); + } + break; + default: + m_CacheStats.BadRequestCount++; + break; + } +} + +void +HttpStructuredCacheService::HandleCacheNamespaceRequest(HttpServerRequest& Request, std::string_view NamespaceName) +{ + switch (Request.RequestVerb()) + { + case HttpVerb::kHead: + case HttpVerb::kGet: + { + std::optional Info = m_CacheStore.GetNamespaceInfo(NamespaceName); + if (!Info.has_value()) + { + return Request.WriteResponse(HttpResponseCode::NotFound); + } + + CbObjectWriter ResponseWriter; + + ResponseWriter.BeginObject("Configuration"); + { + ExtendableStringBuilder<128> BasePathString; + BasePathString << Info->RootDir.u8string(); + ResponseWriter.AddString("RootDir"sv, BasePathString.ToView()); + ResponseWriter.AddInteger("MaxBlockSize"sv, Info->Config.DiskLayerConfig.BucketConfig.MaxBlockSize); + ResponseWriter.AddInteger("PayloadAlignment"sv, Info->Config.DiskLayerConfig.BucketConfig.PayloadAlignment); + ResponseWriter.AddInteger("MemCacheSizeThreshold"sv, Info->Config.DiskLayerConfig.BucketConfig.MemCacheSizeThreshold); + ResponseWriter.AddInteger("LargeObjectThreshold"sv, Info->Config.DiskLayerConfig.BucketConfig.LargeObjectThreshold); + ResponseWriter.AddInteger("MemCacheTargetFootprintBytes"sv, Info->Config.DiskLayerConfig.MemCacheTargetFootprintBytes); + ResponseWriter.AddInteger("MemCacheTrimIntervalSeconds"sv, Info->Config.DiskLayerConfig.MemCacheTrimIntervalSeconds); + ResponseWriter.AddInteger("MemCacheMaxAgeSeconds"sv, Info->Config.DiskLayerConfig.MemCacheMaxAgeSeconds); + } + ResponseWriter.EndObject(); + + std::sort(begin(Info->BucketNames), end(Info->BucketNames), [](std::string_view L, std::string_view R) { + return L.compare(R) < 0; + }); + + ResponseWriter.BeginArray("Buckets"sv); + for (const std::string& BucketName : Info->BucketNames) + { + ResponseWriter.AddString(BucketName); + } + ResponseWriter.EndArray(); + + ResponseWriter.BeginObject("StorageSize"sv); + { + ResponseWriter.AddInteger("DiskSize"sv, Info->DiskLayerInfo.StorageSize.DiskSize); + ResponseWriter.AddInteger("MemorySize"sv, Info->DiskLayerInfo.StorageSize.MemorySize); + } + ResponseWriter.EndObject(); + + ResponseWriter.AddInteger("EntryCount", Info->DiskLayerInfo.EntryCount); + + if (auto Buckets = HttpServerRequest::Decode(Request.GetQueryParams().GetValue("bucketsizes")); !Buckets.empty()) + { + ResponseWriter.BeginObject("BucketSizes"); + + ResponseWriter.BeginArray("Buckets"); + + std::vector BucketNames; + if (Buckets == "*") // Get all - empty FieldFilter equal getting all fields + { + BucketNames = Info.value().BucketNames; + } + else + { + ForEachStrTok(Buckets, ',', [&](std::string_view BucketName) { + BucketNames.push_back(std::string(BucketName)); + return true; + }); + } + WorkerThreadPool& WorkerPool = GetMediumWorkerPool(EWorkloadType::Background); + std::vector AllAttachments; + for (const std::string& BucketName : BucketNames) + { + ResponseWriter.BeginObject(); + ResponseWriter << "Name" << BucketName; + CacheContentStats ContentStats; + bool Success = m_CacheStore.GetContentStats(NamespaceName, BucketName, ContentStats); + if (Success) + { + size_t ValuesSize = 0; + for (const uint64_t Size : ContentStats.ValueSizes) + { + ValuesSize += Size; + } + + std::sort(ContentStats.Attachments.begin(), ContentStats.Attachments.end()); + auto NewEnd = std::unique(ContentStats.Attachments.begin(), ContentStats.Attachments.end()); + ContentStats.Attachments.erase(NewEnd, ContentStats.Attachments.end()); + + ResponseWriter << "Count" << ContentStats.ValueSizes.size(); + ResponseWriter << "StructuredCount" << ContentStats.StructuredValuesCount; + ResponseWriter << "StandaloneCount" << ContentStats.StandaloneValuesCount; + ResponseWriter << "Size" << ValuesSize; + ResponseWriter << "AttachmentCount" << ContentStats.Attachments.size(); + + AllAttachments.insert(AllAttachments.end(), ContentStats.Attachments.begin(), ContentStats.Attachments.end()); + } + ResponseWriter.EndObject(); + } + + ResponseWriter.EndArray(); + + ResponseWriter.BeginObject("Attachments"); + std::sort(AllAttachments.begin(), AllAttachments.end()); + auto NewEnd = std::unique(AllAttachments.begin(), AllAttachments.end()); + AllAttachments.erase(NewEnd, AllAttachments.end()); + + uint64_t AttachmentsSize = 0; + + m_CidStore.IterateChunks( + AllAttachments, + [&](size_t Index, const IoBuffer& Payload) { + ZEN_UNUSED(Index); + AttachmentsSize += Payload.GetSize(); + return true; + }, + &WorkerPool, + 8u * 1024u); + + ResponseWriter << "Count" << AllAttachments.size(); + ResponseWriter << "Size" << AttachmentsSize; + + ResponseWriter.EndObject(); + + ResponseWriter.EndObject(); + } + + return Request.WriteResponse(HttpResponseCode::OK, ResponseWriter.Save()); + } + break; + + case HttpVerb::kDelete: + // Drop namespace + { + if (m_CacheStore.DropNamespace(NamespaceName)) + { + return Request.WriteResponse(HttpResponseCode::OK); + } + else + { + return Request.WriteResponse(HttpResponseCode::NotFound); + } + } + break; + + default: + break; + } +} + +void +HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, + std::string_view NamespaceName, + std::string_view BucketName) +{ + switch (Request.RequestVerb()) + { + case HttpVerb::kHead: + case HttpVerb::kGet: + { + std::optional Info = m_CacheStore.GetBucketInfo(NamespaceName, BucketName); + if (!Info.has_value()) + { + return Request.WriteResponse(HttpResponseCode::NotFound); + } + + CbObjectWriter ResponseWriter; + + ResponseWriter.BeginObject("StorageSize"); + { + ResponseWriter.AddInteger("DiskSize", Info->DiskLayerInfo.StorageSize.DiskSize); + ResponseWriter.AddInteger("MemorySize", Info->DiskLayerInfo.StorageSize.MemorySize); + } + ResponseWriter.EndObject(); + + ResponseWriter.AddInteger("DiskEntryCount", Info->DiskLayerInfo.EntryCount); + + if (auto GetBucketSize = Request.GetQueryParams().GetValue("bucketsize"); GetBucketSize == "true") + { + CacheContentStats ContentStats; + bool Success = m_CacheStore.GetContentStats(NamespaceName, BucketName, ContentStats); + if (Success) + { + size_t ValuesSize = 0; + for (const uint64_t Size : ContentStats.ValueSizes) + { + ValuesSize += Size; + } + + std::sort(ContentStats.Attachments.begin(), ContentStats.Attachments.end()); + auto NewEnd = std::unique(ContentStats.Attachments.begin(), ContentStats.Attachments.end()); + ContentStats.Attachments.erase(NewEnd, ContentStats.Attachments.end()); + + ResponseWriter << "Count" << ContentStats.ValueSizes.size(); + ResponseWriter << "StructuredCount" << ContentStats.StructuredValuesCount; + ResponseWriter << "StandaloneCount" << ContentStats.StandaloneValuesCount; + ResponseWriter << "Size" << ValuesSize; + ResponseWriter << "AttachmentCount" << ContentStats.Attachments.size(); + + uint64_t AttachmentsSize = 0; + + WorkerThreadPool& WorkerPool = GetMediumWorkerPool(EWorkloadType::Background); + + m_CidStore.IterateChunks( + ContentStats.Attachments, + [&](size_t Index, const IoBuffer& Payload) { + ZEN_UNUSED(Index); + AttachmentsSize += Payload.GetSize(); + return true; + }, + &WorkerPool, + 8u * 1024u); + + ResponseWriter << "AttachmentsSize" << AttachmentsSize; + } + } + + return Request.WriteResponse(HttpResponseCode::OK, ResponseWriter.Save()); + } + break; + + case HttpVerb::kDelete: + // Drop bucket + { + if (m_CacheStore.DropBucket(NamespaceName, BucketName)) + { + return Request.WriteResponse(HttpResponseCode::OK); + } + else + { + return Request.WriteResponse(HttpResponseCode::NotFound); + } + } + break; + + default: + break; + } +} + +void +HttpStructuredCacheService::HandleCacheRecordRequest(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl) +{ + switch (Request.RequestVerb()) + { + case HttpVerb::kHead: + case HttpVerb::kGet: + HandleGetCacheRecord(Request, Ref, PolicyFromUrl); + break; + + case HttpVerb::kPut: + HandlePutCacheRecord(Request, Ref, PolicyFromUrl); + break; + + default: + break; + } +} + +void +HttpStructuredCacheService::HandleGetCacheRecord(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl) +{ + const ZenContentType AcceptType = Request.AcceptContentType(); + const bool SkipData = EnumHasAllFlags(PolicyFromUrl, CachePolicy::SkipData); + const bool PartialRecord = EnumHasAllFlags(PolicyFromUrl, CachePolicy::PartialRecord); + + bool Success = false; + uint32_t MissingCount = 0; + ZenCacheValue ClientResultValue; + if (!EnumHasAnyFlags(PolicyFromUrl, CachePolicy::Query)) + { + return Request.WriteResponse(HttpResponseCode::OK); + } + + const bool HasUpstream = m_UpstreamCache.IsActive(); + + CacheRequestContext RequestContext = {.SessionId = Request.SessionId(), .RequestId = Request.RequestId()}; + Stopwatch Timer; + + if (EnumHasAllFlags(PolicyFromUrl, CachePolicy::QueryLocal) && + m_CacheStore.Get(RequestContext, Ref.Namespace, Ref.BucketSegment, Ref.HashKey, ClientResultValue)) + { + Success = true; + ZenContentType ContentType = ClientResultValue.Value.GetContentType(); + + if (AcceptType == ZenContentType::kCbPackage) + { + if (ContentType == ZenContentType::kCbObject) + { + CbPackage Package; + CbValidateError ValidateError = CbValidateError::None; + if (CbObject PackageObject = ValidateAndReadCompactBinaryObject(std::move(ClientResultValue.Value), ValidateError); + ValidateError == CbValidateError::None) + { + CbObjectView CacheRecord(ClientResultValue.Value.Data()); + CacheRecord.IterateAttachments([this, &MissingCount, &Package, SkipData](CbFieldView AttachmentHash) { + if (SkipData) + { + if (!m_CidStore.ContainsChunk(AttachmentHash.AsHash())) + { + MissingCount++; + } + } + else + { + if (IoBuffer Chunk = m_CidStore.FindChunkByCid(AttachmentHash.AsHash())) + { + CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(Chunk)); + if (Compressed) + { + Package.AddAttachment(CbAttachment(Compressed, AttachmentHash.AsHash())); + } + else + { + ZEN_WARN("invalid compressed binary returned for {}", AttachmentHash.AsHash()); + MissingCount++; + } + } + else + { + MissingCount++; + } + } + }); + + Success = MissingCount == 0 || PartialRecord; + } + else + { + ZEN_WARN("Invalid compact binary payload returned for {}/{}/{} ({}). Reason: '{}'", + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + Ref.ValueContentId, + ToString(ValidateError)); + Success = false; + } + + if (Success) + { + CbObject PackageObject = LoadCompactBinaryObject(std::move(ClientResultValue.Value)); + + Package.SetObject(std::move(PackageObject)); + + BinaryWriter MemStream; + Package.Save(MemStream); + + ClientResultValue.Value = IoBuffer(IoBuffer::Clone, MemStream.Data(), MemStream.Size()); + ClientResultValue.Value.SetContentType(HttpContentType::kCbPackage); + } + } + else + { + Success = false; + } + } + else if (AcceptType != ClientResultValue.Value.GetContentType() && AcceptType != ZenContentType::kUnknownContentType && + AcceptType != ZenContentType::kBinary) + { + Success = false; + } + } + + if (Success) + { + ZEN_DEBUG("GETCACHERECORD HIT - '{}/{}/{}' {} '{}' (LOCAL) in {}", + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + NiceBytes(ClientResultValue.Value.Size()), + ToString(ClientResultValue.Value.GetContentType()), + NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); + + m_CacheStats.HitCount++; + if (SkipData && AcceptType != ZenContentType::kCbPackage && AcceptType != ZenContentType::kCbObject) + { + return Request.WriteResponse(HttpResponseCode::OK); + } + else + { + // kCbPackage handled SkipData when constructing the ClientResultValue, kcbObject ignores SkipData + return Request.WriteResponse((MissingCount == 0) ? HttpResponseCode::OK : HttpResponseCode::PartialContent, + ClientResultValue.Value.GetContentType(), + ClientResultValue.Value); + } + } + else if (!HasUpstream || !EnumHasAllFlags(PolicyFromUrl, CachePolicy::QueryRemote)) + { + ZEN_DEBUG("GETCACHERECORD MISS - '{}/{}/{}' '{}' in {}", + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + ToString(AcceptType), + NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); + m_CacheStats.MissCount++; + return Request.WriteResponse(HttpResponseCode::NotFound); + } + + // Issue upstream query asynchronously in order to keep requests flowing without + // hogging I/O servicing threads with blocking work + + uint64_t LocalElapsedTimeUs = Timer.GetElapsedTimeUs(); + + Request.WriteResponseAsync([this, AcceptType, PolicyFromUrl, Ref, LocalElapsedTimeUs, RequestContext](HttpServerRequest& AsyncRequest) { + Stopwatch Timer; + bool Success = false; + const bool PartialRecord = EnumHasAllFlags(PolicyFromUrl, CachePolicy::PartialRecord); + const bool QueryLocal = EnumHasAllFlags(PolicyFromUrl, CachePolicy::QueryLocal); + const bool StoreLocal = EnumHasAllFlags(PolicyFromUrl, CachePolicy::StoreLocal) && AreDiskWritesAllowed(); + const bool SkipData = EnumHasAllFlags(PolicyFromUrl, CachePolicy::SkipData); + ZenCacheValue ClientResultValue; + + metrics::OperationTiming::Scope $(m_UpstreamGetRequestTiming); + + if (GetUpstreamCacheSingleResult UpstreamResult = + m_UpstreamCache.GetCacheRecord(Ref.Namespace, {Ref.BucketSegment, Ref.HashKey}, AcceptType); + UpstreamResult.Status.Success) + { + Success = true; + + ClientResultValue.Value = UpstreamResult.Value; + ClientResultValue.Value.SetContentType(AcceptType); + + if (AcceptType == ZenContentType::kBinary || AcceptType == ZenContentType::kCbObject) + { + if (AcceptType == ZenContentType::kCbObject) + { + const CbValidateError ValidationResult = ValidateCompactBinary(UpstreamResult.Value, CbValidateMode::All); + if (ValidationResult != CbValidateError::None) + { + Success = false; + ZEN_WARN("Get - '{}/{}/{}' '{}' FAILED, invalid compact binary object from upstream", + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + ToString(AcceptType)); + } + + // We do not do anything to the returned object for SkipData, only package attachments are cut when skipping data + } + + if (Success && StoreLocal) + { + const bool Overwrite = !EnumHasAllFlags(PolicyFromUrl, CachePolicy::QueryLocal); + ZenCacheStore::PutResult PutResult = + m_CacheStore + .Put(RequestContext, Ref.Namespace, Ref.BucketSegment, Ref.HashKey, ClientResultValue, {}, Overwrite, nullptr); + if (PutResult.Status == zen::PutStatus::Success) + { + m_CacheStats.WriteCount++; + } + } + } + else if (AcceptType == ZenContentType::kCbPackage) + { + CbPackage Package; + if (Package.TryLoad(ClientResultValue.Value)) + { + CbObject CacheRecord = Package.GetObject(); + AttachmentCount Count; + size_t NumAttachments = Package.GetAttachments().size(); + std::vector ReferencedAttachments; + std::vector WriteAttachmentBuffers; + WriteAttachmentBuffers.reserve(NumAttachments); + std::vector WriteRawHashes; + WriteRawHashes.reserve(NumAttachments); + + CacheRecord.IterateAttachments([this, + &Package, + &Ref, + &WriteAttachmentBuffers, + &WriteRawHashes, + &ReferencedAttachments, + &Count, + QueryLocal, + StoreLocal, + SkipData](CbFieldView HashView) { + IoHash Hash = HashView.AsHash(); + ReferencedAttachments.push_back(Hash); + if (const CbAttachment* Attachment = Package.FindAttachment(Hash)) + { + if (Attachment->IsCompressedBinary()) + { + if (StoreLocal) + { + const CompressedBuffer& Chunk = Attachment->AsCompressedBinary(); + WriteAttachmentBuffers.push_back(Chunk.GetCompressed().Flatten().AsIoBuffer()); + WriteRawHashes.push_back(Attachment->GetHash()); + } + Count.Valid++; + } + else + { + ZEN_WARN("Uncompressed value '{}' from upstream cache record '{}/{}'", + Hash, + Ref.BucketSegment, + Ref.HashKey); + Count.Invalid++; + } + } + else if (QueryLocal) + { + if (SkipData) + { + if (m_CidStore.ContainsChunk(Hash)) + { + Count.Valid++; + } + } + else if (IoBuffer Chunk = m_CidStore.FindChunkByCid(Hash)) + { + CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(Chunk)); + if (Compressed) + { + Package.AddAttachment(CbAttachment(Compressed, Hash)); + Count.Valid++; + } + else + { + ZEN_WARN("Uncompressed value '{}' stored in local cache '{}/{}'", Hash, Ref.BucketSegment, Ref.HashKey); + Count.Invalid++; + } + } + } + Count.Total++; + }); + + if ((Count.Valid == Count.Total) || PartialRecord) + { + ZenCacheValue CacheValue; + CacheValue.Value = CacheRecord.GetBuffer().AsIoBuffer(); + CacheValue.Value.SetContentType(ZenContentType::kCbObject); + + if (StoreLocal) + { + const bool Overwrite = !EnumHasAllFlags(PolicyFromUrl, CachePolicy::QueryLocal); + ZenCacheStore::PutResult PutResult = m_CacheStore.Put(RequestContext, + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + CacheValue, + ReferencedAttachments, + Overwrite, + nullptr); + if (PutResult.Status == zen::PutStatus::Success) + { + m_CacheStats.WriteCount++; + + if (!WriteAttachmentBuffers.empty()) + { + std::vector InsertResults = + m_CidStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes); + for (const CidStore::InsertResult& Result : InsertResults) + { + if (Result.New) + { + Count.New++; + } + } + } + + WriteAttachmentBuffers = {}; + WriteRawHashes = {}; + } + } + + BinaryWriter MemStream; + if (SkipData) + { + // Save a package containing only the object. + CbPackage(Package.GetObject()).Save(MemStream); + } + else + { + Package.Save(MemStream); + } + + ClientResultValue.Value = IoBuffer(IoBuffer::Clone, MemStream.Data(), MemStream.Size()); + ClientResultValue.Value.SetContentType(ZenContentType::kCbPackage); + } + else + { + Success = false; + ZEN_WARN("Get - '{}/{}' '{}' FAILED, attachments missing in upstream package", + Ref.BucketSegment, + Ref.HashKey, + ToString(AcceptType)); + } + } + else + { + Success = false; + ZEN_WARN("Get - '{}/{}/{}' '{}' FAILED, invalid upstream package", + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + ToString(AcceptType)); + } + } + } + + if (Success) + { + ZEN_DEBUG("GETCACHERECORD HIT - '{}/{}/{}' {} '{}' (UPSTREAM) in {}", + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + NiceBytes(ClientResultValue.Value.Size()), + ToString(ClientResultValue.Value.GetContentType()), + NiceLatencyNs((LocalElapsedTimeUs + Timer.GetElapsedTimeUs()) * 1000)); + + m_CacheStats.HitCount++; + m_CacheStats.UpstreamHitCount++; + + if (SkipData && AcceptType == ZenContentType::kBinary) + { + AsyncRequest.WriteResponse(HttpResponseCode::OK); + } + else + { + // Other methods modify ClientResultValue to a version that has skipped the data but keeps the Object and optionally + // metadata. + AsyncRequest.WriteResponse(HttpResponseCode::OK, ClientResultValue.Value.GetContentType(), ClientResultValue.Value); + } + } + else + { + ZEN_DEBUG("GETCACHERECORD MISS - '{}/{}/{}' '{}' in {}", + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + ToString(AcceptType), + NiceLatencyNs((LocalElapsedTimeUs + Timer.GetElapsedTimeUs()) * 1000)); + m_CacheStats.MissCount++; + AsyncRequest.WriteResponse(HttpResponseCode::NotFound); + } + }); +} + +void +HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl) +{ + IoBuffer Body = Request.ReadPayload(); + + if (!Body || Body.Size() == 0) + { + m_CacheStats.BadRequestCount++; + return Request.WriteResponse(HttpResponseCode::BadRequest); + } + if (!AreDiskWritesAllowed()) + { + return Request.WriteResponse(HttpResponseCode::InsufficientStorage); + } + + auto WriteFailureResponse = [&Request](const ZenCacheStore::PutResult& PutResult) { + ZEN_UNUSED(PutResult); + + HttpResponseCode ResponseCode = HttpResponseCode::InternalServerError; + switch (PutResult.Status) + { + case zen::PutStatus::Conflict: + ResponseCode = HttpResponseCode::Conflict; + break; + case zen::PutStatus::Invalid: + ResponseCode = HttpResponseCode::BadRequest; + break; + } + + if (PutResult.Details) + { + Request.WriteResponse(ResponseCode, PutResult.Details); + } + return Request.WriteResponse(ResponseCode); + }; + + const HttpContentType ContentType = Request.RequestContentType(); + + Body.SetContentType(ContentType); + + CacheRequestContext RequestContext = {.SessionId = Request.SessionId(), .RequestId = Request.RequestId()}; + + const bool HasUpstream = m_UpstreamCache.IsActive(); + + Stopwatch Timer; + + if (ContentType == HttpContentType::kBinary || ContentType == HttpContentType::kCompressedBinary) + { + IoHash RawHash = IoHash::Zero; + uint64_t RawSize = Body.GetSize(); + if (ContentType == HttpContentType::kCompressedBinary) + { + if (!CompressedBuffer::ValidateCompressedHeader(Body, RawHash, RawSize)) + { + m_CacheStats.BadRequestCount++; + return Request.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Payload is not a valid compressed binary"sv); + } + } + else + { + RawHash = IoHash::HashBuffer(SharedBuffer(Body)); + } + const bool Overwrite = !EnumHasAllFlags(PolicyFromUrl, CachePolicy::QueryLocal); + // TODO: Propagation for rejected PUTs + ZenCacheStore::PutResult PutResult = m_CacheStore.Put(RequestContext, + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + {.Value = Body, .RawSize = RawSize, .RawHash = RawHash}, + {}, + Overwrite, + nullptr); + if (PutResult.Status != zen::PutStatus::Success) + { + return WriteFailureResponse(PutResult); + } + m_CacheStats.WriteCount++; + + if (HasUpstream && EnumHasAllFlags(PolicyFromUrl, CachePolicy::StoreRemote)) + { + m_UpstreamCache.EnqueueUpstream({.Type = ContentType, .Namespace = Ref.Namespace, .Key = {Ref.BucketSegment, Ref.HashKey}}); + } + + ZEN_DEBUG("PUTCACHERECORD - '{}/{}/{}' {} '{}' in {}", + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + NiceBytes(Body.Size()), + ToString(ContentType), + NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); + Request.WriteResponse(HttpResponseCode::Created); + } + else if (ContentType == HttpContentType::kCbObject) + { + const CbValidateError ValidationResult = ValidateCompactBinary(MemoryView(Body.GetData(), Body.GetSize()), CbValidateMode::All); + + if (ValidationResult != CbValidateError::None) + { + ZEN_WARN("PUTCACHERECORD - '{}/{}/{}' '{}' FAILED, invalid compact binary", + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + ToString(ContentType)); + m_CacheStats.BadRequestCount++; + return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Compact binary validation failed"sv); + } + + Body.SetContentType(ZenContentType::kCbObject); + + CbObjectView CacheRecord(Body.Data()); + std::vector ValidAttachments; + std::vector ReferencedAttachments; + int32_t TotalCount = 0; + + CacheRecord.IterateAttachments([this, &TotalCount, &ValidAttachments, &ReferencedAttachments](CbFieldView AttachmentHash) { + const IoHash Hash = AttachmentHash.AsHash(); + ReferencedAttachments.push_back(Hash); + if (m_CidStore.ContainsChunk(Hash)) + { + ValidAttachments.emplace_back(Hash); + } + TotalCount++; + }); + + const bool Overwrite = !EnumHasAllFlags(PolicyFromUrl, CachePolicy::QueryLocal); + + // TODO: Propagation for rejected PUTs + ZenCacheStore::PutResult PutResult = m_CacheStore.Put(RequestContext, + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + {.Value = Body}, + ReferencedAttachments, + Overwrite, + nullptr); + if (PutResult.Status != zen::PutStatus::Success) + { + return WriteFailureResponse(PutResult); + } + m_CacheStats.WriteCount++; + + ZEN_DEBUG("PUTCACHERECORD - '{}/{}/{}' {} '{}' attachments '{}/{}' (valid/total) in {}", + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + NiceBytes(Body.Size()), + ToString(ContentType), + TotalCount, + ValidAttachments.size(), + NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); + + const bool IsPartialRecord = TotalCount != static_cast(ValidAttachments.size()); + + CachePolicy Policy = PolicyFromUrl; + if (HasUpstream && EnumHasAllFlags(Policy, CachePolicy::StoreRemote) && !IsPartialRecord) + { + m_UpstreamCache.EnqueueUpstream({.Type = ZenContentType::kCbObject, + .Namespace = Ref.Namespace, + .Key = {Ref.BucketSegment, Ref.HashKey}, + .ValueContentIds = std::move(ValidAttachments)}); + } + + Request.WriteResponse(HttpResponseCode::Created); + } + else if (ContentType == HttpContentType::kCbPackage) + { + CbPackage Package; + + if (!Package.TryLoad(Body)) + { + ZEN_WARN("PUTCACHERECORD - '{}/{}/{}' '{}' FAILED, invalid package", + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + ToString(ContentType)); + m_CacheStats.BadRequestCount++; + return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid package"sv); + } + CachePolicy Policy = PolicyFromUrl; + + CbObject CacheRecord = Package.GetObject(); + + AttachmentCount Count; + size_t NumAttachments = Package.GetAttachments().size(); + std::vector ValidAttachments; + std::vector ReferencedAttachments; + ValidAttachments.reserve(NumAttachments); + std::vector WriteAttachmentBuffers; + std::vector WriteRawHashes; + WriteAttachmentBuffers.reserve(NumAttachments); + WriteRawHashes.reserve(NumAttachments); + + CacheRecord.IterateAttachments( + [this, &Ref, &Package, &WriteAttachmentBuffers, &WriteRawHashes, &ValidAttachments, &ReferencedAttachments, &Count]( + CbFieldView HashView) { + const IoHash Hash = HashView.AsHash(); + ReferencedAttachments.push_back(Hash); + if (const CbAttachment* Attachment = Package.FindAttachment(Hash)) + { + if (Attachment->IsCompressedBinary()) + { + WriteAttachmentBuffers.emplace_back(Attachment->AsCompressedBinary().GetCompressed().Flatten().AsIoBuffer()); + WriteRawHashes.push_back(Hash); + ValidAttachments.emplace_back(Hash); + Count.Valid++; + } + else + { + ZEN_WARN("PUTCACHERECORD - '{}/{}/{}' '{}' FAILED, attachment '{}' is not compressed", + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + ToString(HttpContentType::kCbPackage), + Hash); + Count.Invalid++; + } + } + else if (m_CidStore.ContainsChunk(Hash)) + { + ValidAttachments.emplace_back(Hash); + Count.Valid++; + } + Count.Total++; + }); + + if (Count.Invalid > 0) + { + m_CacheStats.BadRequestCount++; + return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid attachment(s)"sv); + } + + const bool Overwrite = !EnumHasAllFlags(Policy, CachePolicy::QueryLocal); + + ZenCacheValue CacheValue; + CacheValue.Value = CacheRecord.GetBuffer().AsIoBuffer(); + CacheValue.Value.SetContentType(ZenContentType::kCbObject); + // TODO: Propagation for rejected PUTs + ZenCacheStore::PutResult PutResult = + m_CacheStore.Put(RequestContext, Ref.Namespace, Ref.BucketSegment, Ref.HashKey, CacheValue, ReferencedAttachments, Overwrite); + if (PutResult.Status != zen::PutStatus::Success) + { + return WriteFailureResponse(PutResult); + } + m_CacheStats.WriteCount++; + + if (!WriteAttachmentBuffers.empty()) + { + std::vector InsertResults = m_CidStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes); + for (const CidStore::InsertResult& InsertResult : InsertResults) + { + if (InsertResult.New) + { + Count.New++; + } + } + WriteAttachmentBuffers = {}; + WriteRawHashes = {}; + } + + ZEN_DEBUG("PUTCACHERECORD - '{}/{}/{}' {} '{}', attachments '{}/{}/{}' (new/valid/total) in {}", + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + NiceBytes(Body.GetSize()), + ToString(ContentType), + Count.New, + Count.Valid, + Count.Total, + NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); + + const bool IsPartialRecord = Count.Valid != Count.Total; + + if (HasUpstream && EnumHasAllFlags(Policy, CachePolicy::StoreRemote) && !IsPartialRecord) + { + m_UpstreamCache.EnqueueUpstream({.Type = ZenContentType::kCbPackage, + .Namespace = Ref.Namespace, + .Key = {Ref.BucketSegment, Ref.HashKey}, + .ValueContentIds = std::move(ValidAttachments)}); + } + + Request.WriteResponse(HttpResponseCode::Created); + } + else + { + m_CacheStats.BadRequestCount++; + return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Content-Type invalid"sv); + } +} + +void +HttpStructuredCacheService::HandleCacheChunkRequest(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl) +{ + switch (Request.RequestVerb()) + { + case HttpVerb::kHead: + case HttpVerb::kGet: + HandleGetCacheChunk(Request, Ref, PolicyFromUrl); + break; + case HttpVerb::kPut: + HandlePutCacheChunk(Request, Ref, PolicyFromUrl); + break; + default: + break; + } +} + +void +HttpStructuredCacheService::HandleGetCacheChunk(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl) +{ + Stopwatch Timer; + + IoBuffer Value = m_CidStore.FindChunkByCid(Ref.ValueContentId); + const UpstreamEndpointInfo* Source = nullptr; + CachePolicy Policy = PolicyFromUrl; + + const bool HasUpstream = m_UpstreamCache.IsActive(); + { + const bool QueryUpstream = HasUpstream && !Value && EnumHasAllFlags(Policy, CachePolicy::QueryRemote); + + if (QueryUpstream) + { + if (GetUpstreamCacheSingleResult UpstreamResult = + m_UpstreamCache.GetCacheChunk(Ref.Namespace, {Ref.BucketSegment, Ref.HashKey}, Ref.ValueContentId); + UpstreamResult.Status.Success) + { + IoHash RawHash; + uint64_t RawSize; + if (CompressedBuffer::ValidateCompressedHeader(UpstreamResult.Value, RawHash, RawSize)) + { + if (RawHash == Ref.ValueContentId) + { + if (AreDiskWritesAllowed()) + { + m_CidStore.AddChunk(UpstreamResult.Value, RawHash); + } + Source = UpstreamResult.Source; + } + else + { + ZEN_WARN("got missmatching upstream cache value"); + } + } + else + { + ZEN_WARN("got uncompressed upstream cache value"); + } + } + } + } + + if (!Value) + { + ZEN_DEBUG("GETCACHECHUNK MISS - '{}/{}/{}/{}' '{}' in {}", + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + Ref.ValueContentId, + ToString(Request.AcceptContentType()), + NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); + m_CacheStats.MissCount++; + return Request.WriteResponse(HttpResponseCode::NotFound); + } + + ZEN_DEBUG("GETCACHECHUNK HIT - '{}/{}/{}/{}' {} '{}' ({}) in {}", + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + Ref.ValueContentId, + NiceBytes(Value.Size()), + ToString(Value.GetContentType()), + Source ? Source->Url : "LOCAL"sv, + NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); + + m_CacheStats.HitCount++; + if (Source) + { + m_CacheStats.UpstreamHitCount++; + } + + if (EnumHasAllFlags(Policy, CachePolicy::SkipData)) + { + Request.WriteResponse(HttpResponseCode::OK); + } + else + { + Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, Value); + } +} + +void +HttpStructuredCacheService::HandlePutCacheChunk(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl) +{ + // Note: Individual cacherecord values are not propagated upstream until a valid cache record has been stored + ZEN_UNUSED(PolicyFromUrl); + + Stopwatch Timer; + + IoBuffer Body = Request.ReadPayload(); + + if (!Body || Body.Size() == 0) + { + m_CacheStats.BadRequestCount++; + return Request.WriteResponse(HttpResponseCode::BadRequest); + } + if (!AreDiskWritesAllowed()) + { + return Request.WriteResponse(HttpResponseCode::InsufficientStorage); + } + + Body.SetContentType(Request.RequestContentType()); + + IoHash RawHash; + uint64_t RawSize; + if (!CompressedBuffer::ValidateCompressedHeader(Body, RawHash, RawSize)) + { + m_CacheStats.BadRequestCount++; + return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Attachments must be compressed"sv); + } + + if (RawHash != Ref.ValueContentId) + { + m_CacheStats.BadRequestCount++; + return Request.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "ValueContentId does not match attachment hash"sv); + } + + CidStore::InsertResult Result = m_CidStore.AddChunk(Body, RawHash); + + ZEN_DEBUG("PUTCACHECHUNK - '{}/{}/{}/{}' {} '{}' ({}) in {}", + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + Ref.ValueContentId, + NiceBytes(Body.Size()), + ToString(Body.GetContentType()), + Result.New ? "NEW" : "OLD", + NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); + + const HttpResponseCode ResponseCode = Result.New ? HttpResponseCode::Created : HttpResponseCode::OK; + + Request.WriteResponse(ResponseCode); +} + +void +HttpStructuredCacheService::ReplayRequestRecorder(const CacheRequestContext& Context, + cache::IRpcRequestReplayer& Replayer, + uint32_t ThreadCount) +{ + WorkerThreadPool WorkerPool(ThreadCount); + uint64_t RequestCount = Replayer.GetRequestCount(); + Stopwatch Timer; + auto _ = MakeGuard([&]() { ZEN_INFO("Replayed {} requests in {}", RequestCount, NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); }); + std::atomic AbortFlag; + std::atomic PauseFlag; + ParallelWork Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::EnableBacklog); + ZEN_INFO("Replaying {} requests", RequestCount); + for (uint64_t RequestIndex = 0; RequestIndex < RequestCount; ++RequestIndex) + { + if (AbortFlag) + { + break; + } + Work.ScheduleWork(WorkerPool, [this, &Context, &Replayer, RequestIndex](std::atomic& AbortFlag) { + IoBuffer Body; + zen::cache::RecordedRequestInfo RequestInfo = Replayer.GetRequest(RequestIndex, /* out */ Body); + + if (AbortFlag) + { + return; + } + + if (Body) + { + uint32_t AcceptMagic = 0; + RpcAcceptOptions AcceptFlags = RpcAcceptOptions::kNone; + int TargetPid = 0; + CbPackage RpcResult; + if (m_RpcHandler.HandleRpcRequest(Context, + /* UriNamespace */ {}, + RequestInfo.ContentType, + std::move(Body), + AcceptMagic, + AcceptFlags, + TargetPid, + RpcResult) == CacheRpcHandler::RpcResponseCode::OK) + { + if (AcceptMagic == kCbPkgMagic) + { + void* TargetProcessHandle = nullptr; + FormatFlags Flags = FormatFlags::kDefault; + if (EnumHasAllFlags(AcceptFlags, RpcAcceptOptions::kAllowLocalReferences)) + { + Flags |= FormatFlags::kAllowLocalReferences; + if (!EnumHasAnyFlags(AcceptFlags, RpcAcceptOptions::kAllowPartialLocalReferences)) + { + Flags |= FormatFlags::kDenyPartialLocalReferences; + } + TargetProcessHandle = m_OpenProcessCache.GetProcessHandle(Context.SessionId, TargetPid); + } + CompositeBuffer RpcResponseBuffer = FormatPackageMessageBuffer(RpcResult, Flags, TargetProcessHandle); + ZEN_ASSERT(RpcResponseBuffer.GetSize() > 0); + } + else + { + BinaryWriter MemStream; + RpcResult.Save(MemStream); + IoBuffer RpcResponseBuffer(IoBuffer::Wrap, MemStream.GetData(), MemStream.GetSize()); + ZEN_ASSERT(RpcResponseBuffer.Size() > 0); + } + } + } + }); + } + Work.Wait(10000, [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) { + ZEN_UNUSED(IsAborted, IsPaused); + ZEN_INFO("Replayed {} of {} requests, elapsed {}", + RequestCount - PendingWork, + RequestCount, + NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); + }); +} + +void +HttpStructuredCacheService::HandleRpcRequest(HttpServerRequest& Request, std::string_view UriNamespace) +{ + ZEN_MEMSCOPE(GetCacheRpcTag()); + + ZEN_TRACE_CPU("z$::Http::HandleRpcRequest"); + + const bool HasUpstream = m_UpstreamCache.IsActive(); + + switch (Request.RequestVerb()) + { + case HttpVerb::kPost: + { + CacheRequestContext RequestContext = {.SessionId = Request.SessionId(), .RequestId = Request.RequestId()}; + + const HttpContentType ContentType = Request.RequestContentType(); + const HttpContentType AcceptType = Request.AcceptContentType(); + + if ((ContentType != HttpContentType::kCbObject && ContentType != HttpContentType::kCbPackage) || + AcceptType != HttpContentType::kCbPackage) + { + m_CacheStats.BadRequestCount++; + return Request.WriteResponse(HttpResponseCode::BadRequest); + } + + auto HandleRpc = [this, + RequestContext, + Body = Request.ReadPayload(), + ContentType, + AcceptType, + UriNamespaceString = std::string{UriNamespace}](HttpServerRequest& AsyncRequest) mutable { + if (m_RequestRecordingEnabled) + { + RwLock::SharedLockScope _(m_RequestRecordingLock); + if (m_RequestRecorder) + { + m_RequestRecorder->RecordRequest( + {.ContentType = ContentType, .AcceptType = AcceptType, .SessionId = RequestContext.SessionId}, + Body); + } + } + + uint32_t AcceptMagic = 0; + RpcAcceptOptions AcceptFlags = RpcAcceptOptions::kNone; + int TargetProcessId = 0; + CbPackage RpcResult; + + CacheRpcHandler::RpcResponseCode ResultCode = m_RpcHandler.HandleRpcRequest(RequestContext, + UriNamespaceString, + ContentType, + std::move(Body), + /* out */ AcceptMagic, + /* out */ AcceptFlags, + /* out */ TargetProcessId, + /* out */ RpcResult); + + HttpResponseCode HttpResultCode = HttpResponseCode(int(ResultCode)); + + if (!IsHttpSuccessCode(HttpResultCode)) + { + return AsyncRequest.WriteResponse(HttpResultCode); + } + + if (AcceptMagic == kCbPkgMagic) + { + void* TargetProcessHandle = nullptr; + FormatFlags Flags = FormatFlags::kDefault; + if (EnumHasAllFlags(AcceptFlags, RpcAcceptOptions::kAllowLocalReferences)) + { + Flags |= FormatFlags::kAllowLocalReferences; + if (!EnumHasAnyFlags(AcceptFlags, RpcAcceptOptions::kAllowPartialLocalReferences)) + { + Flags |= FormatFlags::kDenyPartialLocalReferences; + } + TargetProcessHandle = m_OpenProcessCache.GetProcessHandle(RequestContext.SessionId, TargetProcessId); + } + CompositeBuffer RpcResponseBuffer = FormatPackageMessageBuffer(RpcResult, Flags, TargetProcessHandle); + AsyncRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, RpcResponseBuffer); + } + else + { + BinaryWriter MemStream; + RpcResult.Save(MemStream); + + AsyncRequest.WriteResponse(HttpResponseCode::OK, + HttpContentType::kCbPackage, + IoBuffer(IoBuffer::Wrap, MemStream.GetData(), MemStream.GetSize())); + } + }; + + if (HasUpstream) + { + ZEN_TRACE_CPU("z$::Http::HandleRpcRequest::WriteResponseAsync"); + Request.WriteResponseAsync(std::move(HandleRpc)); + } + else + { + ZEN_TRACE_CPU("z$::Http::HandleRpcRequest::WriteResponse"); + HandleRpc(Request); + } + } + break; + + default: + m_CacheStats.BadRequestCount++; + Request.WriteResponse(HttpResponseCode::BadRequest); + break; + } +} + +void +HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request) +{ + ZEN_MEMSCOPE(GetCacheHttpTag()); + + CbObjectWriter Cbo; + + EmitSnapshot("requests", m_HttpRequests, Cbo); + + const uint64_t HitCount = m_CacheStats.HitCount; + const uint64_t UpstreamHitCount = m_CacheStats.UpstreamHitCount; + const uint64_t MissCount = m_CacheStats.MissCount; + const uint64_t WriteCount = m_CacheStats.WriteCount; + const uint64_t BadRequestCount = m_CacheStats.BadRequestCount; + struct CidStoreStats StoreStats = m_CidStore.Stats(); + const uint64_t ChunkHitCount = StoreStats.HitCount; + const uint64_t ChunkMissCount = StoreStats.MissCount; + const uint64_t ChunkWriteCount = StoreStats.WriteCount; + const uint64_t TotalCount = HitCount + MissCount; + + const uint64_t RpcRequests = m_CacheStats.RpcRequests; + const uint64_t RpcRecordRequests = m_CacheStats.RpcRecordRequests; + const uint64_t RpcRecordBatchRequests = m_CacheStats.RpcRecordBatchRequests; + const uint64_t RpcValueRequests = m_CacheStats.RpcValueRequests; + const uint64_t RpcValueBatchRequests = m_CacheStats.RpcValueBatchRequests; + const uint64_t RpcChunkRequests = m_CacheStats.RpcChunkRequests; + const uint64_t RpcChunkBatchRequests = m_CacheStats.RpcChunkBatchRequests; + + const CidStoreSize CidSize = m_CidStore.TotalSize(); + const CacheStoreSize CacheSize = m_CacheStore.TotalSize(); + + bool ShowCidStoreStats = Request.GetQueryParams().GetValue("cidstorestats") == "true"; + bool ShowCacheStoreStats = Request.GetQueryParams().GetValue("cachestorestats") == "true"; + + CidStoreStats CidStoreStats = {}; + if (ShowCidStoreStats) + { + CidStoreStats = m_CidStore.Stats(); + } + ZenCacheStore::CacheStoreStats CacheStoreStats = {}; + if (ShowCacheStoreStats) + { + CacheStoreStats = m_CacheStore.Stats(); + } + + Cbo.BeginObject("cache"); + { + Cbo << "badrequestcount" << BadRequestCount; + Cbo.BeginObject("rpc"); + Cbo << "count" << RpcRequests; + Cbo << "ops" << RpcRecordBatchRequests + RpcValueBatchRequests + RpcChunkBatchRequests; + Cbo.BeginObject("records"); + Cbo << "count" << RpcRecordRequests; + Cbo << "ops" << RpcRecordBatchRequests; + Cbo.EndObject(); + Cbo.BeginObject("values"); + Cbo << "count" << RpcValueRequests; + Cbo << "ops" << RpcValueBatchRequests; + Cbo.EndObject(); + Cbo.BeginObject("chunks"); + Cbo << "count" << RpcChunkRequests; + Cbo << "ops" << RpcChunkBatchRequests; + Cbo.EndObject(); + Cbo.EndObject(); + + Cbo.BeginObject("size"); + { + Cbo << "disk" << CacheSize.DiskSize; + Cbo << "memory" << CacheSize.MemorySize; + } + Cbo.EndObject(); + + Cbo << "hits" << HitCount << "misses" << MissCount << "writes" << WriteCount; + Cbo << "hit_ratio" << (TotalCount > 0 ? (double(HitCount) / double(TotalCount)) : 0.0); + + if (m_UpstreamCache.IsActive()) + { + Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0); + Cbo << "upstream_hits" << m_CacheStats.UpstreamHitCount; + Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0); + Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0); + } + + Cbo << "cidhits" << ChunkHitCount << "cidmisses" << ChunkMissCount << "cidwrites" << ChunkWriteCount; + + if (ShowCacheStoreStats) + { + Cbo.BeginObject("store"); + Cbo << "hits" << CacheStoreStats.HitCount << "misses" << CacheStoreStats.MissCount << "writes" << CacheStoreStats.WriteCount + << "rejected_writes" << CacheStoreStats.RejectedWriteCount << "rejected_reads" << CacheStoreStats.RejectedReadCount; + const uint64_t StoreTotal = CacheStoreStats.HitCount + CacheStoreStats.MissCount; + Cbo << "hit_ratio" << (StoreTotal > 0 ? (double(CacheStoreStats.HitCount) / double(StoreTotal)) : 0.0); + EmitSnapshot("read", CacheStoreStats.GetOps, Cbo); + EmitSnapshot("write", CacheStoreStats.PutOps, Cbo); + if (!CacheStoreStats.NamespaceStats.empty()) + { + Cbo.BeginArray("namespaces"); + for (const ZenCacheStore::NamedNamespaceStats& NamespaceStats : CacheStoreStats.NamespaceStats) + { + Cbo.BeginObject(); + Cbo.AddString("namespace", NamespaceStats.NamespaceName); + Cbo << "hits" << NamespaceStats.Stats.HitCount << "misses" << NamespaceStats.Stats.MissCount << "writes" + << NamespaceStats.Stats.WriteCount; + const uint64_t NamespaceTotal = NamespaceStats.Stats.HitCount + NamespaceStats.Stats.MissCount; + Cbo << "hit_ratio" << (NamespaceTotal > 0 ? (double(NamespaceStats.Stats.HitCount) / double(NamespaceTotal)) : 0.0); + EmitSnapshot("read", NamespaceStats.Stats.GetOps, Cbo); + EmitSnapshot("write", NamespaceStats.Stats.PutOps, Cbo); + Cbo.BeginObject("size"); + { + Cbo << "disk" << NamespaceStats.Stats.DiskStats.DiskSize; + Cbo << "memory" << NamespaceStats.Stats.DiskStats.MemorySize; + } + Cbo.EndObject(); + if (!NamespaceStats.Stats.DiskStats.BucketStats.empty()) + { + Cbo.BeginArray("buckets"); + for (const ZenCacheDiskLayer::NamedBucketStats& BucketStats : NamespaceStats.Stats.DiskStats.BucketStats) + { + Cbo.BeginObject(); + Cbo.AddString("bucket", BucketStats.BucketName); + if (BucketStats.Stats.DiskSize != 0 || BucketStats.Stats.MemorySize != 0) + { + Cbo.BeginObject("size"); + { + Cbo << "disk" << BucketStats.Stats.DiskSize; + Cbo << "memory" << BucketStats.Stats.MemorySize; + } + Cbo.EndObject(); + } + + if (BucketStats.Stats.DiskSize == 0 && BucketStats.Stats.DiskHitCount == 0 && + BucketStats.Stats.DiskMissCount == 0 && BucketStats.Stats.DiskWriteCount == 0 && + BucketStats.Stats.MemoryHitCount == 0 && BucketStats.Stats.MemoryMissCount == 0 && + BucketStats.Stats.MemoryWriteCount == 0) + { + Cbo.EndObject(); + continue; + } + + const uint64_t BucketDiskTotal = BucketStats.Stats.DiskHitCount + BucketStats.Stats.DiskMissCount; + if (BucketDiskTotal != 0 || BucketStats.Stats.DiskWriteCount != 0) + { + Cbo << "hits" << BucketStats.Stats.DiskHitCount << "misses" << BucketStats.Stats.DiskMissCount << "writes" + << BucketStats.Stats.DiskWriteCount; + Cbo << "hit_ratio" + << (BucketDiskTotal > 0 ? (double(BucketStats.Stats.DiskHitCount) / double(BucketDiskTotal)) : 0.0); + } + + const uint64_t BucketMemoryTotal = BucketStats.Stats.MemoryHitCount + BucketStats.Stats.MemoryMissCount; + if (BucketMemoryTotal != 0 || BucketStats.Stats.MemoryWriteCount != 0) + { + Cbo << "mem_hits" << BucketStats.Stats.MemoryHitCount << "mem_misses" << BucketStats.Stats.MemoryMissCount + << "mem_writes" << BucketStats.Stats.MemoryWriteCount; + Cbo << "mem_hit_ratio" + << (BucketMemoryTotal > 0 ? (double(BucketStats.Stats.MemoryHitCount) / double(BucketMemoryTotal)) + : 0.0); + } + + if (BucketDiskTotal != 0 || BucketStats.Stats.DiskWriteCount != 0 || BucketMemoryTotal != 0 || + BucketStats.Stats.MemoryWriteCount != 0) + { + EmitSnapshot("read", BucketStats.Stats.GetOps, Cbo); + EmitSnapshot("write", BucketStats.Stats.PutOps, Cbo); + } + + Cbo.EndObject(); + } + Cbo.EndArray(); + } + Cbo.EndObject(); + } + Cbo.EndArray(); + } + Cbo.EndObject(); + } + Cbo.EndObject(); + } + + if (m_UpstreamCache.IsActive()) + { + EmitSnapshot("upstream_gets", m_UpstreamGetRequestTiming, Cbo); + Cbo.BeginObject("upstream"); + { + m_UpstreamCache.GetStatus(Cbo); + } + Cbo.EndObject(); + } + + Cbo.BeginObject("cid"); + { + Cbo.BeginObject("size"); + { + Cbo << "tiny" << CidSize.TinySize; + Cbo << "small" << CidSize.SmallSize; + Cbo << "large" << CidSize.LargeSize; + Cbo << "total" << CidSize.TotalSize; + } + Cbo.EndObject(); + + if (ShowCidStoreStats) + { + Cbo.BeginObject("store"); + Cbo << "hits" << CidStoreStats.HitCount << "misses" << CidStoreStats.MissCount << "writes" << CidStoreStats.WriteCount; + EmitSnapshot("read", CidStoreStats.FindChunkOps, Cbo); + EmitSnapshot("write", CidStoreStats.AddChunkOps, Cbo); + // EmitSnapshot("exists", CidStoreStats.ContainChunkOps, Cbo); + Cbo.EndObject(); + } + } + Cbo.EndObject(); + + Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + +void +HttpStructuredCacheService::HandleStatusRequest(HttpServerRequest& Request) +{ + CbObjectWriter Cbo; + Cbo << "ok" << true; + Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + +bool +HttpStructuredCacheService::AreDiskWritesAllowed() const +{ + return (m_DiskWriteBlocker == nullptr || m_DiskWriteBlocker->AreDiskWritesAllowed()); +} + +} // namespace zen diff --git a/src/zenserver/storage/cache/httpstructuredcache.h b/src/zenserver/storage/cache/httpstructuredcache.h new file mode 100644 index 000000000..a157148c9 --- /dev/null +++ b/src/zenserver/storage/cache/httpstructuredcache.h @@ -0,0 +1,138 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace zen { + +struct CacheChunkRequest; +struct CacheKeyRequest; +struct PutRequestData; + +class CidStore; +class CbObjectView; +class DiskWriteBlocker; +class HttpStructuredCacheService; +class ScrubContext; +class UpstreamCache; +class ZenCacheStore; + +enum class CachePolicy : uint32_t; +enum class RpcAcceptOptions : uint16_t; + +namespace cache { + class IRpcRequestReplayer; + class IRpcRequestRecorder; + namespace detail { + struct RecordBody; + struct ChunkRequest; + } // namespace detail +} // namespace cache + +/** + * Structured cache service. Imposes constraints on keys, supports blobs and + * structured values + * + * Keys are structured as: + * + * {BucketId}/{KeyHash} + * + * Where BucketId is a lower-case alphanumeric string, and KeyHash is a 40-character + * hexadecimal sequence. The hash value may be derived in any number of ways, it's + * up to the application to pick an approach. + * + * Values may be structured or unstructured. Structured values are encoded using Unreal + * Engine's compact binary encoding (see CbObject) + * + * Additionally, attachments may be addressed as: + * + * {BucketId}/{KeyHash}/{ValueHash} + * + * Where the two initial components are the same as for the main endpoint + * + * The storage strategy is as follows: + * + * - Structured values are stored in a dedicated backing store per bucket + * - Unstructured values and attachments are stored in the CAS pool + * + */ + +class HttpStructuredCacheService : public HttpService, public IHttpStatsProvider, public IHttpStatusProvider +{ +public: + HttpStructuredCacheService(ZenCacheStore& InCacheStore, + CidStore& InCidStore, + HttpStatsService& StatsService, + HttpStatusService& StatusService, + UpstreamCache& UpstreamCache, + const DiskWriteBlocker* InDiskWriteBlocker, + OpenProcessCache& InOpenProcessCache); + ~HttpStructuredCacheService(); + + virtual const char* BaseUri() const override; + virtual void HandleRequest(HttpServerRequest& Request) override; + + void Flush(); + +private: + struct CacheRef + { + std::string Namespace; + std::string BucketSegment; + IoHash HashKey; + IoHash ValueContentId; + }; + + void HandleCacheRecordRequest(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl); + void HandleGetCacheRecord(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl); + void HandlePutCacheRecord(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl); + void HandleCacheChunkRequest(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl); + void HandleGetCacheChunk(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl); + void HandlePutCacheChunk(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl); + void HandleRpcRequest(HttpServerRequest& Request, std::string_view UriNamespace); + void HandleDetailsRequest(HttpServerRequest& Request); + + void HandleCacheRequest(HttpServerRequest& Request); + void HandleCacheNamespaceRequest(HttpServerRequest& Request, std::string_view Namespace); + void HandleCacheBucketRequest(HttpServerRequest& Request, std::string_view Namespace, std::string_view Bucket); + virtual void HandleStatsRequest(HttpServerRequest& Request) override; + virtual void HandleStatusRequest(HttpServerRequest& Request) override; + + bool AreDiskWritesAllowed() const; + + LoggerRef Log() { return m_Log; } + LoggerRef m_Log; + ZenCacheStore& m_CacheStore; + HttpStatsService& m_StatsService; + HttpStatusService& m_StatusService; + CidStore& m_CidStore; + UpstreamCache& m_UpstreamCache; + metrics::OperationTiming m_HttpRequests; + metrics::OperationTiming m_UpstreamGetRequestTiming; + CacheStats m_CacheStats; + const DiskWriteBlocker* m_DiskWriteBlocker = nullptr; + OpenProcessCache& m_OpenProcessCache; + CacheRpcHandler m_RpcHandler; + + void ReplayRequestRecorder(const CacheRequestContext& Context, cache::IRpcRequestReplayer& Replayer, uint32_t ThreadCount); + + // This exists to avoid taking locks when recording is not enabled + std::atomic_bool m_RequestRecordingEnabled{false}; + + // This lock should be taken in SHARED mode when calling into the recorder, + // and taken in EXCLUSIVE mode whenever the recorder is created or destroyed + RwLock m_RequestRecordingLock; + std::unique_ptr m_RequestRecorder; +}; + +} // namespace zen diff --git a/src/zenserver/storage/objectstore/objectstore.cpp b/src/zenserver/storage/objectstore/objectstore.cpp new file mode 100644 index 000000000..3f40bf616 --- /dev/null +++ b/src/zenserver/storage/objectstore/objectstore.cpp @@ -0,0 +1,618 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "objectstore.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include "zencore/compactbinary.h" +#include "zencore/compactbinarybuilder.h" +#include "zenhttp/httpcommon.h" +#include "zenhttp/httpserver.h" + +#include +#include + +ZEN_THIRD_PARTY_INCLUDES_START +#include +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen { + +using namespace std::literals; + +ZEN_DEFINE_LOG_CATEGORY_STATIC(LogObj, "obj"sv); + +class CbXmlWriter +{ +public: + explicit CbXmlWriter(StringBuilderBase& InBuilder) : Builder(InBuilder) + { + Builder.Append(""); + Builder << LINE_TERMINATOR_ANSI; + } + + void WriteField(CbFieldView Field) + { + using namespace std::literals; + + bool SkipEndTag = false; + const std::u8string_view Tag = Field.GetU8Name(); + + AppendBeginTag(Tag); + + switch (CbValue Accessor = Field.GetValue(); Accessor.GetType()) + { + case CbFieldType::Null: + Builder << "Null"sv; + break; + case CbFieldType::Object: + case CbFieldType::UniformObject: + { + for (CbFieldView It : Field) + { + WriteField(It); + } + } + break; + case CbFieldType::Array: + case CbFieldType::UniformArray: + { + bool FirstField = true; + for (CbFieldView It : Field) + { + if (!FirstField) + AppendBeginTag(Tag); + + WriteField(It); + AppendEndTag(Tag); + FirstField = false; + } + SkipEndTag = true; + } + break; + case CbFieldType::Binary: + AppendBase64String(Accessor.AsBinary()); + break; + case CbFieldType::String: + Builder << Accessor.AsU8String(); + break; + case CbFieldType::IntegerPositive: + Builder << Accessor.AsIntegerPositive(); + break; + case CbFieldType::IntegerNegative: + Builder << Accessor.AsIntegerNegative(); + break; + case CbFieldType::Float32: + { + const float Value = Accessor.AsFloat32(); + if (std::isfinite(Value)) + { + Builder.Append(fmt::format("{:.9g}", Value)); + } + else + { + Builder << "Null"sv; + } + } + break; + case CbFieldType::Float64: + { + const double Value = Accessor.AsFloat64(); + if (std::isfinite(Value)) + { + Builder.Append(fmt::format("{:.17g}", Value)); + } + else + { + Builder << "null"sv; + } + } + break; + case CbFieldType::BoolFalse: + Builder << "False"sv; + break; + case CbFieldType::BoolTrue: + Builder << "True"sv; + break; + case CbFieldType::ObjectAttachment: + case CbFieldType::BinaryAttachment: + { + Accessor.AsAttachment().ToHexString(Builder); + } + break; + case CbFieldType::Hash: + { + Accessor.AsHash().ToHexString(Builder); + } + break; + case CbFieldType::Uuid: + { + Accessor.AsUuid().ToString(Builder); + } + break; + case CbFieldType::DateTime: + Builder << DateTime(Accessor.AsDateTimeTicks()).ToIso8601(); + break; + case CbFieldType::TimeSpan: + { + const TimeSpan Span(Accessor.AsTimeSpanTicks()); + if (Span.GetDays() == 0) + { + Builder << Span.ToString("%h:%m:%s.%n"); + } + else + { + Builder << Span.ToString("%d.%h:%m:%s.%n"); + } + break; + } + case CbFieldType::ObjectId: + Accessor.AsObjectId().ToString(Builder); + break; + case CbFieldType::CustomById: + { + CbCustomById Custom = Accessor.AsCustomById(); + + AppendBeginTag(u8"Id"sv); + Builder << Custom.Id; + AppendEndTag(u8"Id"sv); + + AppendBeginTag(u8"Data"sv); + AppendBase64String(Custom.Data); + AppendEndTag(u8"Data"sv); + break; + } + case CbFieldType::CustomByName: + { + CbCustomByName Custom = Accessor.AsCustomByName(); + + AppendBeginTag(u8"Name"sv); + Builder << Custom.Name; + AppendEndTag(u8"Name"sv); + + AppendBeginTag(u8"Data"sv); + AppendBase64String(Custom.Data); + AppendEndTag(u8"Data"sv); + break; + } + default: + ZEN_ASSERT(false); + break; + } + + if (!SkipEndTag) + AppendEndTag(Tag); + } + +private: + void AppendBeginTag(std::u8string_view Tag) + { + if (!Tag.empty()) + { + Builder << '<' << Tag << '>'; + } + } + + void AppendEndTag(std::u8string_view Tag) + { + if (!Tag.empty()) + { + Builder << "'; + } + } + + void AppendBase64String(MemoryView Value) + { + Builder << '"'; + ZEN_ASSERT(Value.GetSize() <= 512 * 1024 * 1024); + const uint32_t EncodedSize = Base64::GetEncodedDataSize(uint32_t(Value.GetSize())); + const size_t EncodedIndex = Builder.AddUninitialized(size_t(EncodedSize)); + Base64::Encode(static_cast(Value.GetData()), uint32_t(Value.GetSize()), Builder.Data() + EncodedIndex); + } + +private: + StringBuilderBase& Builder; +}; + +HttpObjectStoreService::HttpObjectStoreService(HttpStatusService& StatusService, ObjectStoreConfig Cfg) +: m_StatusService(StatusService) +, m_Cfg(std::move(Cfg)) +{ + Inititalize(); + m_StatusService.RegisterHandler("obj", *this); +} + +HttpObjectStoreService::~HttpObjectStoreService() +{ + m_StatusService.UnregisterHandler("obj", *this); +} + +const char* +HttpObjectStoreService::BaseUri() const +{ + return "/obj/"; +} + +void +HttpObjectStoreService::HandleRequest(zen::HttpServerRequest& Request) +{ + if (m_Router.HandleRequest(Request) == false) + { + ZEN_LOG_WARN(LogObj, "No route found for {0}", Request.RelativeUri()); + return Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Not found"sv); + } +} + +void +HttpObjectStoreService::HandleStatusRequest(HttpServerRequest& Request) +{ + CbObjectWriter Cbo; + Cbo << "ok" << true; + Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + +void +HttpObjectStoreService::Inititalize() +{ + ZEN_TRACE_CPU("HttpObjectStoreService::Inititalize"); + + namespace fs = std::filesystem; + ZEN_LOG_INFO(LogObj, "Initialzing Object Store in '{}'", m_Cfg.RootDirectory); + + const fs::path BucketsPath = m_Cfg.RootDirectory / "buckets"; + if (!IsDir(BucketsPath)) + { + CreateDirectories(BucketsPath); + } + + m_Router.RegisterRoute( + "bucket", + [this](zen::HttpRouterRequest& Request) { CreateBucket(Request); }, + HttpVerb::kPost | HttpVerb::kPut); + + m_Router.RegisterRoute( + "bucket", + [this](zen::HttpRouterRequest& Request) { DeleteBucket(Request); }, + HttpVerb::kDelete); + + m_Router.RegisterRoute( + "bucket/{path}", + [this](zen::HttpRouterRequest& Request) { + const std::string_view Path = Request.GetCapture(1); + const auto Sep = Path.find_last_of('.'); + const bool IsObject = Sep != std::string::npos && Path.size() - Sep > 0; + + if (IsObject) + { + GetObject(Request, Path); + } + else + { + ListBucket(Request, Path); + } + }, + HttpVerb::kHead | HttpVerb::kGet); + + m_Router.RegisterRoute( + "bucket/{bucket}/{path}", + [this](zen::HttpRouterRequest& Request) { PutObject(Request); }, + HttpVerb::kPost | HttpVerb::kPut); +} + +std::filesystem::path +HttpObjectStoreService::GetBucketDirectory(std::string_view BucketName) +{ + { + std::lock_guard _(BucketsMutex); + + if (const auto It = std::find_if(std::begin(m_Cfg.Buckets), + std::end(m_Cfg.Buckets), + [&BucketName](const auto& Bucket) -> bool { return Bucket.Name == BucketName; }); + It != std::end(m_Cfg.Buckets)) + { + return It->Directory.make_preferred(); + } + } + + return (m_Cfg.RootDirectory / "buckets" / BucketName).make_preferred(); +} + +void +HttpObjectStoreService::CreateBucket(zen::HttpRouterRequest& Request) +{ + namespace fs = std::filesystem; + + const CbObject Params = Request.ServerRequest().ReadPayloadObject(); + const std::string_view BucketName = Params["bucketname"].AsString(); + + if (BucketName.empty()) + { + return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); + } + + const fs::path BucketPath = m_Cfg.RootDirectory / "buckets" / BucketName; + { + std::lock_guard _(BucketsMutex); + if (!IsDir(BucketPath)) + { + CreateDirectories(BucketPath); + ZEN_LOG_INFO(LogObj, "CREATE - new bucket '{}' OK", BucketName); + return Request.ServerRequest().WriteResponse(HttpResponseCode::Created); + } + } + + ZEN_LOG_INFO(LogObj, "CREATE - existing bucket '{}' OK", BucketName); + Request.ServerRequest().WriteResponse(HttpResponseCode::OK); +} + +void +HttpObjectStoreService::ListBucket(zen::HttpRouterRequest& Request, const std::string_view Path) +{ + namespace fs = std::filesystem; + + const auto Sep = Path.find_first_of('/'); + const std::string BucketName{Sep == std::string::npos ? Path : Path.substr(0, Sep)}; + if (BucketName.empty()) + { + return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); + } + + std::string BucketPrefix{Sep == std::string::npos || Sep == Path.size() - 1 ? std::string() : Path.substr(BucketName.size() + 1)}; + if (BucketPrefix.empty()) + { + const auto QueryParms = Request.ServerRequest().GetQueryParams(); + if (auto PrefixParam = QueryParms.GetValue("prefix"); PrefixParam.empty() == false) + { + BucketPrefix = PrefixParam; + } + } + BucketPrefix.erase(0, BucketPrefix.find_first_not_of('/')); + BucketPrefix.erase(0, BucketPrefix.find_first_not_of('\\')); + + const fs::path BucketRoot = GetBucketDirectory(BucketName); + const fs::path RelativeBucketPath = fs::path(BucketPrefix).make_preferred(); + const fs::path FullPath = BucketRoot / RelativeBucketPath; + + struct Visitor : FileSystemTraversal::TreeVisitor + { + Visitor(const std::string_view BucketName, const fs::path& Path, const fs::path& Prefix) : BucketPath(Path) + { + Writer.BeginObject("ListBucketResult"sv); + Writer << "Name"sv << BucketName; + std::string Tmp = Prefix.string(); + std::replace(Tmp.begin(), Tmp.end(), '\\', '/'); + Writer << "Prefix"sv << Tmp; + Writer.BeginArray("Contents"sv); + } + + void VisitFile(const fs::path& Parent, const path_view& File, uint64_t FileSize, uint32_t, uint64_t) override + { + const fs::path FullPath = Parent / fs::path(File); + fs::path RelativePath = fs::relative(FullPath, BucketPath); + + std::string Key = RelativePath.string(); + std::replace(Key.begin(), Key.end(), '\\', '/'); + + Writer.BeginObject(); + Writer << "Key"sv << Key; + Writer << "Size"sv << FileSize; + Writer.EndObject(); + } + + bool VisitDirectory(const std::filesystem::path&, const path_view&, uint32_t) override { return false; } + + CbObject GetResult() + { + Writer.EndArray(); + Writer.EndObject(); + return Writer.Save(); + } + + CbObjectWriter Writer; + fs::path BucketPath; + }; + + Visitor FileVisitor(BucketName, BucketRoot, RelativeBucketPath); + FileSystemTraversal Traversal; + + if (IsDir(FullPath)) + { + std::lock_guard _(BucketsMutex); + Traversal.TraverseFileSystem(FullPath, FileVisitor); + } + CbObject Result = FileVisitor.GetResult(); + + if (Request.ServerRequest().AcceptContentType() == HttpContentType::kJSON) + { + ExtendableStringBuilder<1024> Sb; + return Request.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kJSON, Result.ToJson(Sb).ToView()); + } + + ExtendableStringBuilder<1024> Xml; + CbXmlWriter XmlWriter(Xml); + XmlWriter.WriteField(Result.AsFieldView()); + + Request.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kXML, Xml.ToView()); +} + +void +HttpObjectStoreService::DeleteBucket(zen::HttpRouterRequest& Request) +{ + namespace fs = std::filesystem; + + const CbObject Params = Request.ServerRequest().ReadPayloadObject(); + const std::string_view BucketName = Params["bucketname"].AsString(); + + if (BucketName.empty()) + { + return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); + } + + const fs::path BucketPath = m_Cfg.RootDirectory / "buckets" / BucketName; + { + std::lock_guard _(BucketsMutex); + DeleteDirectories(BucketPath); + } + + ZEN_LOG_INFO(LogObj, "DELETE - bucket '{}' OK", BucketName); + Request.ServerRequest().WriteResponse(HttpResponseCode::OK); +} + +void +HttpObjectStoreService::GetObject(zen::HttpRouterRequest& Request, const std::string_view Path) +{ + namespace fs = std::filesystem; + + const auto Sep = Path.find_first_of('/'); + const std::string BucketName{Sep == std::string::npos ? Path : Path.substr(0, Sep)}; + const std::string BucketPrefix{Sep == std::string::npos || Sep == Path.size() - 1 ? std::string() : Path.substr(BucketName.size() + 1)}; + + const fs::path BucketDir = GetBucketDirectory(BucketName); + + if (BucketDir.empty()) + { + ZEN_LOG_DEBUG(LogObj, "GET - [FAILED], unknown bucket '{}'", BucketName); + return Request.ServerRequest().WriteResponse(HttpResponseCode::NotFound); + } + + const fs::path RelativeBucketPath = fs::path(BucketPrefix).make_preferred(); + + if (RelativeBucketPath.is_absolute() || RelativeBucketPath.string().starts_with("..")) + { + ZEN_LOG_DEBUG(LogObj, "GET - from bucket '{}' [FAILED], invalid file path", BucketName); + return Request.ServerRequest().WriteResponse(HttpResponseCode::Forbidden); + } + + const fs::path FilePath = BucketDir / RelativeBucketPath; + if (!IsFile(FilePath)) + { + ZEN_LOG_DEBUG(LogObj, "GET - '{}/{}' [FAILED], doesn't exist", BucketName, FilePath); + return Request.ServerRequest().WriteResponse(HttpResponseCode::NotFound); + } + + zen::HttpRanges Ranges; + if (Request.ServerRequest().TryGetRanges(Ranges); Ranges.size() > 1) + { + // Only a single range is supported + return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); + } + + FileContents File; + { + std::lock_guard _(BucketsMutex); + File = ReadFile(FilePath); + } + + if (File.ErrorCode) + { + ZEN_LOG_WARN(LogObj, + "GET - '{}/{}' [FAILED] ('{}': {})", + BucketName, + FilePath, + File.ErrorCode.category().name(), + File.ErrorCode.value()); + + return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); + } + + const IoBuffer& FileBuf = File.Data[0]; + + if (Ranges.empty()) + { + const uint64_t TotalServed = TotalBytesServed.fetch_add(FileBuf.Size()) + FileBuf.Size(); + + ZEN_LOG_DEBUG(LogObj, + "GET - '{}/{}' ({}) [OK] (Served: {})", + BucketName, + RelativeBucketPath, + NiceBytes(FileBuf.Size()), + NiceBytes(TotalServed)); + + Request.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, FileBuf); + } + else + { + const auto Range = Ranges[0]; + const uint64_t RangeSize = 1 + (Range.End - Range.Start); + const uint64_t TotalServed = TotalBytesServed.fetch_add(RangeSize) + RangeSize; + + ZEN_LOG_DEBUG(LogObj, + "GET - '{}/{}' (Range: {}-{}) ({}/{}) [OK] (Served: {})", + BucketName, + RelativeBucketPath, + Range.Start, + Range.End, + NiceBytes(RangeSize), + NiceBytes(FileBuf.Size()), + NiceBytes(TotalServed)); + + MemoryView RangeView = FileBuf.GetView().Mid(Range.Start, RangeSize); + if (RangeView.GetSize() != RangeSize) + { + return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); + } + + IoBuffer RangeBuf = IoBuffer(IoBuffer::Wrap, RangeView.GetData(), RangeView.GetSize()); + Request.ServerRequest().WriteResponse(HttpResponseCode::PartialContent, HttpContentType::kBinary, RangeBuf); + } +} + +void +HttpObjectStoreService::PutObject(zen::HttpRouterRequest& Request) +{ + namespace fs = std::filesystem; + + const std::string_view BucketName = Request.GetCapture(1); + const fs::path BucketDir = GetBucketDirectory(BucketName); + + if (BucketDir.empty()) + { + ZEN_LOG_DEBUG(LogObj, "PUT - [FAILED], unknown bucket '{}'", BucketName); + return Request.ServerRequest().WriteResponse(HttpResponseCode::NotFound); + } + + const fs::path RelativeBucketPath = fs::path(Request.GetCapture(2)).make_preferred(); + + if (RelativeBucketPath.is_absolute() || RelativeBucketPath.string().starts_with("..")) + { + ZEN_LOG_DEBUG(LogObj, "PUT - bucket '{}' [FAILED], invalid file path", BucketName); + return Request.ServerRequest().WriteResponse(HttpResponseCode::Forbidden); + } + + const fs::path FilePath = BucketDir / RelativeBucketPath; + const fs::path FileDirectory = FilePath.parent_path(); + + { + std::lock_guard _(BucketsMutex); + + if (!IsDir(FileDirectory)) + { + CreateDirectories(FileDirectory); + } + + const IoBuffer FileBuf = Request.ServerRequest().ReadPayload(); + + if (FileBuf.Size() == 0) + { + ZEN_LOG_DEBUG(LogObj, "PUT - '{}' [FAILED], empty file", FilePath); + return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); + } + + TemporaryFile::SafeWriteFile(FilePath, FileBuf.GetView()); + + ZEN_LOG_DEBUG(LogObj, + "PUT - '{}' [OK] ({})", + (fs::path(BucketName) / RelativeBucketPath).make_preferred(), + NiceBytes(FileBuf.Size())); + } + + Request.ServerRequest().WriteResponse(HttpResponseCode::OK); +} + +} // namespace zen diff --git a/src/zenserver/storage/objectstore/objectstore.h b/src/zenserver/storage/objectstore/objectstore.h new file mode 100644 index 000000000..44e50e208 --- /dev/null +++ b/src/zenserver/storage/objectstore/objectstore.h @@ -0,0 +1,53 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include +#include +#include +#include + +namespace zen { + +class HttpRouterRequest; + +struct ObjectStoreConfig +{ + struct BucketConfig + { + std::string Name; + std::filesystem::path Directory; + }; + + std::filesystem::path RootDirectory; + std::vector Buckets; +}; + +class HttpObjectStoreService final : public zen::HttpService, public IHttpStatusProvider +{ +public: + HttpObjectStoreService(HttpStatusService& StatusService, ObjectStoreConfig Cfg); + virtual ~HttpObjectStoreService(); + + virtual const char* BaseUri() const override; + virtual void HandleRequest(zen::HttpServerRequest& Request) override; + virtual void HandleStatusRequest(HttpServerRequest& Request) override; + +private: + void Inititalize(); + std::filesystem::path GetBucketDirectory(std::string_view BucketName); + void CreateBucket(zen::HttpRouterRequest& Request); + void ListBucket(zen::HttpRouterRequest& Request, const std::string_view Path); + void DeleteBucket(zen::HttpRouterRequest& Request); + void GetObject(zen::HttpRouterRequest& Request, const std::string_view Path); + void PutObject(zen::HttpRouterRequest& Request); + + HttpStatusService& m_StatusService; + ObjectStoreConfig m_Cfg; + std::mutex BucketsMutex; + HttpRequestRouter m_Router; + std::atomic_uint64_t TotalBytesServed{0}; +}; + +} // namespace zen diff --git a/src/zenserver/storage/projectstore/httpprojectstore.cpp b/src/zenserver/storage/projectstore/httpprojectstore.cpp new file mode 100644 index 000000000..1c6b5d6b0 --- /dev/null +++ b/src/zenserver/storage/projectstore/httpprojectstore.cpp @@ -0,0 +1,3307 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "httpprojectstore.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace zen { + +const FLLMTag& +GetProjectHttpTag() +{ + static FLLMTag _("http", FLLMTag("project")); + + return _; +} + +void +CSVHeader(bool Details, bool AttachmentDetails, StringBuilderBase& CSVWriter) +{ + if (AttachmentDetails) + { + CSVWriter << "Project, Oplog, LSN, Key, Cid, Size"; + } + else if (Details) + { + CSVWriter << "Project, Oplog, LSN, Key, Size, AttachmentCount, AttachmentsSize"; + } + else + { + CSVWriter << "Project, Oplog, Key"; + } +} + +void +CSVWriteOp(CidStore& CidStore, + std::string_view ProjectId, + std::string_view OplogId, + bool Details, + bool AttachmentDetails, + ProjectStore::LogSequenceNumber LSN, + const Oid& Key, + CbObjectView Op, + StringBuilderBase& CSVWriter) +{ + StringBuilder<32> KeyStringBuilder; + Key.ToString(KeyStringBuilder); + const std::string_view KeyString = KeyStringBuilder.ToView(); + + if (AttachmentDetails) + { + Op.IterateAttachments([&CidStore, &CSVWriter, &ProjectId, &OplogId, LSN, &KeyString](CbFieldView FieldView) { + const IoHash AttachmentHash = FieldView.AsAttachment(); + IoBuffer Attachment = CidStore.FindChunkByCid(AttachmentHash); + CSVWriter << "\r\n" + << ProjectId << ", " << OplogId << ", " << LSN.Number << ", " << KeyString << ", " << AttachmentHash.ToHexString() + << ", " << gsl::narrow(Attachment.GetSize()); + }); + } + else if (Details) + { + uint64_t AttachmentCount = 0; + size_t AttachmentsSize = 0; + Op.IterateAttachments([&CidStore, &AttachmentCount, &AttachmentsSize](CbFieldView FieldView) { + const IoHash AttachmentHash = FieldView.AsAttachment(); + AttachmentCount++; + IoBuffer Attachment = CidStore.FindChunkByCid(AttachmentHash); + AttachmentsSize += Attachment.GetSize(); + }); + CSVWriter << "\r\n" + << ProjectId << ", " << OplogId << ", " << LSN.Number << ", " << KeyString << ", " << gsl::narrow(Op.GetSize()) + << ", " << AttachmentCount << ", " << gsl::narrow(AttachmentsSize); + } + else + { + CSVWriter << "\r\n" << ProjectId << ", " << OplogId << ", " << KeyString; + } +}; + +////////////////////////////////////////////////////////////////////////// + +namespace { + + void CbWriteOp(CidStore& CidStore, + bool Details, + bool OpDetails, + bool AttachmentDetails, + ProjectStore::LogSequenceNumber LSN, + const Oid& Key, + CbObjectView Op, + CbObjectWriter& CbWriter) + { + CbWriter.BeginObject(); + { + CbWriter.AddObjectId("key", Key); + if (Details) + { + CbWriter.AddInteger("lsn", LSN.Number); + CbWriter.AddInteger("size", gsl::narrow(Op.GetSize())); + } + if (AttachmentDetails) + { + CbWriter.BeginArray("attachments"); + Op.IterateAttachments([&CidStore, &CbWriter](CbFieldView FieldView) { + const IoHash AttachmentHash = FieldView.AsAttachment(); + CbWriter.BeginObject(); + { + IoBuffer Attachment = CidStore.FindChunkByCid(AttachmentHash); + CbWriter.AddString("cid", AttachmentHash.ToHexString()); + CbWriter.AddInteger("size", gsl::narrow(Attachment.GetSize())); + } + CbWriter.EndObject(); + }); + CbWriter.EndArray(); + } + else if (Details) + { + uint64_t AttachmentCount = 0; + size_t AttachmentsSize = 0; + Op.IterateAttachments([&CidStore, &AttachmentCount, &AttachmentsSize](CbFieldView FieldView) { + const IoHash AttachmentHash = FieldView.AsAttachment(); + AttachmentCount++; + IoBuffer Attachment = CidStore.FindChunkByCid(AttachmentHash); + AttachmentsSize += Attachment.GetSize(); + }); + if (AttachmentCount > 0) + { + CbWriter.AddInteger("attachments", AttachmentCount); + CbWriter.AddInteger("attachmentssize", gsl::narrow(AttachmentsSize)); + } + } + if (OpDetails) + { + CbWriter.BeginObject("op"); + for (const CbFieldView& Field : Op) + { + if (!Field.HasName()) + { + CbWriter.AddField(Field); + continue; + } + std::string_view FieldName = Field.GetName(); + CbWriter.AddField(FieldName, Field); + } + CbWriter.EndObject(); + } + } + CbWriter.EndObject(); + }; + + void CbWriteOplogOps(CidStore& CidStore, + ProjectStore::Oplog& Oplog, + bool Details, + bool OpDetails, + bool AttachmentDetails, + CbObjectWriter& Cbo) + { + Cbo.BeginArray("ops"); + { + Oplog.IterateOplogWithKey([&Cbo, &CidStore, Details, OpDetails, AttachmentDetails](ProjectStore::LogSequenceNumber LSN, + const Oid& Key, + CbObjectView Op) { + CbWriteOp(CidStore, Details, OpDetails, AttachmentDetails, LSN, Key, Op, Cbo); + }); + } + Cbo.EndArray(); + } + + void CbWriteOplog(CidStore& CidStore, + ProjectStore::Oplog& Oplog, + bool Details, + bool OpDetails, + bool AttachmentDetails, + CbObjectWriter& Cbo) + { + Cbo.BeginObject(); + { + Cbo.AddString("name", Oplog.OplogId()); + CbWriteOplogOps(CidStore, Oplog, Details, OpDetails, AttachmentDetails, Cbo); + } + Cbo.EndObject(); + } + + void CbWriteOplogs(CidStore& CidStore, + ProjectStore::Project& Project, + std::vector OpLogs, + bool Details, + bool OpDetails, + bool AttachmentDetails, + CbObjectWriter& Cbo) + { + Cbo.BeginArray("oplogs"); + { + for (const std::string& OpLogId : OpLogs) + { + Ref Oplog = Project.OpenOplog(OpLogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ true); + if (Oplog) + { + CbWriteOplog(CidStore, *Oplog, Details, OpDetails, AttachmentDetails, Cbo); + } + } + } + Cbo.EndArray(); + } + + void CbWriteProject(CidStore& CidStore, + ProjectStore::Project& Project, + std::vector OpLogs, + bool Details, + bool OpDetails, + bool AttachmentDetails, + CbObjectWriter& Cbo) + { + Cbo.BeginObject(); + { + Cbo.AddString("name", Project.Identifier); + CbWriteOplogs(CidStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo); + } + Cbo.EndObject(); + } + + struct CreateRemoteStoreResult + { + std::shared_ptr Store; + std::string Description; + }; + + CreateRemoteStoreResult CreateRemoteStore(CbObjectView Params, + AuthMgr& AuthManager, + size_t MaxBlockSize, + size_t MaxChunkEmbedSize, + const std::filesystem::path& TempFilePath) + { + ZEN_MEMSCOPE(GetProjectHttpTag()); + + using namespace std::literals; + + std::shared_ptr RemoteStore; + + if (CbObjectView File = Params["file"sv].AsObjectView(); File) + { + std::filesystem::path FolderPath(File["path"sv].AsString()); + if (FolderPath.empty()) + { + return {nullptr, "Missing file path"}; + } + std::string_view Name(File["name"sv].AsString()); + if (Name.empty()) + { + return {nullptr, "Missing file name"}; + } + std::string_view OptionalBaseName(File["basename"sv].AsString()); + bool ForceDisableBlocks = File["disableblocks"sv].AsBool(false); + bool ForceEnableTempBlocks = File["enabletempblocks"sv].AsBool(false); + + FileRemoteStoreOptions Options = { + RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize}, + FolderPath, + std::string(Name), + std::string(OptionalBaseName), + ForceDisableBlocks, + ForceEnableTempBlocks}; + RemoteStore = CreateFileRemoteStore(Options); + } + + if (CbObjectView Cloud = Params["cloud"sv].AsObjectView(); Cloud) + { + std::string_view CloudServiceUrl = Cloud["url"sv].AsString(); + if (CloudServiceUrl.empty()) + { + return {nullptr, "Missing service url"}; + } + + std::string Url = UrlDecode(CloudServiceUrl); + std::string_view Namespace = Cloud["namespace"sv].AsString(); + if (Namespace.empty()) + { + return {nullptr, "Missing namespace"}; + } + std::string_view Bucket = Cloud["bucket"sv].AsString(); + if (Bucket.empty()) + { + return {nullptr, "Missing bucket"}; + } + std::string_view OpenIdProvider = Cloud["openid-provider"sv].AsString(); + std::string AccessToken = std::string(Cloud["access-token"sv].AsString()); + if (AccessToken.empty()) + { + std::string_view AccessTokenEnvVariable = Cloud["access-token-env"].AsString(); + if (!AccessTokenEnvVariable.empty()) + { + AccessToken = GetEnvVariable(AccessTokenEnvVariable); + } + } + std::filesystem::path OidcExePath; + if (std::string_view OidcExePathString = Cloud["oidc-exe-path"].AsString(); !OidcExePathString.empty()) + { + std::filesystem::path OidcExePathMaybe(OidcExePathString); + if (IsFile(OidcExePathMaybe)) + { + OidcExePath = std::move(OidcExePathMaybe); + } + else + { + ZEN_WARN("Path to OidcToken executable '{}' can not be reached by server", OidcExePathString); + } + } + std::string_view KeyParam = Cloud["key"sv].AsString(); + if (KeyParam.empty()) + { + return {nullptr, "Missing key"}; + } + if (KeyParam.length() != IoHash::StringLength) + { + return {nullptr, "Invalid key"}; + } + IoHash Key = IoHash::FromHexString(KeyParam); + if (Key == IoHash::Zero) + { + return {nullptr, "Invalid key string"}; + } + IoHash BaseKey = IoHash::Zero; + std::string_view BaseKeyParam = Cloud["basekey"sv].AsString(); + if (!BaseKeyParam.empty()) + { + if (BaseKeyParam.length() != IoHash::StringLength) + { + return {nullptr, "Invalid base key"}; + } + BaseKey = IoHash::FromHexString(BaseKeyParam); + if (BaseKey == IoHash::Zero) + { + return {nullptr, "Invalid base key string"}; + } + } + + bool ForceDisableBlocks = Cloud["disableblocks"sv].AsBool(false); + bool ForceDisableTempBlocks = Cloud["disabletempblocks"sv].AsBool(false); + bool AssumeHttp2 = Cloud["assumehttp2"sv].AsBool(false); + + JupiterRemoteStoreOptions Options = { + RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize}, + Url, + std::string(Namespace), + std::string(Bucket), + Key, + BaseKey, + std::string(OpenIdProvider), + AccessToken, + AuthManager, + OidcExePath, + ForceDisableBlocks, + ForceDisableTempBlocks, + AssumeHttp2}; + RemoteStore = CreateJupiterRemoteStore(Options, TempFilePath, /*Quiet*/ false, /*Unattended*/ false, /*Hidden*/ true); + } + + if (CbObjectView Zen = Params["zen"sv].AsObjectView(); Zen) + { + std::string_view Url = Zen["url"sv].AsString(); + std::string_view Project = Zen["project"sv].AsString(); + if (Project.empty()) + { + return {nullptr, "Missing project"}; + } + std::string_view Oplog = Zen["oplog"sv].AsString(); + if (Oplog.empty()) + { + return {nullptr, "Missing oplog"}; + } + ZenRemoteStoreOptions Options = { + RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize}, + std::string(Url), + std::string(Project), + std::string(Oplog)}; + RemoteStore = CreateZenRemoteStore(Options, TempFilePath); + } + + if (CbObjectView Builds = Params["builds"sv].AsObjectView(); Builds) + { + std::string_view BuildsServiceUrl = Builds["url"sv].AsString(); + if (BuildsServiceUrl.empty()) + { + return {nullptr, "Missing service url"}; + } + + std::string Url = UrlDecode(BuildsServiceUrl); + std::string_view Namespace = Builds["namespace"sv].AsString(); + if (Namespace.empty()) + { + return {nullptr, "Missing namespace"}; + } + std::string_view Bucket = Builds["bucket"sv].AsString(); + if (Bucket.empty()) + { + return {nullptr, "Missing bucket"}; + } + std::string_view OpenIdProvider = Builds["openid-provider"sv].AsString(); + std::string AccessToken = std::string(Builds["access-token"sv].AsString()); + if (AccessToken.empty()) + { + std::string_view AccessTokenEnvVariable = Builds["access-token-env"].AsString(); + if (!AccessTokenEnvVariable.empty()) + { + AccessToken = GetEnvVariable(AccessTokenEnvVariable); + } + } + std::filesystem::path OidcExePath; + if (std::string_view OidcExePathString = Builds["oidc-exe-path"].AsString(); !OidcExePathString.empty()) + { + std::filesystem::path OidcExePathMaybe(OidcExePathString); + if (IsFile(OidcExePathMaybe)) + { + OidcExePath = std::move(OidcExePathMaybe); + } + else + { + ZEN_WARN("Path to OidcToken executable '{}' can not be reached by server", OidcExePathString); + } + } + std::string_view BuildIdParam = Builds["buildsid"sv].AsString(); + if (BuildIdParam.empty()) + { + return {nullptr, "Missing build id"}; + } + if (BuildIdParam.length() != Oid::StringLength) + { + return {nullptr, "Invalid build id"}; + } + Oid BuildId = Oid::FromHexString(BuildIdParam); + if (BuildId == Oid::Zero) + { + return {nullptr, "Invalid build id string"}; + } + + bool ForceDisableBlocks = Builds["disableblocks"sv].AsBool(false); + bool ForceDisableTempBlocks = Builds["disabletempblocks"sv].AsBool(false); + bool AssumeHttp2 = Builds["assumehttp2"sv].AsBool(false); + + MemoryView MetaDataSection = Builds["metadata"sv].AsBinaryView(); + IoBuffer MetaData(IoBuffer::Wrap, MetaDataSection.GetData(), MetaDataSection.GetSize()); + + BuildsRemoteStoreOptions Options = { + RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize}, + Url, + std::string(Namespace), + std::string(Bucket), + BuildId, + std::string(OpenIdProvider), + AccessToken, + AuthManager, + OidcExePath, + ForceDisableBlocks, + ForceDisableTempBlocks, + AssumeHttp2, + MetaData}; + RemoteStore = CreateJupiterBuildsRemoteStore(Options, TempFilePath, /*Quiet*/ false, /*Unattended*/ false, /*Hidden*/ true); + } + + if (!RemoteStore) + { + return {nullptr, "Unknown remote store type"}; + } + + return {std::move(RemoteStore), ""}; + } + + std::pair ConvertResult(const RemoteProjectStore::Result& Result) + { + if (Result.ErrorCode == 0) + { + return {HttpResponseCode::OK, Result.Text}; + } + return {static_cast(Result.ErrorCode), + Result.Reason.empty() ? Result.Text + : Result.Text.empty() ? Result.Reason + : fmt::format("{}: {}", Result.Reason, Result.Text)}; + } + +} // namespace + +////////////////////////////////////////////////////////////////////////// + +HttpProjectService::HttpProjectService(CidStore& Store, + ProjectStore* Projects, + HttpStatusService& StatusService, + HttpStatsService& StatsService, + AuthMgr& AuthMgr, + OpenProcessCache& InOpenProcessCache, + JobQueue& InJobQueue) +: m_Log(logging::Get("project")) +, m_CidStore(Store) +, m_ProjectStore(Projects) +, m_StatusService(StatusService) +, m_StatsService(StatsService) +, m_AuthMgr(AuthMgr) +, m_OpenProcessCache(InOpenProcessCache) +, m_JobQueue(InJobQueue) +{ + ZEN_MEMSCOPE(GetProjectHttpTag()); + + using namespace std::literals; + + m_Router.AddPattern("project", "([[:alnum:]_.]+)"); + m_Router.AddPattern("log", "([[:alnum:]_.]+)"); + m_Router.AddPattern("op", "([[:digit:]]+?)"); + m_Router.AddPattern("chunk", "([[:xdigit:]]{24})"); + m_Router.AddPattern("hash", "([[:xdigit:]]{40})"); + + m_Router.RegisterRoute( + "", + [this](HttpRouterRequest& Req) { HandleProjectListRequest(Req); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "list", + [this](HttpRouterRequest& Req) { HandleProjectListRequest(Req); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "{project}/oplog/{log}/batch", + [this](HttpRouterRequest& Req) { HandleChunkBatchRequest(Req); }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "{project}/oplog/{log}/files", + [this](HttpRouterRequest& Req) { HandleFilesRequest(Req); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "{project}/oplog/{log}/chunkinfos", + [this](HttpRouterRequest& Req) { HandleChunkInfosRequest(Req); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "{project}/oplog/{log}/{chunk}/info", + [this](HttpRouterRequest& Req) { HandleChunkInfoRequest(Req); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "{project}/oplog/{log}/{chunk}", + [this](HttpRouterRequest& Req) { HandleChunkByIdRequest(Req); }, + HttpVerb::kGet | HttpVerb::kHead); + + m_Router.RegisterRoute( + "{project}/oplog/{log}/{hash}", + [this](HttpRouterRequest& Req) { HandleChunkByCidRequest(Req); }, + HttpVerb::kGet | HttpVerb::kPost); + + m_Router.RegisterRoute( + "{project}/oplog/{log}/prep", + [this](HttpRouterRequest& Req) { HandleOplogOpPrepRequest(Req); }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "{project}/oplog/{log}/new", + [this](HttpRouterRequest& Req) { HandleOplogOpNewRequest(Req); }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "{project}/oplog/{log}/validate", + [this](HttpRouterRequest& Req) { HandleOplogValidateRequest(Req); }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "{project}/oplog/{log}/{op}", + [this](HttpRouterRequest& Req) { HandleOpLogOpRequest(Req); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "{project}/oplog/{log}", + [this](HttpRouterRequest& Req) { HandleOpLogRequest(Req); }, + HttpVerb::kGet | HttpVerb::kPut | HttpVerb::kPost | HttpVerb::kDelete); + + m_Router.RegisterRoute( + "{project}/oplog/{log}/entries", + [this](HttpRouterRequest& Req) { HandleOpLogEntriesRequest(Req); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "{project}", + [this](HttpRouterRequest& Req) { HandleProjectRequest(Req); }, + HttpVerb::kGet | HttpVerb::kPut | HttpVerb::kPost | HttpVerb::kDelete); + + // Push a oplog container + m_Router.RegisterRoute( + "{project}/oplog/{log}/save", + [this](HttpRouterRequest& Req) { HandleOplogSaveRequest(Req); }, + HttpVerb::kPost); + + // Pull a oplog container + m_Router.RegisterRoute( + "{project}/oplog/{log}/load", + [this](HttpRouterRequest& Req) { HandleOplogLoadRequest(Req); }, + HttpVerb::kGet); + + // Do an rpc style operation on project/oplog + m_Router.RegisterRoute( + "{project}/oplog/{log}/rpc", + [this](HttpRouterRequest& Req) { HandleRpcRequest(Req); }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "details\\$", + [this](HttpRouterRequest& Req) { HandleDetailsRequest(Req); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "details\\$/{project}", + [this](HttpRouterRequest& Req) { HandleProjectDetailsRequest(Req); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "details\\$/{project}/{log}", + [this](HttpRouterRequest& Req) { HandleOplogDetailsRequest(Req); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "details\\$/{project}/{log}/{chunk}", + [this](HttpRouterRequest& Req) { HandleOplogOpDetailsRequest(Req); }, + HttpVerb::kGet); + + m_StatusService.RegisterHandler("prj", *this); + m_StatsService.RegisterHandler("prj", *this); +} + +HttpProjectService::~HttpProjectService() +{ + m_StatsService.UnregisterHandler("prj", *this); + m_StatusService.UnregisterHandler("prj", *this); +} + +const char* +HttpProjectService::BaseUri() const +{ + return "/prj/"; +} + +void +HttpProjectService::HandleRequest(HttpServerRequest& Request) +{ + m_ProjectStats.RequestCount++; + + ZEN_MEMSCOPE(GetProjectHttpTag()); + + metrics::OperationTiming::Scope $(m_HttpRequests); + + if (m_Router.HandleRequest(Request) == false) + { + m_ProjectStats.BadRequestCount++; + ZEN_WARN("No route found for {0}", Request.RelativeUri()); + } +} + +void +HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq) +{ + ZEN_TRACE_CPU("ProjectService::Stats"); + + const GcStorageSize StoreSize = m_ProjectStore->StorageSize(); + const CidStoreSize CidSize = m_CidStore.TotalSize(); + + CbObjectWriter Cbo; + + EmitSnapshot("requests", m_HttpRequests, Cbo); + + Cbo.BeginObject("store"); + { + Cbo.BeginObject("size"); + { + Cbo << "disk" << StoreSize.DiskSize; + Cbo << "memory" << StoreSize.MemorySize; + } + Cbo.EndObject(); + + Cbo.BeginObject("project"); + { + Cbo << "readcount" << m_ProjectStats.ProjectReadCount << "writecount" << m_ProjectStats.ProjectWriteCount << "deletecount" + << m_ProjectStats.ProjectDeleteCount; + } + Cbo.EndObject(); + + Cbo.BeginObject("oplog"); + { + Cbo << "readcount" << m_ProjectStats.OpLogReadCount << "writecount" << m_ProjectStats.OpLogWriteCount << "deletecount" + << m_ProjectStats.OpLogDeleteCount; + } + Cbo.EndObject(); + + Cbo.BeginObject("op"); + { + Cbo << "hitcount" << m_ProjectStats.OpHitCount << "misscount" << m_ProjectStats.OpMissCount << "writecount" + << m_ProjectStats.OpWriteCount; + } + Cbo.EndObject(); + + Cbo.BeginObject("chunk"); + { + Cbo << "hitcount" << m_ProjectStats.ChunkHitCount << "misscount" << m_ProjectStats.ChunkMissCount << "writecount" + << m_ProjectStats.ChunkWriteCount; + } + Cbo.EndObject(); + + Cbo << "requestcount" << m_ProjectStats.RequestCount; + Cbo << "badrequestcount" << m_ProjectStats.BadRequestCount; + } + Cbo.EndObject(); + + Cbo.BeginObject("cid"); + { + Cbo.BeginObject("size"); + { + Cbo << "tiny" << CidSize.TinySize; + Cbo << "small" << CidSize.SmallSize; + Cbo << "large" << CidSize.LargeSize; + Cbo << "total" << CidSize.TotalSize; + } + Cbo.EndObject(); + } + Cbo.EndObject(); + + return HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + +void +HttpProjectService::HandleStatusRequest(HttpServerRequest& Request) +{ + ZEN_TRACE_CPU("HttpProjectService::Status"); + CbObjectWriter Cbo; + Cbo << "ok" << true; + Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + +void +HttpProjectService::HandleProjectListRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("ProjectService::ProjectList"); + + HttpServerRequest& HttpReq = Req.ServerRequest(); + CbArray ProjectsList = m_ProjectStore->GetProjectsList(); + HttpReq.WriteResponse(HttpResponseCode::OK, ProjectsList); +} + +void +HttpProjectService::HandleChunkBatchRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("ProjectService::ChunkBatch"); + + HttpServerRequest& HttpReq = Req.ServerRequest(); + const auto& ProjectId = Req.GetCapture(1); + const auto& OplogId = Req.GetCapture(2); + + Ref Project = m_ProjectStore->OpenProject(ProjectId); + if (!Project) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound); + } + Project->TouchProject(); + + Ref FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ false); + if (!FoundLog) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound); + } + Project->TouchOplog(OplogId); + + // Parse Request + + IoBuffer Payload = HttpReq.ReadPayload(); + BinaryReader Reader(Payload); + + struct RequestHeader + { + enum + { + kMagic = 0xAAAA'77AC + }; + uint32_t Magic; + uint32_t ChunkCount; + uint32_t Reserved1; + uint32_t Reserved2; + }; + + struct RequestChunkEntry + { + Oid ChunkId; + uint32_t CorrelationId; + uint64_t Offset; + uint64_t RequestBytes; + }; + + if (Payload.Size() <= sizeof(RequestHeader)) + { + m_ProjectStats.BadRequestCount++; + HttpReq.WriteResponse(HttpResponseCode::BadRequest); + } + + RequestHeader RequestHdr; + Reader.Read(&RequestHdr, sizeof RequestHdr); + + if (RequestHdr.Magic != RequestHeader::kMagic) + { + m_ProjectStats.BadRequestCount++; + HttpReq.WriteResponse(HttpResponseCode::BadRequest); + } + + std::vector RequestedChunks; + RequestedChunks.resize(RequestHdr.ChunkCount); + Reader.Read(RequestedChunks.data(), sizeof(RequestChunkEntry) * RequestHdr.ChunkCount); + + // Make Response + + struct ResponseHeader + { + uint32_t Magic = 0xbada'b00f; + uint32_t ChunkCount; + uint32_t Reserved1 = 0; + uint32_t Reserved2 = 0; + }; + + struct ResponseChunkEntry + { + uint32_t CorrelationId; + uint32_t Flags = 0; + uint64_t ChunkSize; + }; + + std::vector OutBlobs; + OutBlobs.emplace_back(sizeof(ResponseHeader) + RequestHdr.ChunkCount * sizeof(ResponseChunkEntry)); + for (uint32_t ChunkIndex = 0; ChunkIndex < RequestHdr.ChunkCount; ++ChunkIndex) + { + const RequestChunkEntry& RequestedChunk = RequestedChunks[ChunkIndex]; + IoBuffer FoundChunk = FoundLog->FindChunk(Project->RootDir, RequestedChunk.ChunkId, nullptr); + if (FoundChunk) + { + if (RequestedChunk.Offset > 0 || RequestedChunk.RequestBytes < uint64_t(-1)) + { + uint64_t Offset = RequestedChunk.Offset; + if (Offset > FoundChunk.Size()) + { + Offset = FoundChunk.Size(); + } + uint64_t Size = RequestedChunk.RequestBytes; + if ((Offset + Size) > FoundChunk.Size()) + { + Size = FoundChunk.Size() - Offset; + } + FoundChunk = IoBuffer(FoundChunk, Offset, Size); + } + } + OutBlobs.emplace_back(std::move(FoundChunk)); + } + uint8_t* ResponsePtr = reinterpret_cast(OutBlobs[0].MutableData()); + ResponseHeader ResponseHdr; + ResponseHdr.ChunkCount = RequestHdr.ChunkCount; + memcpy(ResponsePtr, &ResponseHdr, sizeof(ResponseHdr)); + ResponsePtr += sizeof(ResponseHdr); + for (uint32_t ChunkIndex = 0; ChunkIndex < RequestHdr.ChunkCount; ++ChunkIndex) + { + const RequestChunkEntry& RequestedChunk = RequestedChunks[ChunkIndex]; + const IoBuffer& FoundChunk(OutBlobs[ChunkIndex + 1]); + ResponseChunkEntry ResponseChunk; + ResponseChunk.CorrelationId = RequestedChunk.CorrelationId; + if (FoundChunk) + { + ResponseChunk.ChunkSize = FoundChunk.Size(); + m_ProjectStats.ChunkHitCount++; + } + else + { + ResponseChunk.ChunkSize = uint64_t(-1); + m_ProjectStats.ChunkMissCount++; + } + memcpy(ResponsePtr, &ResponseChunk, sizeof(ResponseChunk)); + ResponsePtr += sizeof(ResponseChunk); + } + std::erase_if(OutBlobs, [](IoBuffer Buffer) -> bool { return !Buffer; }); + return HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, OutBlobs); +} + +void +HttpProjectService::HandleFilesRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("ProjectService::Files"); + + using namespace std::literals; + + HttpServerRequest& HttpReq = Req.ServerRequest(); + + // File manifest fetch, returns the client file list + + const auto& ProjectId = Req.GetCapture(1); + const auto& OplogId = Req.GetCapture(2); + + HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); + + std::unordered_set WantedFieldNames; + if (auto FieldFilter = HttpServerRequest::Decode(Params.GetValue("fieldnames")); !FieldFilter.empty()) + { + if (FieldFilter != "*") // Get all - empty FieldFilter equal getting all fields + { + ForEachStrTok(FieldFilter, ',', [&](std::string_view FieldName) { + WantedFieldNames.insert(std::string(FieldName)); + return true; + }); + } + } + else + { + const bool FilterClient = Params.GetValue("filter"sv) == "client"sv; + WantedFieldNames.insert("id"); + WantedFieldNames.insert("clientpath"); + if (!FilterClient) + { + WantedFieldNames.insert("serverpath"); + } + } + + Ref Project = m_ProjectStore->OpenProject(ProjectId); + if (!Project) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Project files request for unknown project '{}'", ProjectId)); + } + Project->TouchProject(); + + Ref FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ true); + if (!FoundLog) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Project files for unknown oplog '{}/{}'", ProjectId, OplogId)); + } + Project->TouchOplog(OplogId); + + CbObject ResponsePayload = ProjectStore::GetProjectFiles(Log(), *Project, *FoundLog, WantedFieldNames); + + if (HttpReq.AcceptContentType() == HttpContentType::kCompressedBinary) + { + CompositeBuffer Payload = CompressedBuffer::Compress(ResponsePayload.GetBuffer()).GetCompressed(); + return HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kCompressedBinary, Payload); + } + else + { + return HttpReq.WriteResponse(HttpResponseCode::OK, ResponsePayload); + } +} + +void +HttpProjectService::HandleChunkInfosRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("ProjectService::ChunkInfos"); + + HttpServerRequest& HttpReq = Req.ServerRequest(); + + const auto& ProjectId = Req.GetCapture(1); + const auto& OplogId = Req.GetCapture(2); + + HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); + + std::unordered_set WantedFieldNames; + if (auto FieldFilter = HttpServerRequest::Decode(Params.GetValue("fieldnames")); !FieldFilter.empty()) + { + if (FieldFilter != "*") // Get all - empty FieldFilter equal getting all fields + { + ForEachStrTok(FieldFilter, ',', [&](std::string_view FieldName) { + WantedFieldNames.insert(std::string(FieldName)); + return true; + }); + } + } + else + { + WantedFieldNames.insert("id"); + WantedFieldNames.insert("rawhash"); + WantedFieldNames.insert("rawsize"); + } + + Ref Project = m_ProjectStore->OpenProject(ProjectId); + if (!Project) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Chunk infos request for unknown project '{}'", ProjectId)); + } + Project->TouchProject(); + + Ref FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ true); + if (!FoundLog) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Chunk infos for unknown oplog '{}/{}'", ProjectId, OplogId)); + } + Project->TouchOplog(OplogId); + + CbObject ResponsePayload = ProjectStore::GetProjectChunkInfos(Log(), *Project, *FoundLog, WantedFieldNames); + if (HttpReq.AcceptContentType() == HttpContentType::kCompressedBinary) + { + CompositeBuffer Payload = CompressedBuffer::Compress(ResponsePayload.GetBuffer()).GetCompressed(); + return HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kCompressedBinary, Payload); + } + else + { + return HttpReq.WriteResponse(HttpResponseCode::OK, ResponsePayload); + } +} + +void +HttpProjectService::HandleChunkInfoRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("ProjectService::ChunkInfo"); + + HttpServerRequest& HttpReq = Req.ServerRequest(); + + const auto& ProjectId = Req.GetCapture(1); + const auto& OplogId = Req.GetCapture(2); + const auto& ChunkId = Req.GetCapture(3); + + Ref Project = m_ProjectStore->OpenProject(ProjectId); + if (!Project) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Chunk info request for unknown project '{}'", ProjectId)); + } + Project->TouchProject(); + + Ref FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ true); + if (!FoundLog) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Chunk info for unknown oplog '{}/{}'", ProjectId, OplogId)); + } + Project->TouchOplog(OplogId); + + if (ChunkId.size() != 2 * sizeof(Oid::OidBits)) + { + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Chunk info request for invalid chunk id '{}/{}'/'{}'", ProjectId, OplogId, ChunkId)); + } + + const Oid Obj = Oid::FromHexString(ChunkId); + + CbObject ResponsePayload = ProjectStore::GetChunkInfo(Log(), *Project, *FoundLog, Obj); + if (ResponsePayload) + { + m_ProjectStats.ChunkHitCount++; + return HttpReq.WriteResponse(HttpResponseCode::OK, ResponsePayload); + } + else + { + m_ProjectStats.ChunkMissCount++; + ZEN_DEBUG("chunk - '{}/{}/{}' MISSING", ProjectId, OplogId, ChunkId); + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Chunk info for unknown chunk '{}/{}/{}'", ProjectId, OplogId, ChunkId)); + } +} + +void +HttpProjectService::HandleChunkByIdRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("ProjectService::ChunkById"); + + HttpServerRequest& HttpReq = Req.ServerRequest(); + + const auto& ProjectId = Req.GetCapture(1); + const auto& OplogId = Req.GetCapture(2); + const auto& ChunkId = Req.GetCapture(3); + + uint64_t Offset = 0; + uint64_t Size = ~(0ull); + + auto QueryParms = HttpReq.GetQueryParams(); + + if (auto OffsetParm = QueryParms.GetValue("offset"); OffsetParm.empty() == false) + { + if (auto OffsetVal = ParseInt(OffsetParm)) + { + Offset = OffsetVal.value(); + } + else + { + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest); + } + } + + if (auto SizeParm = QueryParms.GetValue("size"); SizeParm.empty() == false) + { + if (auto SizeVal = ParseInt(SizeParm)) + { + Size = SizeVal.value(); + } + else + { + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest); + } + } + + Ref Project = m_ProjectStore->OpenProject(ProjectId); + if (!Project) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Chunk request for unknown project '{}'", ProjectId)); + } + Project->TouchProject(); + + Ref FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ false); + if (!FoundLog) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Chunk request for unknown oplog '{}/{}'", ProjectId, OplogId)); + } + Project->TouchOplog(OplogId); + + if (ChunkId.size() != 2 * sizeof(Oid::OidBits)) + { + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Chunk request for invalid chunk id '{}/{}/{}'", ProjectId, OplogId, ChunkId)); + } + + const Oid Obj = Oid::FromHexString(ChunkId); + + HttpContentType AcceptType = HttpReq.AcceptContentType(); + + ProjectStore::GetChunkRangeResult Result = + ProjectStore::GetChunkRange(Log(), *Project, *FoundLog, Obj, Offset, Size, AcceptType, /*OptionalInOutModificationTag*/ nullptr); + + switch (Result.Error) + { + case ProjectStore::GetChunkRangeResult::EError::Ok: + m_ProjectStats.ChunkHitCount++; + ZEN_DEBUG("chunk - '{}/{}/{}' '{}'", ProjectId, OplogId, ChunkId, ToString(Result.ContentType)); + return HttpReq.WriteResponse(HttpResponseCode::OK, Result.ContentType, Result.Chunk); + case ProjectStore::GetChunkRangeResult::EError::NotFound: + m_ProjectStats.ChunkMissCount++; + ZEN_DEBUG("chunk - '{}/{}/{}' MISSING", ProjectId, OplogId, ChunkId); + return HttpReq.WriteResponse(HttpResponseCode::NotFound, Result.ContentType, Result.Chunk); + case ProjectStore::GetChunkRangeResult::EError::MalformedContent: + return HttpReq.WriteResponse( + HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Get chunk {}/{}/{} failed. Reason: {}", ProjectId, OplogId, ChunkId, Result.ErrorDescription)); + case ProjectStore::GetChunkRangeResult::EError::OutOfRange: + m_ProjectStats.ChunkMissCount++; + ZEN_DEBUG("chunk - '{}/{}/{}' OUT OF RANGE", ProjectId, OplogId, ChunkId); + return HttpReq.WriteResponse( + HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Get chunk {}/{}/{} failed. Reason: {}", ProjectId, OplogId, ChunkId, Result.ErrorDescription)); + default: + ZEN_ASSERT(false); + break; + } +} + +void +HttpProjectService::HandleChunkByCidRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("ProjectService::ChunkByCid"); + + HttpServerRequest& HttpReq = Req.ServerRequest(); + + const auto& ProjectId = Req.GetCapture(1); + const auto& OplogId = Req.GetCapture(2); + const auto& Cid = Req.GetCapture(3); + HttpContentType AcceptType = HttpReq.AcceptContentType(); + HttpContentType RequestType = HttpReq.RequestContentType(); + + Ref Project = m_ProjectStore->OpenProject(ProjectId); + if (!Project) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Chunk request for unknown project '{}'", ProjectId)); + } + Project->TouchProject(); + + Ref FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ false); + if (!FoundLog) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Chunk request for unknown oplog '{}/{}'", ProjectId, OplogId)); + } + Project->TouchOplog(OplogId); + + if (Cid.length() != IoHash::StringLength) + { + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Chunk request for invalid chunk id '{}/{}/{}'", ProjectId, OplogId, Cid)); + } + + const IoHash Hash = IoHash::FromHexString(Cid); + + switch (HttpReq.RequestVerb()) + { + case HttpVerb::kGet: + { + IoBuffer Value = m_ProjectStore->GetChunk(*Project, *FoundLog, Hash); + if (Value) + { + if (AcceptType == ZenContentType::kUnknownContentType || AcceptType == ZenContentType::kBinary || + AcceptType == ZenContentType::kJSON || AcceptType == ZenContentType::kYAML || + AcceptType == ZenContentType::kCbObject) + { + CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(Value)); + IoBuffer DecompressedBuffer = Compressed.Decompress().AsIoBuffer(); + + if (DecompressedBuffer) + { + if (AcceptType == ZenContentType::kJSON || AcceptType == ZenContentType::kYAML || + AcceptType == ZenContentType::kCbObject) + { + CbValidateError CbErr = ValidateCompactBinary(DecompressedBuffer.GetView(), CbValidateMode::Default); + if (!!CbErr) + { + m_ProjectStats.BadRequestCount++; + ZEN_DEBUG( + "chunk - '{}/{}/{}' WRONGTYPE. Reason: `Requested {} format, but could not convert to object`", + ProjectId, + OplogId, + Cid, + ToString(AcceptType)); + return HttpReq.WriteResponse( + HttpResponseCode::NotAcceptable, + HttpContentType::kText, + fmt::format("Content format not supported, requested {} format, but could not convert to object", + ToString(AcceptType))); + } + + m_ProjectStats.ChunkHitCount++; + CbObject ContainerObject = LoadCompactBinaryObject(DecompressedBuffer); + return HttpReq.WriteResponse(HttpResponseCode::OK, ContainerObject); + } + else + { + Value = DecompressedBuffer; + Value.SetContentType(ZenContentType::kBinary); + } + } + else + { + m_ProjectStats.BadRequestCount++; + ZEN_DEBUG("chunk - '{}/{}/{}' WRONGTYPE. Reason: `Requested {} format, but could not decompress stored data`", + ProjectId, + OplogId, + Cid, + ToString(AcceptType)); + return HttpReq.WriteResponse( + HttpResponseCode::NotAcceptable, + HttpContentType::kText, + fmt::format("Content format not supported, requested {} format, but could not decompress stored data", + ToString(AcceptType))); + } + } + m_ProjectStats.ChunkHitCount++; + return HttpReq.WriteResponse(HttpResponseCode::OK, Value.GetContentType(), Value); + } + else + { + m_ProjectStats.ChunkMissCount++; + ZEN_DEBUG("chunk - '{}/{}/{}' MISSING", ProjectId, OplogId, Cid); + return HttpReq.WriteResponse(HttpResponseCode::NotFound); + } + } + case HttpVerb::kPost: + { + if (!m_ProjectStore->AreDiskWritesAllowed()) + { + return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); + } + if (RequestType != HttpContentType::kCompressedBinary) + { + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Chunk request for chunk id '{}/{}'/'{}' as unexpected content type: '{}'", + ProjectId, + OplogId, + Cid, + ToString(RequestType))); + } + IoBuffer Payload = HttpReq.ReadPayload(); + Payload.SetContentType(RequestType); + bool IsNew = m_ProjectStore->PutChunk(*Project, *FoundLog, Hash, std::move(Payload)); + + m_ProjectStats.ChunkWriteCount++; + return HttpReq.WriteResponse(IsNew ? HttpResponseCode::Created : HttpResponseCode::OK); + } + break; + } +} + +void +HttpProjectService::HandleOplogOpPrepRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("ProjectService::OplogOpPrep"); + + using namespace std::literals; + + HttpServerRequest& HttpReq = Req.ServerRequest(); + + const auto& ProjectId = Req.GetCapture(1); + const auto& OplogId = Req.GetCapture(2); + + Ref Project = m_ProjectStore->OpenProject(ProjectId); + if (!Project) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound); + } + Project->TouchProject(); + + Ref FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ false); + if (!FoundLog) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound); + } + Project->TouchOplog(OplogId); + + // This operation takes a list of referenced hashes and decides which + // chunks are not present on this server. This list is then returned in + // the "need" list in the response + + CbValidateError ValidateResult; + if (CbObject RequestObject = ValidateAndReadCompactBinaryObject(HttpReq.ReadPayload(), ValidateResult); + ValidateResult == CbValidateError::None) + { + std::vector NeedList; + + { + eastl::fixed_vector ChunkList; + CbArrayView HaveList = RequestObject["have"sv].AsArrayView(); + ChunkList.reserve(HaveList.Num()); + for (auto& Entry : HaveList) + { + ChunkList.push_back(Entry.AsHash()); + } + + NeedList = FoundLog->CheckPendingChunkReferences(std::span(begin(ChunkList), end(ChunkList)), std::chrono::minutes(2)); + } + + CbObjectWriter Cbo(1 + 1 + 5 + NeedList.size() * (1 + sizeof(IoHash::Hash)) + 1); + Cbo.BeginArray("need"); + { + for (const IoHash& Hash : NeedList) + { + ZEN_DEBUG("prep - NEED: {}", Hash); + Cbo << Hash; + } + } + Cbo.EndArray(); + CbObject Response = Cbo.Save(); + + return HttpReq.WriteResponse(HttpResponseCode::OK, Response); + } + else + { + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid compact binary format: '{}'", ToString(ValidateResult))); + } +} + +void +HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("ProjectService::OplogOpNew"); + + using namespace std::literals; + + HttpServerRequest& HttpReq = Req.ServerRequest(); + + if (!m_ProjectStore->AreDiskWritesAllowed()) + { + return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); + } + + const auto& ProjectId = Req.GetCapture(1); + const auto& OplogId = Req.GetCapture(2); + + HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); + + bool IsUsingSalt = false; + IoHash SaltHash = IoHash::Zero; + + if (std::string_view SaltParam = Params.GetValue("salt"sv); SaltParam.empty() == false) + { + const uint32_t Salt = std::stoi(std::string(SaltParam)); + SaltHash = IoHash::HashBuffer(&Salt, sizeof Salt); + IsUsingSalt = true; + } + + Ref Project = m_ProjectStore->OpenProject(ProjectId); + if (!Project) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound); + } + Project->TouchProject(); + + Ref FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ false); + if (!FoundLog) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound); + } + Project->TouchOplog(OplogId); + + ProjectStore::Oplog& Oplog = *FoundLog; + + IoBuffer Payload = HttpReq.ReadPayload(); + + // This will attempt to open files which may not exist for the case where + // the prep step rejected the chunk. This should be fixed since there's + // a performance cost associated with any file system activity + + bool IsValid = true; + std::vector MissingChunks; + + CbPackage::AttachmentResolver Resolver = [&](const IoHash& Hash) -> SharedBuffer { + if (m_CidStore.ContainsChunk(Hash)) + { + // Return null attachment as we already have it, no point in reading it and storing it again + return {}; + } + + IoHash AttachmentId; + if (IsUsingSalt) + { + IoHash AttachmentSpec[]{SaltHash, Hash}; + AttachmentId = IoHash::HashBuffer(MakeMemoryView(AttachmentSpec)); + } + else + { + AttachmentId = Hash; + } + + std::filesystem::path AttachmentPath = Oplog.TempPath() / AttachmentId.ToHexString(); + if (IoBuffer Data = IoBufferBuilder::MakeFromTemporaryFile(AttachmentPath)) + { + Data.SetDeleteOnClose(true); + return SharedBuffer(std::move(Data)); + } + else + { + IsValid = false; + MissingChunks.push_back(Hash); + + return {}; + } + }; + + CbPackage Package; + + if (!legacy::TryLoadCbPackage(Package, Payload, &UniqueBuffer::Alloc, &Resolver)) + { + CbValidateError ValidateResult; + if (CbObject Core = ValidateAndReadCompactBinaryObject(IoBuffer(Payload), ValidateResult); + ValidateResult == CbValidateError::None && Core) + { + Package.SetObject(Core); + } + else + { + std::filesystem::path BadPackagePath = + Oplog.TempPath() / "bad_packages"sv / fmt::format("session{}_request{}"sv, HttpReq.SessionId(), HttpReq.RequestId()); + + ZEN_WARN("Received malformed package ('{}')! Saving payload to '{}'", ToString(ValidateResult), BadPackagePath); + + WriteFile(BadPackagePath, Payload); + + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + u8"request body must be a compact binary object or package in legacy format"); + } + } + + m_ProjectStats.ChunkMissCount += MissingChunks.size(); + + if (!IsValid) + { + ExtendableStringBuilder<256> ResponseText; + ResponseText.Append("Missing chunk references: "); + + bool IsFirst = true; + for (const auto& Hash : MissingChunks) + { + if (IsFirst) + { + IsFirst = false; + } + else + { + ResponseText.Append(", "); + } + Hash.ToHexString(ResponseText); + } + + return HttpReq.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, ResponseText); + } + + CbObject Core = Package.GetObject(); + + if (!Core["key"sv]) + { + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "No oplog entry key specified"); + } + + eastl::fixed_vector ReferencedChunks; + Core.IterateAttachments([&ReferencedChunks](CbFieldView View) { ReferencedChunks.push_back(View.AsAttachment()); }); + + // Write core to oplog + + size_t AttachmentCount = Package.GetAttachments().size(); + const ProjectStore::LogSequenceNumber OpLsn = Oplog.AppendNewOplogEntry(Package); + if (!OpLsn) + { + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest); + } + m_ProjectStats.ChunkWriteCount += AttachmentCount; + + // Once we stored the op, we no longer need to retain any chunks this op references + if (!ReferencedChunks.empty()) + { + FoundLog->RemovePendingChunkReferences(std::span(begin(ReferencedChunks), end(ReferencedChunks))); + } + + m_ProjectStats.OpWriteCount++; + ZEN_DEBUG("'{}/{}' op #{} ({}) - '{}'", ProjectId, OplogId, OpLsn.Number, NiceBytes(Payload.Size()), Core["key"sv].AsString()); + HttpReq.WriteResponse(HttpResponseCode::Created); +} + +void +HttpProjectService::HandleOplogValidateRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("ProjectService::OplogOpValidate"); + + using namespace std::literals; + + HttpServerRequest& HttpReq = Req.ServerRequest(); + + if (!m_ProjectStore->AreDiskWritesAllowed()) + { + return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); + } + + const auto& ProjectId = Req.GetCapture(1); + const auto& OplogId = Req.GetCapture(2); + + Ref Project = m_ProjectStore->OpenProject(ProjectId); + if (!Project) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, ZenContentType::kText, fmt::format("Project '{}' not found", ProjectId)); + } + Project->TouchProject(); + + Ref FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ true); + if (!FoundLog) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + ZenContentType::kText, + fmt::format("Oplog '{}' not found in project '{}'", OplogId, ProjectId)); + } + Project->TouchOplog(OplogId); + + ProjectStore::Oplog& Oplog = *FoundLog; + + std::atomic_bool CancelFlag = false; + ProjectStore::Oplog::ValidationResult Result = Oplog.Validate(Project->RootDir, CancelFlag, &GetSmallWorkerPool(EWorkloadType::Burst)); + tsl::robin_map KeyNameLookup; + KeyNameLookup.reserve(Result.OpKeys.size()); + for (const auto& It : Result.OpKeys) + { + KeyNameLookup.insert_or_assign(It.first, It.second); + } + CbObjectWriter Writer; + Writer << "HasMissingData" << !Result.IsEmpty(); + Writer << "OpCount" << Result.OpCount; + Writer << "LSNLow" << Result.LSNLow.Number; + Writer << "LSNHigh" << Result.LSNHigh.Number; + if (!Result.MissingFiles.empty()) + { + Writer.BeginArray("MissingFiles"); + for (const auto& MissingFile : Result.MissingFiles) + { + Writer.BeginObject(); + { + Writer << "Key" << MissingFile.first; + Writer << "KeyName" << KeyNameLookup[MissingFile.first]; + Writer << "Id" << MissingFile.second.Id; + Writer << "Hash" << MissingFile.second.Hash; + Writer << "ServerPath" << MissingFile.second.ServerPath; + Writer << "ClientPath" << MissingFile.second.ClientPath; + } + Writer.EndObject(); + } + Writer.EndArray(); + } + if (!Result.MissingChunks.empty()) + { + Writer.BeginArray("MissingChunks"); + for (const auto& MissingChunk : Result.MissingChunks) + { + Writer.BeginObject(); + { + Writer << "Key" << MissingChunk.first; + Writer << "KeyName" << KeyNameLookup[MissingChunk.first]; + Writer << "Id" << MissingChunk.second.Id; + Writer << "Hash" << MissingChunk.second.Hash; + } + Writer.EndObject(); + } + Writer.EndArray(); + } + if (!Result.MissingMetas.empty()) + { + Writer.BeginArray("MissingMetas"); + for (const auto& MissingMeta : Result.MissingMetas) + { + Writer.BeginObject(); + { + Writer << "Key" << MissingMeta.first; + Writer << "KeyName" << KeyNameLookup[MissingMeta.first]; + Writer << "Id" << MissingMeta.second.Id; + Writer << "Hash" << MissingMeta.second.Hash; + } + Writer.EndObject(); + } + Writer.EndArray(); + } + if (!Result.MissingAttachments.empty()) + { + Writer.BeginArray("MissingAttachments"); + for (const auto& MissingMeta : Result.MissingAttachments) + { + Writer.BeginObject(); + { + Writer << "Key" << MissingMeta.first; + Writer << "KeyName" << KeyNameLookup[MissingMeta.first]; + Writer << "Hash" << MissingMeta.second; + } + Writer.EndObject(); + } + Writer.EndArray(); + } + CbObject Response = Writer.Save(); + HttpReq.WriteResponse(HttpResponseCode::OK, Response); +} + +void +HttpProjectService::HandleOpLogOpRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("ProjectService::OplogOp"); + + HttpServerRequest& HttpReq = Req.ServerRequest(); + + const std::string_view ProjectId = Req.GetCapture(1); + const std::string_view OplogId = Req.GetCapture(2); + const std::string_view OpIdString = Req.GetCapture(3); + + Ref Project = m_ProjectStore->OpenProject(ProjectId); + if (!Project) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound); + } + Project->TouchProject(); + + Ref FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ false); + if (!FoundLog) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound); + } + Project->TouchOplog(OplogId); + + ProjectStore::Oplog& Oplog = *FoundLog; + + if (const std::optional OpId = ParseInt(OpIdString)) + { + if (std::optional MaybeOp = Oplog.GetOpByIndex(ProjectStore::LogSequenceNumber(OpId.value()))) + { + CbObject& Op = MaybeOp.value(); + if (HttpReq.AcceptContentType() == ZenContentType::kCbPackage) + { + CbPackage Package; + Package.SetObject(Op); + + Op.IterateAttachments([&](CbFieldView FieldView) { + const IoHash AttachmentHash = FieldView.AsAttachment(); + IoBuffer Payload = m_CidStore.FindChunkByCid(AttachmentHash); + if (Payload) + { + switch (Payload.GetContentType()) + { + case ZenContentType::kCbObject: + { + CbValidateError ValidateResult; + if (CbObject Object = ValidateAndReadCompactBinaryObject(std::move(Payload), ValidateResult); + ValidateResult == CbValidateError::None && Object) + { + Package.AddAttachment(CbAttachment(Object)); + } + else + { + // Error - malformed object + ZEN_WARN("malformed object returned for {} ('{}')", AttachmentHash, ToString(ValidateResult)); + } + } + break; + + case ZenContentType::kCompressedBinary: + if (CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(Payload))) + { + Package.AddAttachment(CbAttachment(Compressed, AttachmentHash)); + } + else + { + // Error - not compressed! + + ZEN_WARN("invalid compressed binary returned for {}", AttachmentHash); + } + break; + + default: + Package.AddAttachment(CbAttachment(SharedBuffer(Payload))); + break; + } + } + }); + m_ProjectStats.OpHitCount++; + return HttpReq.WriteResponse(HttpResponseCode::Accepted, Package); + } + else + { + // Client cannot accept a package, so we only send the core object + m_ProjectStats.OpHitCount++; + return HttpReq.WriteResponse(HttpResponseCode::Accepted, Op); + } + } + } + m_ProjectStats.OpMissCount++; + return HttpReq.WriteResponse(HttpResponseCode::NotFound); +} + +void +HttpProjectService::HandleOpLogRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("ProjectService::Oplog"); + + HttpServerRequest& HttpReq = Req.ServerRequest(); + + using namespace std::literals; + + const auto& ProjectId = Req.GetCapture(1); + const auto& OplogId = Req.GetCapture(2); + + Ref Project = m_ProjectStore->OpenProject(ProjectId); + + if (!Project) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, fmt::format("project {} not found", ProjectId)); + } + Project->TouchProject(); + + switch (HttpReq.RequestVerb()) + { + case HttpVerb::kGet: + { + Ref OplogIt = Project->ReadOplog(OplogId); + if (!OplogIt) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("oplog {} not found in project {}", OplogId, ProjectId)); + } + + ProjectStore::Oplog& Log = *OplogIt; + + CbObjectWriter Cb; + Cb << "id"sv << Log.OplogId() << "project"sv << Project->Identifier << "tempdir"sv << Log.TempPath().c_str() + << "markerpath"sv << Log.MarkerPath().c_str() << "totalsize"sv << Log.TotalSize() << "opcount" << Log.OplogCount() + << "expired"sv << Project->IsExpired(GcClock::TimePoint::min(), Log); + HttpReq.WriteResponse(HttpResponseCode::OK, Cb.Save()); + + m_ProjectStats.OpLogReadCount++; + } + break; + + case HttpVerb::kPost: + { + if (!m_ProjectStore->AreDiskWritesAllowed()) + { + return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); + } + std::filesystem::path OplogMarkerPath; + if (CbObject Params = HttpReq.ReadPayloadObject()) + { + OplogMarkerPath = Params["gcpath"sv].AsString(); + } + + Ref OplogIt = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ true); + if (!OplogIt) + { + if (!Project->NewOplog(OplogId, OplogMarkerPath)) + { + // TODO: indicate why the operation failed! + return HttpReq.WriteResponse(HttpResponseCode::InternalServerError); + } + Project->TouchOplog(OplogId); + + m_ProjectStats.OpLogWriteCount++; + ZEN_INFO("established oplog '{}/{}', gc marker file at '{}'", ProjectId, OplogId, OplogMarkerPath); + + return HttpReq.WriteResponse(HttpResponseCode::Created); + } + + // I guess this should ultimately be used to execute RPCs but for now, it + // does absolutely nothing + + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest); + } + break; + + case HttpVerb::kPut: + { + if (!m_ProjectStore->AreDiskWritesAllowed()) + { + return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); + } + + std::filesystem::path OplogMarkerPath; + if (CbObject Params = HttpReq.ReadPayloadObject()) + { + OplogMarkerPath = Params["gcpath"sv].AsString(); + } + + Ref FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ true); + if (!FoundLog) + { + if (!Project->NewOplog(OplogId, OplogMarkerPath)) + { + // TODO: indicate why the operation failed! + return HttpReq.WriteResponse(HttpResponseCode::InternalServerError); + } + Project->TouchOplog(OplogId); + + m_ProjectStats.OpLogWriteCount++; + ZEN_INFO("established oplog '{}/{}', gc marker file at '{}'", ProjectId, OplogId, OplogMarkerPath); + + return HttpReq.WriteResponse(HttpResponseCode::Created); + } + Project->TouchOplog(OplogId); + + FoundLog->Update(OplogMarkerPath); + + m_ProjectStats.OpLogWriteCount++; + ZEN_INFO("updated oplog '{}/{}', gc marker file at '{}'", ProjectId, OplogId, OplogMarkerPath); + + return HttpReq.WriteResponse(HttpResponseCode::OK); + } + break; + + case HttpVerb::kDelete: + { + ZEN_INFO("deleting oplog '{}/{}'", ProjectId, OplogId); + + if (Project->DeleteOplog(OplogId)) + { + m_ProjectStats.OpLogDeleteCount++; + return HttpReq.WriteResponse(HttpResponseCode::OK); + } + else + { + return HttpReq.WriteResponse(HttpResponseCode::Locked, + HttpContentType::kText, + fmt::format("oplog {}/{} is in use", ProjectId, OplogId)); + } + } + break; + + default: + break; + } +} + +std::optional +LoadReferencedSet(ProjectStore::Project& Project, ProjectStore::Oplog& Log) +{ + using namespace std::literals; + + Oid ReferencedSetOplogId = OpKeyStringAsOid(OplogReferencedSet::ReferencedSetOplogKey); + std::optional ReferencedSetOp = Log.GetOpByKey(ReferencedSetOplogId); + if (!ReferencedSetOp) + { + return std::optional(); + } + // We expect only a single file in the "files" array; get the chunk for the first file + CbFieldView FileField = *(*ReferencedSetOp)["files"sv].AsArrayView().CreateViewIterator(); + Oid ChunkId = FileField.AsObjectView()["id"sv].AsObjectId(); + if (ChunkId == Oid::Zero) + { + return std::optional(); + } + + return OplogReferencedSet::LoadFromChunk(Log.FindChunk(Project.RootDir, ChunkId, nullptr)); +} + +void +HttpProjectService::HandleOpLogEntriesRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("ProjectService::OplogEntries"); + + using namespace std::literals; + + HttpServerRequest& HttpReq = Req.ServerRequest(); + + const auto& ProjectId = Req.GetCapture(1); + const auto& OplogId = Req.GetCapture(2); + + Ref Project = m_ProjectStore->OpenProject(ProjectId); + if (!Project) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound); + } + Project->TouchProject(); + + Ref FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ true); + if (!FoundLog) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound); + } + Project->TouchOplog(OplogId); + + CbObjectWriter Response; + + if (FoundLog->OplogCount() > 0) + { + std::unordered_set FieldNamesFilter; + auto FilterObject = [&FieldNamesFilter](CbObjectView& Object) -> CbObject { + CbObject RewrittenOp = RewriteCbObject(Object, [&FieldNamesFilter](CbObjectWriter&, CbFieldView Field) -> bool { + if (FieldNamesFilter.contains(std::string(Field.GetName()))) + { + return false; + } + + return true; + }); + + return RewrittenOp; + }; + + HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); + if (auto FieldFilter = HttpServerRequest::Decode(Params.GetValue("fieldfilter")); !FieldFilter.empty()) + { + ForEachStrTok(FieldFilter, ',', [&](std::string_view FieldName) { + FieldNamesFilter.insert(std::string(FieldName)); + return true; + }); + } + + if (auto OpKey = Params.GetValue("opkey"); !OpKey.empty()) + { + Oid OpKeyId = OpKeyStringAsOid(OpKey); + std::optional Op = FoundLog->GetOpByKey(OpKeyId); + + if (Op.has_value()) + { + if (FieldNamesFilter.empty()) + { + Response << "entry"sv << Op.value(); + } + else + { + Response << "entry"sv << FilterObject(Op.value()); + } + } + else + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound); + } + } + else + { + ProjectStore::Oplog::Paging EntryPaging; + if (std::string_view Param = Params.GetValue("start"); !Param.empty()) + { + if (auto Value = ParseInt(Param)) + { + EntryPaging.Start = *Value; + } + } + if (std::string_view Param = Params.GetValue("count"); !Param.empty()) + { + if (auto Value = ParseInt(Param)) + { + EntryPaging.Count = *Value; + } + } + + std::optional MaybeReferencedSet; + if (auto TrimString = Params.GetValue("trim_by_referencedset"); TrimString == "true") + { + MaybeReferencedSet = LoadReferencedSet(*Project, *FoundLog); + } + Response.BeginArray("entries"sv); + + bool ShouldFilterFields = !FieldNamesFilter.empty(); + + if (MaybeReferencedSet) + { + const OplogReferencedSet& ReferencedSet = MaybeReferencedSet.value(); + FoundLog->IterateOplogWithKey( + [this, &Response, &FilterObject, ShouldFilterFields, &ReferencedSet](ProjectStore::LogSequenceNumber /* LSN */, + const Oid& Key, + CbObjectView Op) { + if (!ReferencedSet.Contains(Key)) + { + if (!OplogReferencedSet::IsNonPackage(Op["key"].AsString())) + { + return; + } + } + + if (ShouldFilterFields) + { + Response << FilterObject(Op); + } + else + { + Response << Op; + } + }, + EntryPaging); + } + else + { + FoundLog->IterateOplog( + [this, &Response, &FilterObject, ShouldFilterFields](CbObjectView Op) { + if (ShouldFilterFields) + { + Response << FilterObject(Op); + } + else + { + Response << Op; + } + }, + EntryPaging); + } + + Response.EndArray(); + } + } + if (HttpReq.AcceptContentType() == HttpContentType::kCompressedBinary) + { + CompositeBuffer Payload = CompressedBuffer::Compress(Response.Save().GetBuffer()).GetCompressed(); + return HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kCompressedBinary, Payload); + } + else + { + return HttpReq.WriteResponse(HttpResponseCode::OK, Response.Save()); + } +} + +void +HttpProjectService::HandleProjectRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("ProjectService::Project"); + + using namespace std::literals; + + HttpServerRequest& HttpReq = Req.ServerRequest(); + const std::string_view ProjectId = Req.GetCapture(1); + + switch (HttpReq.RequestVerb()) + { + case HttpVerb::kPost: + { + if (!m_ProjectStore->AreDiskWritesAllowed()) + { + return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); + } + + CbValidateError ValidateResult; + if (CbObject Params = ValidateAndReadCompactBinaryObject(HttpReq.ReadPayload(), ValidateResult); + ValidateResult == CbValidateError::None) + { + std::filesystem::path Root = Params["root"sv].AsU8String(); // Workspace root (i.e `D:/UE5/`) + std::filesystem::path EngineRoot = Params["engine"sv].AsU8String(); // Engine root (i.e `D:/UE5/Engine`) + std::filesystem::path ProjectRoot = + Params["project"sv].AsU8String(); // Project root directory (i.e `D:/UE5/Samples/Games/Lyra`) + std::filesystem::path ProjectFilePath = + Params["projectfile"sv].AsU8String(); // Project file path (i.e `D:/UE5/Samples/Games/Lyra/Lyra.uproject`) + + const std::filesystem::path BasePath = m_ProjectStore->BasePath() / ProjectId; + m_ProjectStore->NewProject(BasePath, ProjectId, Root, EngineRoot, ProjectRoot, ProjectFilePath); + + ZEN_INFO("established project (id: '{}', roots: '{}', '{}', '{}', '{}'{})", + ProjectId, + Root, + EngineRoot, + ProjectRoot, + ProjectFilePath, + ProjectFilePath.empty() ? ", project will not be GCd due to empty project file path" : ""); + + m_ProjectStats.ProjectWriteCount++; + HttpReq.WriteResponse(HttpResponseCode::Created); + } + else + { + HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Malformed compact binary object: '{}'", ToString(ValidateResult))); + } + } + break; + + case HttpVerb::kPut: + { + if (!m_ProjectStore->AreDiskWritesAllowed()) + { + return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); + } + CbValidateError ValidateResult; + if (CbObject Params = ValidateAndReadCompactBinaryObject(HttpReq.ReadPayload(), ValidateResult); + ValidateResult == CbValidateError::None) + { + std::filesystem::path Root = Params["root"sv].AsU8String(); // Workspace root (i.e `D:/UE5/`) + std::filesystem::path EngineRoot = Params["engine"sv].AsU8String(); // Engine root (i.e `D:/UE5/Engine`) + std::filesystem::path ProjectRoot = + Params["project"sv].AsU8String(); // Project root directory (i.e `D:/UE5/Samples/Games/Lyra`) + std::filesystem::path ProjectFilePath = + Params["projectfile"sv].AsU8String(); // Project file path (i.e `D:/UE5/Samples/Games/Lyra/Lyra.uproject`) + + if (m_ProjectStore->UpdateProject(ProjectId, Root, EngineRoot, ProjectRoot, ProjectFilePath)) + { + m_ProjectStats.ProjectWriteCount++; + ZEN_INFO("updated project (id: '{}', roots: '{}', '{}', '{}', '{}'{})", + ProjectId, + Root, + EngineRoot, + ProjectRoot, + ProjectFilePath, + ProjectFilePath.empty() ? ", project will not be GCd due to empty project file path" : ""); + + HttpReq.WriteResponse(HttpResponseCode::OK); + } + else + { + const std::filesystem::path BasePath = m_ProjectStore->BasePath() / ProjectId; + m_ProjectStore->NewProject(BasePath, ProjectId, Root, EngineRoot, ProjectRoot, ProjectFilePath); + + m_ProjectStats.ProjectWriteCount++; + ZEN_INFO("established project (id: '{}', roots: '{}', '{}', '{}', '{}'{})", + ProjectId, + Root, + EngineRoot, + ProjectRoot, + ProjectFilePath, + ProjectFilePath.empty() ? ", project will not be GCd due to empty project file path" : ""); + + HttpReq.WriteResponse(HttpResponseCode::Created); + } + } + else + { + HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Malformed compact binary object: '{}'", ToString(ValidateResult))); + } + } + break; + + case HttpVerb::kGet: + { + Ref Project = m_ProjectStore->OpenProject(ProjectId); + if (!Project) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("project {} not found", ProjectId)); + } + Project->TouchProject(); + + std::vector OpLogs = Project->ScanForOplogs(); + + CbObjectWriter Response; + Response << "id"sv << Project->Identifier; + Response << "root"sv << PathToUtf8(Project->RootDir); + Response << "engine"sv << PathToUtf8(Project->EngineRootDir); + Response << "project"sv << PathToUtf8(Project->ProjectRootDir); + Response << "projectfile"sv << PathToUtf8(Project->ProjectFilePath); + + Response.BeginArray("oplogs"sv); + for (const std::string& OplogId : OpLogs) + { + Response.BeginObject(); + Response << "id"sv << OplogId; + Response.EndObject(); + } + Response.EndArray(); // oplogs + + HttpReq.WriteResponse(HttpResponseCode::OK, Response.Save()); + + m_ProjectStats.ProjectReadCount++; + } + break; + + case HttpVerb::kDelete: + { + Ref Project = m_ProjectStore->OpenProject(ProjectId); + if (!Project) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("project {} not found", ProjectId)); + } + + ZEN_INFO("deleting project '{}'", ProjectId); + if (!m_ProjectStore->DeleteProject(ProjectId)) + { + return HttpReq.WriteResponse(HttpResponseCode::Locked, + HttpContentType::kText, + fmt::format("project {} is in use", ProjectId)); + } + + m_ProjectStats.ProjectDeleteCount++; + return HttpReq.WriteResponse(HttpResponseCode::NoContent); + } + break; + + default: + break; + } +} + +void +HttpProjectService::HandleOplogSaveRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("ProjectService::OplogSave"); + + HttpServerRequest& HttpReq = Req.ServerRequest(); + + if (!m_ProjectStore->AreDiskWritesAllowed()) + { + return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); + } + + const auto& ProjectId = Req.GetCapture(1); + const auto& OplogId = Req.GetCapture(2); + if (HttpReq.RequestContentType() != HttpContentType::kCbObject) + { + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid content type"); + } + IoBuffer Payload = HttpReq.ReadPayload(); + + Ref Project = m_ProjectStore->OpenProject(ProjectId); + if (!Project) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Write oplog request for unknown project '{}'", ProjectId)); + } + Project->TouchProject(); + + Ref Oplog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ false); + if (!Oplog) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Write oplog request for unknown oplog '{}/{}'", ProjectId, OplogId)); + } + Project->TouchOplog(OplogId); + + CbValidateError ValidateResult; + if (CbObject ContainerObject = ValidateAndReadCompactBinaryObject(std::move(Payload), ValidateResult); + ValidateResult == CbValidateError::None && ContainerObject) + { + RwLock AttachmentsLock; + tsl::robin_set Attachments; + + auto HasAttachment = [this](const IoHash& RawHash) { return m_CidStore.ContainsChunk(RawHash); }; + auto OnNeedBlock = [&AttachmentsLock, &Attachments](const IoHash& BlockHash, const std::vector&& ChunkHashes) { + RwLock::ExclusiveLockScope _(AttachmentsLock); + if (BlockHash != IoHash::Zero) + { + Attachments.insert(BlockHash); + } + else + { + Attachments.insert(ChunkHashes.begin(), ChunkHashes.end()); + } + }; + auto OnNeedAttachment = [&AttachmentsLock, &Attachments](const IoHash& RawHash) { + RwLock::ExclusiveLockScope _(AttachmentsLock); + Attachments.insert(RawHash); + }; + + auto OnChunkedAttachment = [](const ChunkedInfo&) {}; + + auto OnReferencedAttachments = [&Oplog](std::span RawHashes) { Oplog->CaptureAddedAttachments(RawHashes); }; + + // Make sure we retain any attachments we download before writing the oplog + Oplog->EnableUpdateCapture(); + auto _ = MakeGuard([&Oplog]() { Oplog->DisableUpdateCapture(); }); + + RemoteProjectStore::Result Result = SaveOplogContainer(*Oplog, + ContainerObject, + OnReferencedAttachments, + HasAttachment, + OnNeedBlock, + OnNeedAttachment, + OnChunkedAttachment, + nullptr); + + if (Result.ErrorCode == 0) + { + if (Attachments.empty()) + { + HttpReq.WriteResponse(HttpResponseCode::OK); + } + else + { + CbObjectWriter Cbo(1 + 1 + 5 + Attachments.size() * (1 + sizeof(IoHash::Hash)) + 1); + Cbo.BeginArray("need"); + { + for (const IoHash& Hash : Attachments) + { + ZEN_DEBUG("Need attachment {}", Hash); + Cbo << Hash; + } + } + Cbo.EndArray(); // "need" + + CbObject ResponsePayload = Cbo.Save(); + return HttpReq.WriteResponse(HttpResponseCode::OK, ResponsePayload); + } + } + else + { + ZEN_DEBUG("Request {}: '{}' failed with {}. Reason: `{}`", + ToString(HttpReq.RequestVerb()), + HttpReq.QueryString(), + Result.ErrorCode, + Result.Reason); + + if (Result.Reason.empty()) + { + return HttpReq.WriteResponse(HttpResponseCode(Result.ErrorCode)); + } + else + { + return HttpReq.WriteResponse(HttpResponseCode(Result.ErrorCode), HttpContentType::kText, Result.Reason); + } + } + } + else + { + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid payload: '{}'", ToString(ValidateResult))); + } +} + +void +HttpProjectService::HandleOplogLoadRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("ProjectService::OplogLoad"); + + HttpServerRequest& HttpReq = Req.ServerRequest(); + const auto& ProjectId = Req.GetCapture(1); + const auto& OplogId = Req.GetCapture(2); + + const HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); + + if (HttpReq.AcceptContentType() != HttpContentType::kCbObject) + { + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid accept content type"); + } + + Ref Project = m_ProjectStore->OpenProject(ProjectId); + if (!Project) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Read oplog request for unknown project '{}'", ProjectId)); + } + Project->TouchProject(); + + Ref Oplog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ true); + if (!Oplog) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Read oplog request for unknown oplog '{}/{}'", ProjectId, OplogId)); + } + Project->TouchOplog(OplogId); + + size_t MaxBlockSize = RemoteStoreOptions::DefaultMaxBlockSize; + if (auto Param = Params.GetValue("maxblocksize"); Param.empty() == false) + { + if (auto Value = ParseInt(Param)) + { + MaxBlockSize = Value.value(); + } + } + size_t MaxChunkEmbedSize = RemoteStoreOptions::DefaultMaxChunkEmbedSize; + if (auto Param = Params.GetValue("maxchunkembedsize"); Param.empty() == false) + { + if (auto Value = ParseInt(Param)) + { + MaxChunkEmbedSize = Value.value(); + } + } + size_t MaxChunksPerBlock = RemoteStoreOptions::DefaultMaxChunksPerBlock; + if (auto Param = Params.GetValue("maxchunksperblock"); Param.empty() == false) + { + if (auto Value = ParseInt(Param)) + { + MaxChunksPerBlock = Value.value(); + } + } + + size_t ChunkFileSizeLimit = RemoteStoreOptions::DefaultChunkFileSizeLimit; + if (auto Param = Params.GetValue("chunkfilesizelimit"); Param.empty() == false) + { + if (auto Value = ParseInt(Param)) + { + ChunkFileSizeLimit = Value.value(); + } + } + + WorkerThreadPool& WorkerPool = GetLargeWorkerPool(EWorkloadType::Background); + + RemoteProjectStore::LoadContainerResult ContainerResult = BuildContainer( + m_CidStore, + *Project, + *Oplog, + WorkerPool, + MaxBlockSize, + MaxChunkEmbedSize, + MaxChunksPerBlock, + ChunkFileSizeLimit, + /* BuildBlocks */ false, + /* IgnoreMissingAttachments */ false, + /* AllowChunking*/ false, + [](CompressedBuffer&&, ChunkBlockDescription&&) {}, + [](const IoHash&, TGetAttachmentBufferFunc&&) {}, + [](std::vector>&&) {}, + /* EmbedLooseFiles*/ false); + + if (ContainerResult.ErrorCode == 0) + { + return HttpReq.WriteResponse(HttpResponseCode::OK, ContainerResult.ContainerObject); + } + else + { + ZEN_DEBUG("Request {}: '{}' failed with {}. Reason: `{}`", + ToString(HttpReq.RequestVerb()), + HttpReq.QueryString(), + ContainerResult.ErrorCode, + ContainerResult.Reason); + + if (ContainerResult.Reason.empty()) + { + return HttpReq.WriteResponse(HttpResponseCode(ContainerResult.ErrorCode)); + } + else + { + return HttpReq.WriteResponse(HttpResponseCode(ContainerResult.ErrorCode), HttpContentType::kText, ContainerResult.Reason); + } + } +} + +void +HttpProjectService::HandleRpcRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("ProjectService::Rpc"); + using namespace std::literals; + + HttpServerRequest& HttpReq = Req.ServerRequest(); + + const auto& ProjectId = Req.GetCapture(1); + const auto& OplogId = Req.GetCapture(2); + IoBuffer Payload = HttpReq.ReadPayload(); + + HttpContentType PayloadContentType = HttpReq.RequestContentType(); + CbPackage Package; + CbObject Cb; + switch (PayloadContentType) + { + case HttpContentType::kJSON: + case HttpContentType::kUnknownContentType: + case HttpContentType::kText: + { + std::string JsonText(reinterpret_cast(Payload.GetData()), Payload.GetSize()); + Cb = LoadCompactBinaryFromJson(JsonText).AsObject(); + if (!Cb) + { + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Content format not supported, expected JSON format"); + } + } + break; + case HttpContentType::kCbObject: + { + CbValidateError ValidateResult; + if (Cb = ValidateAndReadCompactBinaryObject(std::move(Payload), ValidateResult); + ValidateResult != CbValidateError::None || !Cb) + { + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse( + HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Content format not supported, expected compact binary format ('{}')", ToString(ValidateResult))); + } + break; + } + case HttpContentType::kCbPackage: + try + { + Package = ParsePackageMessage(Payload); + Cb = Package.GetObject(); + } + catch (const std::invalid_argument& ex) + { + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Failed to parse package request, reason: '{}'", ex.what())); + } + if (!Cb) + { + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Content format not supported, expected package message format"); + } + break; + default: + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid request content type"); + } + + Ref Project = m_ProjectStore->OpenProject(ProjectId); + if (!Project) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Rpc oplog request for unknown project '{}'", ProjectId)); + } + Project->TouchProject(); + + std::string_view Method = Cb["method"sv].AsString(); + + bool VerifyPathOnDisk = Method != "getchunks"sv; + + Ref Oplog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, VerifyPathOnDisk); + if (!Oplog) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Rpc oplog request for unknown oplog '{}/{}'", ProjectId, OplogId)); + } + Project->TouchOplog(OplogId); + + uint32_t MethodHash = HashStringDjb2(Method); + + switch (MethodHash) + { + case HashStringDjb2("import"sv): + { + if (!m_ProjectStore->AreDiskWritesAllowed()) + { + return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); + } + + CbObjectView Params = Cb["params"sv].AsObjectView(); + size_t MaxBlockSize = Params["maxblocksize"sv].AsUInt64(RemoteStoreOptions::DefaultMaxBlockSize); + size_t MaxChunkEmbedSize = Params["maxchunkembedsize"sv].AsUInt64(RemoteStoreOptions::DefaultMaxChunkEmbedSize); + bool Force = Params["force"sv].AsBool(false); + bool IgnoreMissingAttachments = Params["ignoremissingattachments"sv].AsBool(false); + bool CleanOplog = Params["clean"].AsBool(false); + + CreateRemoteStoreResult RemoteStoreResult = + CreateRemoteStore(Params, m_AuthMgr, MaxBlockSize, MaxChunkEmbedSize, Oplog->TempPath()); + + if (RemoteStoreResult.Store == nullptr) + { + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, RemoteStoreResult.Description); + } + std::shared_ptr RemoteStore = std::move(RemoteStoreResult.Store); + RemoteProjectStore::RemoteStoreInfo StoreInfo = RemoteStore->GetInfo(); + + JobId JobId = m_JobQueue.QueueJob( + fmt::format("Import oplog '{}/{}'", Project->Identifier, Oplog->OplogId()), + [this, + ChunkStore = &m_CidStore, + ActualRemoteStore = std::move(RemoteStore), + Oplog, + Force, + IgnoreMissingAttachments, + CleanOplog](JobContext& Context) { + Context.ReportMessage(fmt::format("Loading oplog '{}/{}' from {}", + Oplog->GetOuterProjectIdentifier(), + Oplog->OplogId(), + ActualRemoteStore->GetInfo().Description)); + + WorkerThreadPool& WorkerPool = GetLargeWorkerPool(EWorkloadType::Background); + WorkerThreadPool& NetworkWorkerPool = GetMediumWorkerPool(EWorkloadType::Background); + + RemoteProjectStore::Result Result = LoadOplog(m_CidStore, + *ActualRemoteStore, + *Oplog, + NetworkWorkerPool, + WorkerPool, + Force, + IgnoreMissingAttachments, + CleanOplog, + &Context); + auto Response = ConvertResult(Result); + ZEN_INFO("LoadOplog: Status: {} '{}'", ToString(Response.first), Response.second); + if (!IsHttpSuccessCode(Response.first)) + { + throw JobError(Response.second.empty() ? fmt::format("Status: {}", ToString(Response.first)) : Response.second, + (int)Response.first); + } + }); + + return HttpReq.WriteResponse(HttpResponseCode::Accepted, HttpContentType::kText, fmt::format("{}", JobId.Id)); + } + case HashStringDjb2("export"sv): + { + CbObjectView Params = Cb["params"sv].AsObjectView(); + size_t MaxBlockSize = Params["maxblocksize"sv].AsUInt64(RemoteStoreOptions::DefaultMaxBlockSize); + size_t MaxChunkEmbedSize = Params["maxchunkembedsize"sv].AsUInt64(RemoteStoreOptions::DefaultMaxChunkEmbedSize); + size_t MaxChunksPerBlock = Params["maxchunksperblock"sv].AsUInt64(RemoteStoreOptions::DefaultMaxChunksPerBlock); + size_t ChunkFileSizeLimit = Params["chunkfilesizelimit"sv].AsUInt64(RemoteStoreOptions::DefaultChunkFileSizeLimit); + bool Force = Params["force"sv].AsBool(false); + bool IgnoreMissingAttachments = Params["ignoremissingattachments"sv].AsBool(false); + bool EmbedLooseFile = Params["embedloosefiles"sv].AsBool(false); + + CreateRemoteStoreResult RemoteStoreResult = + CreateRemoteStore(Params, m_AuthMgr, MaxBlockSize, MaxChunkEmbedSize, Oplog->TempPath()); + + if (RemoteStoreResult.Store == nullptr) + { + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, RemoteStoreResult.Description); + } + std::shared_ptr RemoteStore = std::move(RemoteStoreResult.Store); + RemoteProjectStore::RemoteStoreInfo StoreInfo = RemoteStore->GetInfo(); + + JobId JobId = m_JobQueue.QueueJob( + fmt::format("Export oplog '{}/{}'", Project->Identifier, Oplog->OplogId()), + [this, + ActualRemoteStore = std::move(RemoteStore), + Project, + Oplog, + MaxBlockSize, + MaxChunksPerBlock, + MaxChunkEmbedSize, + ChunkFileSizeLimit, + EmbedLooseFile, + Force, + IgnoreMissingAttachments](JobContext& Context) { + Context.ReportMessage(fmt::format("Saving oplog '{}/{}' to {}, maxblocksize {}, maxchunkembedsize {}", + Project->Identifier, + Oplog->OplogId(), + ActualRemoteStore->GetInfo().Description, + NiceBytes(MaxBlockSize), + NiceBytes(MaxChunkEmbedSize))); + + WorkerThreadPool& WorkerPool = GetLargeWorkerPool(EWorkloadType::Background); + WorkerThreadPool& NetworkWorkerPool = GetMediumWorkerPool(EWorkloadType::Background); + + RemoteProjectStore::Result Result = SaveOplog(m_CidStore, + *ActualRemoteStore, + *Project, + *Oplog, + NetworkWorkerPool, + WorkerPool, + MaxBlockSize, + MaxChunksPerBlock, + MaxChunkEmbedSize, + ChunkFileSizeLimit, + EmbedLooseFile, + Force, + IgnoreMissingAttachments, + &Context); + auto Response = ConvertResult(Result); + ZEN_INFO("SaveOplog: Status: {} '{}'", ToString(Response.first), Response.second); + if (!IsHttpSuccessCode(Response.first)) + { + throw JobError(Response.second.empty() ? fmt::format("Status: {}", ToString(Response.first)) : Response.second, + (int)Response.first); + } + }); + + return HttpReq.WriteResponse(HttpResponseCode::Accepted, HttpContentType::kText, fmt::format("{}", JobId.Id)); + } + case HashStringDjb2("getchunks"sv): + { + RpcAcceptOptions AcceptFlags = static_cast(Cb["AcceptFlags"sv].AsUInt16(0u)); + int32_t TargetProcessId = Cb["Pid"sv].AsInt32(0); + + std::vector Requests = m_ProjectStore->ParseChunksRequests(*Project, *Oplog, Cb); + std::vector Results = + Requests.empty() ? std::vector{} : m_ProjectStore->GetChunks(*Project, *Oplog, Requests); + CbPackage Response = m_ProjectStore->WriteChunksRequestResponse(*Project, *Oplog, std::move(Requests), std::move(Results)); + + void* TargetProcessHandle = nullptr; + FormatFlags Flags = FormatFlags::kDefault; + if (EnumHasAllFlags(AcceptFlags, RpcAcceptOptions::kAllowLocalReferences)) + { + Flags |= FormatFlags::kAllowLocalReferences; + if (!EnumHasAnyFlags(AcceptFlags, RpcAcceptOptions::kAllowPartialLocalReferences)) + { + Flags |= FormatFlags::kDenyPartialLocalReferences; + } + TargetProcessHandle = m_OpenProcessCache.GetProcessHandle(HttpReq.SessionId(), TargetProcessId); + } + + CompositeBuffer RpcResponseBuffer = FormatPackageMessageBuffer(Response, Flags, TargetProcessHandle); + return HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, RpcResponseBuffer); + } + case HashStringDjb2("putchunks"sv): + { + ZEN_TRACE_CPU("Store::Rpc::putchunks"); + if (!m_ProjectStore->AreDiskWritesAllowed()) + { + return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); + } + + CbObject Object = Package.GetObject(); + const bool UsingTempFiles = Object["usingtmpfiles"].AsBool(false); + + std::span Attachments = Package.GetAttachments(); + if (!Attachments.empty()) + { + std::vector WriteAttachmentBuffers; + std::vector WriteRawHashes; + + WriteAttachmentBuffers.reserve(Attachments.size()); + WriteRawHashes.reserve(Attachments.size()); + + for (const CbAttachment& Attachment : Attachments) + { + IoHash RawHash = Attachment.GetHash(); + const CompressedBuffer& Compressed = Attachment.AsCompressedBinary(); + IoBuffer AttachmentPayload = Compressed.GetCompressed().Flatten().AsIoBuffer(); + if (UsingTempFiles) + { + AttachmentPayload.SetDeleteOnClose(true); + } + WriteAttachmentBuffers.push_back(std::move(AttachmentPayload)); + WriteRawHashes.push_back(RawHash); + } + + Oplog->CaptureAddedAttachments(WriteRawHashes); + m_CidStore.AddChunks(WriteAttachmentBuffers, + WriteRawHashes, + UsingTempFiles ? CidStore::InsertMode::kMayBeMovedInPlace : CidStore::InsertMode::kCopyOnly); + } + return HttpReq.WriteResponse(HttpResponseCode::OK); + } + case HashStringDjb2("snapshot"sv): + { + ZEN_TRACE_CPU("Store::Rpc::snapshot"); + if (!m_ProjectStore->AreDiskWritesAllowed()) + { + return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); + } + + // Snapshot all referenced files. This brings the content of all + // files into the CID store + + uint32_t OpCount = 0; + uint64_t InlinedBytes = 0; + uint64_t InlinedFiles = 0; + uint64_t TotalBytes = 0; + uint64_t TotalFiles = 0; + + std::vector NewOps; + struct AddedChunk + { + IoBuffer Buffer; + uint64_t RawSize = 0; + }; + tsl::robin_map AddedChunks; + + Oplog->IterateOplog( + [&](CbObjectView Op) { + bool OpRewritten = false; + bool AllOk = true; + + CbWriter FilesArrayWriter; + FilesArrayWriter.BeginArray("files"sv); + + for (CbFieldView& Field : Op["files"sv]) + { + bool CopyField = true; + + if (CbObjectView View = Field.AsObjectView()) + { + const IoHash DataHash = View["data"sv].AsHash(); + + if (DataHash == IoHash::Zero) + { + std::string_view ServerPath = View["serverpath"sv].AsString(); + std::filesystem::path FilePath = Project->RootDir / ServerPath; + BasicFile DataFile; + std::error_code Ec; + DataFile.Open(FilePath, BasicFile::Mode::kRead, Ec); + + if (Ec) + { + // Error... + + ZEN_ERROR("unable to read data from file '{}': {}", FilePath, Ec.message()); + + AllOk = false; + } + else + { + // Read file contents into memory, compress and keep in map of chunks to add to Cid store + IoBuffer FileIoBuffer = DataFile.ReadAll(); + CompressedBuffer Compressed = CompressedBuffer::Compress(SharedBuffer(std::move(FileIoBuffer))); + const uint64_t RawSize = Compressed.DecodeRawSize(); + const IoHash RawHash = Compressed.DecodeRawHash(); + if (!AddedChunks.contains(RawHash)) + { + const std::filesystem::path TempChunkPath = Oplog->TempPath() / RawHash.ToHexString(); + BasicFile ChunkTempFile; + ChunkTempFile.Open(TempChunkPath, BasicFile::Mode::kTruncateDelete); + ChunkTempFile.Write(Compressed.GetCompressed(), 0, Ec); + if (Ec) + { + Oid ChunkId = View["id"sv].AsObjectId(); + ZEN_ERROR("unable to write external file as compressed chunk '{}', id {}: {}", + FilePath, + ChunkId, + Ec.message()); + AllOk = false; + } + else + { + void* FileHandle = ChunkTempFile.Detach(); + IoBuffer ChunkBuffer(IoBuffer::File, + FileHandle, + 0, + Compressed.GetCompressed().GetSize(), + /*IsWholeFile*/ true); + ChunkBuffer.SetDeleteOnClose(true); + AddedChunks.insert_or_assign( + RawHash, + AddedChunk{.Buffer = std::move(ChunkBuffer), .RawSize = RawSize}); + } + } + + TotalBytes += RawSize; + ++TotalFiles; + + // Rewrite file array entry with new data reference + CbObjectWriter Writer(View.GetSize()); + RewriteCbObject(Writer, View, [&](CbObjectWriter&, CbFieldView Field) -> bool { + if (Field.GetName() == "data"sv) + { + // omit this field as we will write it explicitly ourselves + return true; + } + return false; + }); + Writer.AddBinaryAttachment("data"sv, RawHash); + + CbObject RewrittenOp = Writer.Save(); + FilesArrayWriter.AddObject(std::move(RewrittenOp)); + CopyField = false; + } + } + } + + if (CopyField) + { + FilesArrayWriter.AddField(Field); + } + else + { + OpRewritten = true; + } + } + + if (OpRewritten && AllOk) + { + FilesArrayWriter.EndArray(); + CbArray FilesArray = FilesArrayWriter.Save().AsArray(); + + CbObject RewrittenOp = RewriteCbObject(Op, [&](CbObjectWriter& NewWriter, CbFieldView Field) -> bool { + if (Field.GetName() == "files"sv) + { + NewWriter.AddArray("files"sv, FilesArray); + + return true; + } + + return false; + }); + + NewOps.push_back(std::move(RewrittenOp)); + } + + OpCount++; + }, + ProjectStore::Oplog::Paging{}); + + CbObjectWriter ResponseObj; + + // Persist rewritten oplog entries + if (!NewOps.empty()) + { + ResponseObj.BeginArray("rewritten_ops"); + + for (CbObject& NewOp : NewOps) + { + ProjectStore::LogSequenceNumber NewLsn = Oplog->AppendNewOplogEntry(std::move(NewOp)); + + ZEN_DEBUG("appended rewritten op at LSN: {}", NewLsn.Number); + + ResponseObj.AddInteger(NewLsn.Number); + } + + ResponseObj.EndArray(); + } + + // Ops that have moved chunks to a compressed buffer for storage in m_CidStore have been rewritten with references to the + // new chunk(s). Make sure we add the chunks to m_CidStore, and do it after we update the oplog so GC doesn't think we have + // unreferenced chunks. + for (auto It : AddedChunks) + { + const IoHash& RawHash = It.first; + AddedChunk& Chunk = It.second; + CidStore::InsertResult Result = m_CidStore.AddChunk(Chunk.Buffer, RawHash); + if (Result.New) + { + InlinedBytes += Chunk.RawSize; + ++InlinedFiles; + } + } + + ResponseObj << "inlined_bytes" << InlinedBytes << "inlined_files" << InlinedFiles; + ResponseObj << "total_bytes" << TotalBytes << "total_files" << TotalFiles; + + ZEN_INFO("oplog '{}/{}': rewrote {} oplog entries (out of {})", ProjectId, OplogId, NewOps.size(), OpCount); + + return HttpReq.WriteResponse(HttpResponseCode::OK, ResponseObj.Save()); + } + case HashStringDjb2("appendops"sv): + { + ZEN_TRACE_CPU("Store::Rpc::appendops"); + if (!m_ProjectStore->AreDiskWritesAllowed()) + { + return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); + } + + CbArrayView OpsArray = Cb["ops"sv].AsArrayView(); + + size_t OpsBufferSize = 0; + for (CbFieldView OpView : OpsArray) + { + OpsBufferSize += OpView.GetSize(); + } + UniqueBuffer OpsBuffers = UniqueBuffer::Alloc(OpsBufferSize); + MutableMemoryView OpsBuffersMemory = OpsBuffers.GetMutableView(); + + std::vector Ops; + Ops.reserve(OpsArray.Num()); + for (CbFieldView OpView : OpsArray) + { + OpView.CopyTo(OpsBuffersMemory); + Ops.push_back(CbObjectView(OpsBuffersMemory.GetData())); + OpsBuffersMemory.MidInline(OpView.GetSize()); + } + + std::vector LSNs = Oplog->AppendNewOplogEntries(Ops); + ZEN_ASSERT(LSNs.size() == Ops.size()); + + std::vector MissingAttachments; + for (size_t OpIndex = 0; OpIndex < Ops.size(); OpIndex++) + { + if (LSNs[OpIndex]) + { + CbObjectView Op = Ops[OpIndex]; + Op.IterateAttachments([this, &MissingAttachments](CbFieldView AttachmentView) { + const IoHash Cid = AttachmentView.AsAttachment(); + if (!m_CidStore.ContainsChunk(Cid)) + { + MissingAttachments.push_back(Cid); + } + }); + } + } + + CbPackage ResponsePackage; + + { + CbObjectWriter ResponseObj; + ResponseObj.BeginArray("written_ops"); + + for (ProjectStore::LogSequenceNumber NewLsn : LSNs) + { + ZEN_DEBUG("appended written op at LSN: {}", NewLsn.Number); + ResponseObj.AddInteger(NewLsn.Number); + } + ResponseObj.EndArray(); + + if (!MissingAttachments.empty()) + { + ResponseObj.BeginArray("need"); + + for (const IoHash& Cid : MissingAttachments) + { + ResponseObj.AddHash(Cid); + } + ResponseObj.EndArray(); + } + ResponsePackage.SetObject(ResponseObj.Save()); + } + + std::vector ResponseBuffers = FormatPackageMessage(ResponsePackage); + + return HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, ResponseBuffers); + } + default: + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Unknown rpc method '{}'", Method)); + } +} +void +HttpProjectService::HandleDetailsRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("ProjectService::Details"); + + using namespace std::literals; + + HttpServerRequest& HttpReq = Req.ServerRequest(); + + HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); + bool CSV = Params.GetValue("csv"sv) == "true"sv; + bool Details = Params.GetValue("details"sv) == "true"sv; + bool OpDetails = Params.GetValue("opdetails"sv) == "true"sv; + bool AttachmentDetails = Params.GetValue("attachmentdetails"sv) == "true"sv; + + if (CSV) + { + ExtendableStringBuilder<4096> CSVWriter; + CSVHeader(Details, AttachmentDetails, CSVWriter); + + m_ProjectStore->IterateProjects([&](ProjectStore::Project& Project) { + Project.IterateOplogs([&](const RwLock::SharedLockScope&, ProjectStore::Oplog& Oplog) { + Oplog.IterateOplogWithKey( + [this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](ProjectStore::LogSequenceNumber LSN, + const Oid& Key, + CbObjectView Op) { + CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter); + }); + }); + }); + + HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView()); + } + else + { + CbObjectWriter Cbo; + Cbo.BeginArray("projects"); + { + m_ProjectStore->DiscoverProjects(); + + m_ProjectStore->IterateProjects([&](ProjectStore::Project& Project) { + std::vector OpLogs = Project.ScanForOplogs(); + CbWriteProject(m_CidStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo); + }); + } + Cbo.EndArray(); + HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); + } +} + +void +HttpProjectService::HandleProjectDetailsRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("ProjectService::ProjectDetails"); + + using namespace std::literals; + + HttpServerRequest& HttpReq = Req.ServerRequest(); + const auto& ProjectId = Req.GetCapture(1); + + HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); + bool CSV = Params.GetValue("csv"sv) == "true"sv; + bool Details = Params.GetValue("details"sv) == "true"sv; + bool OpDetails = Params.GetValue("opdetails"sv) == "true"sv; + bool AttachmentDetails = Params.GetValue("attachmentdetails"sv) == "true"sv; + + Ref FoundProject = m_ProjectStore->OpenProject(ProjectId); + if (!FoundProject) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound); + } + ProjectStore::Project& Project = *FoundProject.Get(); + + if (CSV) + { + ExtendableStringBuilder<4096> CSVWriter; + CSVHeader(Details, AttachmentDetails, CSVWriter); + + FoundProject->IterateOplogs([&](const RwLock::SharedLockScope&, ProjectStore::Oplog& Oplog) { + Oplog.IterateOplogWithKey([this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](ProjectStore::LogSequenceNumber LSN, + const Oid& Key, + CbObjectView Op) { + CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter); + }); + }); + HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView()); + } + else + { + CbObjectWriter Cbo; + std::vector OpLogs = FoundProject->ScanForOplogs(); + Cbo.BeginArray("projects"); + { + CbWriteProject(m_CidStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo); + } + Cbo.EndArray(); + HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); + } +} + +void +HttpProjectService::HandleOplogDetailsRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("ProjectService::OplogDetails"); + + using namespace std::literals; + + HttpServerRequest& HttpReq = Req.ServerRequest(); + const auto& ProjectId = Req.GetCapture(1); + const auto& OplogId = Req.GetCapture(2); + + HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); + bool CSV = Params.GetValue("csv"sv) == "true"sv; + bool Details = Params.GetValue("details"sv) == "true"sv; + bool OpDetails = Params.GetValue("opdetails"sv) == "true"sv; + bool AttachmentDetails = Params.GetValue("attachmentdetails"sv) == "true"sv; + + Ref FoundProject = m_ProjectStore->OpenProject(ProjectId); + if (!FoundProject) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound); + } + + Ref FoundLog = FoundProject->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ true); + if (!FoundLog) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound); + } + + ProjectStore::Project& Project = *FoundProject.Get(); + ProjectStore::Oplog& Oplog = *FoundLog; + if (CSV) + { + ExtendableStringBuilder<4096> CSVWriter; + CSVHeader(Details, AttachmentDetails, CSVWriter); + + Oplog.IterateOplogWithKey([this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](ProjectStore::LogSequenceNumber LSN, + const Oid& Key, + CbObjectView Op) { + CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter); + }); + HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView()); + } + else + { + CbObjectWriter Cbo; + Cbo.BeginArray("oplogs"); + { + CbWriteOplog(m_CidStore, Oplog, Details, OpDetails, AttachmentDetails, Cbo); + } + Cbo.EndArray(); + HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); + } +} + +void +HttpProjectService::HandleOplogOpDetailsRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("ProjectService::OplogOpDetails"); + + using namespace std::literals; + + HttpServerRequest& HttpReq = Req.ServerRequest(); + const auto& ProjectId = Req.GetCapture(1); + const auto& OplogId = Req.GetCapture(2); + const auto& OpId = Req.GetCapture(3); + + HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); + bool CSV = Params.GetValue("csv"sv) == "true"sv; + bool Details = Params.GetValue("details"sv) == "true"sv; + bool OpDetails = Params.GetValue("opdetails"sv) == "true"sv; + bool AttachmentDetails = Params.GetValue("attachmentdetails"sv) == "true"sv; + + Ref FoundProject = m_ProjectStore->OpenProject(ProjectId); + if (!FoundProject) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound); + } + + Ref FoundLog = FoundProject->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ true); + if (!FoundLog) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound); + } + + if (OpId.size() != 2 * sizeof(Oid::OidBits)) + { + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Chunk info request for invalid chunk id '{}/{}'/'{}'", ProjectId, OplogId, OpId)); + } + + const Oid ObjId = Oid::FromHexString(OpId); + ProjectStore::Project& Project = *FoundProject.Get(); + ProjectStore::Oplog& Oplog = *FoundLog; + + std::optional Op = Oplog.GetOpByKey(ObjId); + if (!Op.has_value()) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound); + } + ProjectStore::LogSequenceNumber LSN = Oplog.GetOpIndexByKey(ObjId); + if (!LSN) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound); + } + + if (CSV) + { + ExtendableStringBuilder<4096> CSVWriter; + CSVHeader(Details, AttachmentDetails, CSVWriter); + + CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, ObjId, Op.value(), CSVWriter); + HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView()); + } + else + { + CbObjectWriter Cbo; + Cbo.BeginArray("ops"); + { + CbWriteOp(m_CidStore, Details, OpDetails, AttachmentDetails, LSN, ObjId, Op.value(), Cbo); + } + Cbo.EndArray(); + HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); + } +} + +} // namespace zen diff --git a/src/zenserver/storage/projectstore/httpprojectstore.h b/src/zenserver/storage/projectstore/httpprojectstore.h new file mode 100644 index 000000000..f0a0bcfa1 --- /dev/null +++ b/src/zenserver/storage/projectstore/httpprojectstore.h @@ -0,0 +1,111 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include +#include +#include +#include + +namespace zen { + +class AuthMgr; +class JobQueue; +class OpenProcessCache; +class ProjectStore; + +////////////////////////////////////////////////////////////////////////// +// +// {project} a project identifier +// {target} a variation of the project, typically a build target +// {lsn} oplog entry sequence number +// +// /prj/{project} +// /prj/{project}/oplog/{target} +// /prj/{project}/oplog/{target}/{lsn} +// +// oplog entry +// +// id: {id} +// key: {} +// meta: {} +// data: [] +// refs: +// + +class HttpProjectService : public HttpService, public IHttpStatusProvider, public IHttpStatsProvider +{ +public: + HttpProjectService(CidStore& Store, + ProjectStore* InProjectStore, + HttpStatusService& StatusService, + HttpStatsService& StatsService, + AuthMgr& AuthMgr, + OpenProcessCache& InOpenProcessCache, + JobQueue& InJobQueue); + ~HttpProjectService(); + + virtual const char* BaseUri() const override; + virtual void HandleRequest(HttpServerRequest& Request) override; + + virtual void HandleStatsRequest(HttpServerRequest& Request) override; + virtual void HandleStatusRequest(HttpServerRequest& Request) override; + +private: + struct ProjectStats + { + std::atomic_uint64_t ProjectReadCount{}; + std::atomic_uint64_t ProjectWriteCount{}; + std::atomic_uint64_t ProjectDeleteCount{}; + std::atomic_uint64_t OpLogReadCount{}; + std::atomic_uint64_t OpLogWriteCount{}; + std::atomic_uint64_t OpLogDeleteCount{}; + std::atomic_uint64_t OpHitCount{}; + std::atomic_uint64_t OpMissCount{}; + std::atomic_uint64_t OpWriteCount{}; + std::atomic_uint64_t ChunkHitCount{}; + std::atomic_uint64_t ChunkMissCount{}; + std::atomic_uint64_t ChunkWriteCount{}; + std::atomic_uint64_t RequestCount{}; + std::atomic_uint64_t BadRequestCount{}; + }; + + void HandleProjectListRequest(HttpRouterRequest& Req); + void HandleChunkBatchRequest(HttpRouterRequest& Req); + void HandleFilesRequest(HttpRouterRequest& Req); + void HandleChunkInfosRequest(HttpRouterRequest& Req); + void HandleChunkInfoRequest(HttpRouterRequest& Req); + void HandleChunkByIdRequest(HttpRouterRequest& Req); + void HandleChunkByCidRequest(HttpRouterRequest& Req); + void HandleOplogOpPrepRequest(HttpRouterRequest& Req); + void HandleOplogOpNewRequest(HttpRouterRequest& Req); + void HandleOplogValidateRequest(HttpRouterRequest& Req); + void HandleOpLogOpRequest(HttpRouterRequest& Req); + void HandleOpLogRequest(HttpRouterRequest& Req); + void HandleOpLogEntriesRequest(HttpRouterRequest& Req); + void HandleProjectRequest(HttpRouterRequest& Req); + void HandleOplogSaveRequest(HttpRouterRequest& Req); + void HandleOplogLoadRequest(HttpRouterRequest& Req); + void HandleRpcRequest(HttpRouterRequest& Req); + void HandleDetailsRequest(HttpRouterRequest& Req); + void HandleProjectDetailsRequest(HttpRouterRequest& Req); + void HandleOplogDetailsRequest(HttpRouterRequest& Req); + void HandleOplogOpDetailsRequest(HttpRouterRequest& Req); + + inline LoggerRef Log() { return m_Log; } + + LoggerRef m_Log; + CidStore& m_CidStore; + HttpRequestRouter m_Router; + Ref m_ProjectStore; + HttpStatusService& m_StatusService; + HttpStatsService& m_StatsService; + AuthMgr& m_AuthMgr; + OpenProcessCache& m_OpenProcessCache; + JobQueue& m_JobQueue; + ProjectStats m_ProjectStats; + metrics::OperationTiming m_HttpRequests; +}; + +} // namespace zen diff --git a/src/zenserver/storage/storageconfig.cpp b/src/zenserver/storage/storageconfig.cpp new file mode 100644 index 000000000..86bb09c21 --- /dev/null +++ b/src/zenserver/storage/storageconfig.cpp @@ -0,0 +1,1055 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "storageconfig.h" + +#include +#include +#include +#include + +#include "config/luaconfig.h" + +ZEN_THIRD_PARTY_INCLUDES_START +#include +#include +#include +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen { + +void +ValidateOptions(ZenStorageServerOptions& ServerOptions) +{ + if (ServerOptions.EncryptionKey.empty() == false) + { + const auto Key = AesKey256Bit::FromString(ServerOptions.EncryptionKey); + + if (Key.IsValid() == false) + { + throw OptionParseException(fmt::format("'--encryption-aes-key' ('{}') is malformed", ServerOptions.EncryptionKey), {}); + } + } + + if (ServerOptions.EncryptionIV.empty() == false) + { + const auto IV = AesIV128Bit::FromString(ServerOptions.EncryptionIV); + + if (IV.IsValid() == false) + { + throw OptionParseException(fmt::format("'--encryption-aes-iv' ('{}') is malformed", ServerOptions.EncryptionIV), {}); + } + } + if (ServerOptions.HttpServerConfig.ForceLoopback && ServerOptions.IsDedicated) + { + throw OptionParseException("'--dedicated' conflicts with '--http-forceloopback'", {}); + } + if (ServerOptions.GcConfig.AttachmentPassCount > ZenGcConfig::GcMaxAttachmentPassCount) + { + throw OptionParseException(fmt::format("'--gc-attachment-passes' ('{}') is invalid, maximum is {}.", + ServerOptions.GcConfig.AttachmentPassCount, + ZenGcConfig::GcMaxAttachmentPassCount), + {}); + } + if (ServerOptions.GcConfig.UseGCV2 == false) + { + ZEN_WARN("'--gc-v2=false' is deprecated, reverting to '--gc-v2=true'"); + ServerOptions.GcConfig.UseGCV2 = true; + } +} + +class ZenStructuredCacheBucketsConfigOption : public LuaConfig::OptionValue +{ +public: + ZenStructuredCacheBucketsConfigOption(std::vector>& Value) : Value(Value) {} + virtual void Print(std::string_view Indent, StringBuilderBase& StringBuilder) override + { + if (Value.empty()) + { + StringBuilder.Append("{}"); + return; + } + LuaConfig::LuaContainerWriter Writer(StringBuilder, Indent); + for (const std::pair& Bucket : Value) + { + Writer.BeginContainer(""); + { + Writer.WriteValue("name", Bucket.first); + const ZenStructuredCacheBucketConfig& BucketConfig = Bucket.second; + + Writer.WriteValue("maxblocksize", fmt::format("{}", BucketConfig.MaxBlockSize)); + Writer.BeginContainer("memlayer"); + { + Writer.WriteValue("sizethreshold", fmt::format("{}", BucketConfig.MemCacheSizeThreshold)); + } + Writer.EndContainer(); + + Writer.WriteValue("payloadalignment", fmt::format("{}", BucketConfig.PayloadAlignment)); + Writer.WriteValue("largeobjectthreshold", fmt::format("{}", BucketConfig.PayloadAlignment)); + Writer.WriteValue("limitoverwrites", fmt::format("{}", BucketConfig.LimitOverwrites)); + } + Writer.EndContainer(); + } + } + virtual void Parse(sol::object Object) override + { + if (sol::optional Buckets = Object.as()) + { + for (const auto& Kv : Buckets.value()) + { + if (sol::optional Bucket = Kv.second.as()) + { + ZenStructuredCacheBucketConfig BucketConfig; + std::string Name = Kv.first.as(); + if (Name.empty()) + { + throw OptionParseException("Cache bucket option must have a name.", {}); + } + + const uint64_t MaxBlockSize = Bucket.value().get_or("maxblocksize", BucketConfig.MaxBlockSize); + if (MaxBlockSize == 0) + { + throw OptionParseException( + fmt::format("'maxblocksize' option for cache bucket '{}' is invalid. It must be non-zero.", Name), + {}); + } + BucketConfig.MaxBlockSize = MaxBlockSize; + + if (sol::optional Memlayer = Bucket.value().get_or("memlayer", sol::table())) + { + const uint64_t MemCacheSizeThreshold = Bucket.value().get_or("sizethreshold", BucketConfig.MemCacheSizeThreshold); + if (MemCacheSizeThreshold == 0) + { + throw OptionParseException( + fmt::format("'memlayer.sizethreshold' option for cache bucket '{}' is invalid. It must be non-zero.", Name), + {}); + } + BucketConfig.MemCacheSizeThreshold = Bucket.value().get_or("sizethreshold", BucketConfig.MemCacheSizeThreshold); + } + + const uint32_t PayloadAlignment = Bucket.value().get_or("payloadalignment", BucketConfig.PayloadAlignment); + if (PayloadAlignment == 0 || !IsPow2(PayloadAlignment)) + { + throw OptionParseException( + fmt::format( + "'payloadalignment' option for cache bucket '{}' is invalid. It needs to be non-zero and a power of two.", + Name), + {}); + } + BucketConfig.PayloadAlignment = PayloadAlignment; + + const uint64_t LargeObjectThreshold = Bucket.value().get_or("largeobjectthreshold", BucketConfig.LargeObjectThreshold); + if (LargeObjectThreshold == 0) + { + throw OptionParseException( + fmt::format("'largeobjectthreshold' option for cache bucket '{}' is invalid. It must be non-zero.", Name), + {}); + } + BucketConfig.LargeObjectThreshold = LargeObjectThreshold; + + BucketConfig.LimitOverwrites = Bucket.value().get_or("limitoverwrites", BucketConfig.LimitOverwrites); + + Value.push_back(std::make_pair(std::move(Name), BucketConfig)); + } + } + } + } + std::vector>& Value; +}; + +UpstreamCachePolicy +ParseUpstreamCachePolicy(std::string_view Options) +{ + if (Options == "readonly") + { + return UpstreamCachePolicy::Read; + } + else if (Options == "writeonly") + { + return UpstreamCachePolicy::Write; + } + else if (Options == "disabled") + { + return UpstreamCachePolicy::Disabled; + } + else + { + return UpstreamCachePolicy::ReadWrite; + } +} + +ZenObjectStoreConfig +ParseBucketConfigs(std::span Buckets) +{ + using namespace std::literals; + + ZenObjectStoreConfig Cfg; + + // split bucket args in the form of "{BucketName};{LocalPath}" + for (std::string_view Bucket : Buckets) + { + ZenObjectStoreConfig::BucketConfig NewBucket; + + if (auto Idx = Bucket.find_first_of(";"); Idx != std::string_view::npos) + { + NewBucket.Name = Bucket.substr(0, Idx); + NewBucket.Directory = Bucket.substr(Idx + 1); + } + else + { + NewBucket.Name = Bucket; + } + + Cfg.Buckets.push_back(std::move(NewBucket)); + } + + return Cfg; +} + +class CachePolicyOption : public LuaConfig::OptionValue +{ +public: + CachePolicyOption(UpstreamCachePolicy& Value) : Value(Value) {} + virtual void Print(std::string_view, StringBuilderBase& StringBuilder) override + { + switch (Value) + { + case UpstreamCachePolicy::Read: + StringBuilder.Append("readonly"); + break; + case UpstreamCachePolicy::Write: + StringBuilder.Append("writeonly"); + break; + case UpstreamCachePolicy::Disabled: + StringBuilder.Append("disabled"); + break; + case UpstreamCachePolicy::ReadWrite: + StringBuilder.Append("readwrite"); + break; + default: + ZEN_ASSERT(false); + } + } + virtual void Parse(sol::object Object) override + { + std::string PolicyString = Object.as(); + if (PolicyString == "readonly") + { + Value = UpstreamCachePolicy::Read; + } + else if (PolicyString == "writeonly") + { + Value = UpstreamCachePolicy::Write; + } + else if (PolicyString == "disabled") + { + Value = UpstreamCachePolicy::Disabled; + } + else if (PolicyString == "readwrite") + { + Value = UpstreamCachePolicy::ReadWrite; + } + } + UpstreamCachePolicy& Value; +}; + +class ZenAuthConfigOption : public LuaConfig::OptionValue +{ +public: + ZenAuthConfigOption(ZenAuthConfig& Value) : Value(Value) {} + virtual void Print(std::string_view Indent, StringBuilderBase& StringBuilder) override + { + if (Value.OpenIdProviders.empty()) + { + StringBuilder.Append("{}"); + return; + } + LuaConfig::LuaContainerWriter Writer(StringBuilder, Indent); + for (const ZenOpenIdProviderConfig& Config : Value.OpenIdProviders) + { + Writer.BeginContainer(""); + { + Writer.WriteValue("name", Config.Name); + Writer.WriteValue("url", Config.Url); + Writer.WriteValue("clientid", Config.ClientId); + } + Writer.EndContainer(); + } + } + virtual void Parse(sol::object Object) override + { + if (sol::optional OpenIdProviders = Object.as()) + { + for (const auto& Kv : OpenIdProviders.value()) + { + if (sol::optional OpenIdProvider = Kv.second.as()) + { + std::string Name = OpenIdProvider.value().get_or("name", std::string("Default")); + std::string Url = OpenIdProvider.value().get_or("url", std::string()); + std::string ClientId = OpenIdProvider.value().get_or("clientid", std::string()); + + Value.OpenIdProviders.push_back({.Name = std::move(Name), .Url = std::move(Url), .ClientId = std::move(ClientId)}); + } + } + } + } + ZenAuthConfig& Value; +}; + +class ZenObjectStoreConfigOption : public LuaConfig::OptionValue +{ +public: + ZenObjectStoreConfigOption(ZenObjectStoreConfig& Value) : Value(Value) {} + virtual void Print(std::string_view Indent, StringBuilderBase& StringBuilder) override + { + if (Value.Buckets.empty()) + { + StringBuilder.Append("{}"); + return; + } + LuaConfig::LuaContainerWriter Writer(StringBuilder, Indent); + for (const ZenObjectStoreConfig::BucketConfig& Config : Value.Buckets) + { + Writer.BeginContainer(""); + { + Writer.WriteValue("name", Config.Name); + std::string Directory = Config.Directory.string(); + LuaConfig::EscapeBackslash(Directory); + Writer.WriteValue("directory", Directory); + } + Writer.EndContainer(); + } + } + virtual void Parse(sol::object Object) override + { + if (sol::optional Buckets = Object.as()) + { + for (const auto& Kv : Buckets.value()) + { + if (sol::optional Bucket = Kv.second.as()) + { + std::string Name = Bucket.value().get_or("name", std::string("Default")); + std::string Directory = Bucket.value().get_or("directory", std::string()); + + Value.Buckets.push_back({.Name = std::move(Name), .Directory = MakeSafeAbsolutePath(Directory)}); + } + } + } + } + ZenObjectStoreConfig& Value; +}; + +std::shared_ptr +MakeOption(UpstreamCachePolicy& Value) +{ + return std::make_shared(Value); +}; + +std::shared_ptr +MakeOption(ZenAuthConfig& Value) +{ + return std::make_shared(Value); +}; + +std::shared_ptr +MakeOption(ZenObjectStoreConfig& Value) +{ + return std::make_shared(Value); +}; + +std::shared_ptr +MakeOption(std::vector>& Value) +{ + return std::make_shared(Value); +}; + +void +ParseConfigFile(const std::filesystem::path& Path, + ZenStorageServerOptions& ServerOptions, + const cxxopts::ParseResult& CmdLineResult, + std::string_view OutputConfigFile) +{ + ZEN_TRACE_CPU("ParseConfigFile"); + + using namespace std::literals; + + LuaConfig::Options LuaOptions; + + AddServerConfigOptions(LuaOptions, ServerOptions); + + ////// server + LuaOptions.AddOption("server.pluginsconfigfile"sv, ServerOptions.PluginsConfigFile, "plugins-config"sv); + + ////// objectstore + LuaOptions.AddOption("server.objectstore.enabled"sv, ServerOptions.ObjectStoreEnabled, "objectstore-enabled"sv); + LuaOptions.AddOption("server.objectstore.buckets"sv, ServerOptions.ObjectStoreConfig); + + ////// buildsstore + LuaOptions.AddOption("server.buildstore.enabled"sv, ServerOptions.BuildStoreConfig.Enabled, "buildstore-enabled"sv); + LuaOptions.AddOption("server.buildstore.disksizelimit"sv, ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit, "buildstore-disksizelimit"); + + ////// cache + LuaOptions.AddOption("cache.enable"sv, ServerOptions.StructuredCacheConfig.Enabled); + LuaOptions.AddOption("cache.writelog"sv, ServerOptions.StructuredCacheConfig.WriteLogEnabled, "cache-write-log"sv); + LuaOptions.AddOption("cache.accesslog"sv, ServerOptions.StructuredCacheConfig.AccessLogEnabled, "cache-access-log"sv); + + LuaOptions.AddOption("cache.buckets"sv, ServerOptions.StructuredCacheConfig.PerBucketConfigs, "cache.buckets"sv); + + LuaOptions.AddOption("cache.memlayer.sizethreshold"sv, + ServerOptions.StructuredCacheConfig.BucketConfig.MemCacheSizeThreshold, + "cache-memlayer-sizethreshold"sv); + LuaOptions.AddOption("cache.memlayer.targetfootprint"sv, + ServerOptions.StructuredCacheConfig.MemTargetFootprintBytes, + "cache-memlayer-targetfootprint"sv); + LuaOptions.AddOption("cache.memlayer.triminterval"sv, + ServerOptions.StructuredCacheConfig.MemTrimIntervalSeconds, + "cache-memlayer-triminterval"sv); + LuaOptions.AddOption("cache.memlayer.maxage"sv, ServerOptions.StructuredCacheConfig.MemMaxAgeSeconds, "cache-memlayer-maxage"sv); + + LuaOptions.AddOption("cache.bucket.maxblocksize"sv, + ServerOptions.StructuredCacheConfig.BucketConfig.MaxBlockSize, + "cache-bucket-maxblocksize"sv); + LuaOptions.AddOption("cache.bucket.memlayer.sizethreshold"sv, + ServerOptions.StructuredCacheConfig.BucketConfig.MemCacheSizeThreshold, + "cache-bucket-memlayer-sizethreshold"sv); + LuaOptions.AddOption("cache.bucket.payloadalignment"sv, + ServerOptions.StructuredCacheConfig.BucketConfig.PayloadAlignment, + "cache-bucket-payloadalignment"sv); + LuaOptions.AddOption("cache.bucket.largeobjectthreshold"sv, + ServerOptions.StructuredCacheConfig.BucketConfig.LargeObjectThreshold, + "cache-bucket-largeobjectthreshold"sv); + LuaOptions.AddOption("cache.bucket.limitoverwrites"sv, + ServerOptions.StructuredCacheConfig.BucketConfig.LimitOverwrites, + "cache-bucket-limit-overwrites"sv); + + ////// cache.upstream + LuaOptions.AddOption("cache.upstream.policy"sv, ServerOptions.UpstreamCacheConfig.CachePolicy, "upstream-cache-policy"sv); + LuaOptions.AddOption("cache.upstream.upstreamthreadcount"sv, + ServerOptions.UpstreamCacheConfig.UpstreamThreadCount, + "upstream-thread-count"sv); + LuaOptions.AddOption("cache.upstream.connecttimeoutms"sv, + ServerOptions.UpstreamCacheConfig.ConnectTimeoutMilliseconds, + "upstream-connect-timeout-ms"sv); + LuaOptions.AddOption("cache.upstream.timeoutms"sv, ServerOptions.UpstreamCacheConfig.TimeoutMilliseconds, "upstream-timeout-ms"sv); + + ////// cache.upstream.jupiter + LuaOptions.AddOption("cache.upstream.jupiter.name"sv, ServerOptions.UpstreamCacheConfig.JupiterConfig.Name); + LuaOptions.AddOption("cache.upstream.jupiter.url"sv, ServerOptions.UpstreamCacheConfig.JupiterConfig.Url, "upstream-jupiter-url"sv); + LuaOptions.AddOption("cache.upstream.jupiter.oauthprovider"sv, + ServerOptions.UpstreamCacheConfig.JupiterConfig.OAuthUrl, + "upstream-jupiter-oauth-url"sv); + LuaOptions.AddOption("cache.upstream.jupiter.oauthclientid"sv, + ServerOptions.UpstreamCacheConfig.JupiterConfig.OAuthClientId, + "upstream-jupiter-oauth-clientid"); + LuaOptions.AddOption("cache.upstream.jupiter.oauthclientsecret"sv, + ServerOptions.UpstreamCacheConfig.JupiterConfig.OAuthClientSecret, + "upstream-jupiter-oauth-clientsecret"sv); + LuaOptions.AddOption("cache.upstream.jupiter.openidprovider"sv, + ServerOptions.UpstreamCacheConfig.JupiterConfig.OpenIdProvider, + "upstream-jupiter-openid-provider"sv); + LuaOptions.AddOption("cache.upstream.jupiter.token"sv, + ServerOptions.UpstreamCacheConfig.JupiterConfig.AccessToken, + "upstream-jupiter-token"sv); + LuaOptions.AddOption("cache.upstream.jupiter.namespace"sv, + ServerOptions.UpstreamCacheConfig.JupiterConfig.Namespace, + "upstream-jupiter-namespace"sv); + LuaOptions.AddOption("cache.upstream.jupiter.ddcnamespace"sv, + ServerOptions.UpstreamCacheConfig.JupiterConfig.DdcNamespace, + "upstream-jupiter-namespace-ddc"sv); + + ////// cache.upstream.zen + // LuaOptions.AddOption("cache.upstream.zen"sv, ServerOptions.UpstreamCacheConfig.ZenConfig); + LuaOptions.AddOption("cache.upstream.zen.name"sv, ServerOptions.UpstreamCacheConfig.ZenConfig.Name); + LuaOptions.AddOption("cache.upstream.zen.dns"sv, ServerOptions.UpstreamCacheConfig.ZenConfig.Dns); + LuaOptions.AddOption("cache.upstream.zen.url"sv, ServerOptions.UpstreamCacheConfig.ZenConfig.Urls); + + ////// gc + LuaOptions.AddOption("gc.enabled"sv, ServerOptions.GcConfig.Enabled, "gc-enabled"sv); + LuaOptions.AddOption("gc.v2"sv, ServerOptions.GcConfig.UseGCV2, "gc-v2"sv); + + LuaOptions.AddOption("gc.monitorintervalseconds"sv, ServerOptions.GcConfig.MonitorIntervalSeconds, "gc-monitor-interval-seconds"sv); + LuaOptions.AddOption("gc.intervalseconds"sv, ServerOptions.GcConfig.IntervalSeconds, "gc-interval-seconds"sv); + LuaOptions.AddOption("gc.collectsmallobjects"sv, ServerOptions.GcConfig.CollectSmallObjects, "gc-small-objects"sv); + LuaOptions.AddOption("gc.diskreservesize"sv, ServerOptions.GcConfig.DiskReserveSize, "disk-reserve-size"sv); + LuaOptions.AddOption("gc.disksizesoftlimit"sv, ServerOptions.GcConfig.DiskSizeSoftLimit, "gc-disksize-softlimit"sv); + LuaOptions.AddOption("gc.lowdiskspacethreshold"sv, + ServerOptions.GcConfig.MinimumFreeDiskSpaceToAllowWrites, + "gc-low-diskspace-threshold"sv); + LuaOptions.AddOption("gc.lightweightintervalseconds"sv, + ServerOptions.GcConfig.LightweightIntervalSeconds, + "gc-lightweight-interval-seconds"sv); + LuaOptions.AddOption("gc.compactblockthreshold"sv, + ServerOptions.GcConfig.CompactBlockUsageThresholdPercent, + "gc-compactblock-threshold"sv); + LuaOptions.AddOption("gc.verbose"sv, ServerOptions.GcConfig.Verbose, "gc-verbose"sv); + LuaOptions.AddOption("gc.single-threaded"sv, ServerOptions.GcConfig.SingleThreaded, "gc-single-threaded"sv); + LuaOptions.AddOption("gc.cache.attachment.store"sv, ServerOptions.GcConfig.StoreCacheAttachmentMetaData, "gc-cache-attachment-store"); + LuaOptions.AddOption("gc.projectstore.attachment.store"sv, + ServerOptions.GcConfig.StoreProjectAttachmentMetaData, + "gc-projectstore-attachment-store"); + LuaOptions.AddOption("gc.attachment.passes"sv, ServerOptions.GcConfig.AttachmentPassCount, "gc-attachment-passes"sv); + LuaOptions.AddOption("gc.validation"sv, ServerOptions.GcConfig.EnableValidation, "gc-validation"); + + LuaOptions.AddOption("gc.cache.maxdurationseconds"sv, ServerOptions.GcConfig.Cache.MaxDurationSeconds, "gc-cache-duration-seconds"sv); + LuaOptions.AddOption("gc.projectstore.duration.seconds"sv, + ServerOptions.GcConfig.ProjectStore.MaxDurationSeconds, + "gc-projectstore-duration-seconds"); + LuaOptions.AddOption("gc.buildstore.duration.seconds"sv, + ServerOptions.GcConfig.BuildStore.MaxDurationSeconds, + "gc-buildstore-duration-seconds"); + + ////// security + LuaOptions.AddOption("security.encryptionaeskey"sv, ServerOptions.EncryptionKey, "encryption-aes-key"sv); + LuaOptions.AddOption("security.encryptionaesiv"sv, ServerOptions.EncryptionIV, "encryption-aes-iv"sv); + LuaOptions.AddOption("security.openidproviders"sv, ServerOptions.AuthConfig); + + ////// workspaces + LuaOptions.AddOption("workspaces.enabled"sv, ServerOptions.WorksSpacesConfig.Enabled, "workspaces-enabled"sv); + LuaOptions.AddOption("workspaces.allowconfigchanges"sv, + ServerOptions.WorksSpacesConfig.AllowConfigurationChanges, + "workspaces-allow-changes"sv); + + LuaOptions.Parse(Path, CmdLineResult); + + // These have special command line processing so we make sure we export them if they were configured on command line + if (!ServerOptions.AuthConfig.OpenIdProviders.empty()) + { + LuaOptions.Touch("security.openidproviders"sv); + } + if (!ServerOptions.ObjectStoreConfig.Buckets.empty()) + { + LuaOptions.Touch("server.objectstore.buckets"sv); + } + if (!ServerOptions.StructuredCacheConfig.PerBucketConfigs.empty()) + { + LuaOptions.Touch("cache.buckets"sv); + } + + if (!OutputConfigFile.empty()) + { + std::filesystem::path WritePath(MakeSafeAbsolutePath(OutputConfigFile)); + ExtendableStringBuilder<512> ConfigStringBuilder; + LuaOptions.Print(ConfigStringBuilder, CmdLineResult); + BasicFile Output; + Output.Open(WritePath, BasicFile::Mode::kTruncate); + Output.Write(ConfigStringBuilder.Data(), ConfigStringBuilder.Size(), 0); + } +} + +void +ParsePluginsConfigFile(const std::filesystem::path& Path, ZenStorageServerOptions& ServerOptions, int BasePort) +{ + using namespace std::literals; + + IoBuffer Body = IoBufferBuilder::MakeFromFile(Path); + std::string JsonText(reinterpret_cast(Body.GetData()), Body.GetSize()); + std::string JsonError; + json11::Json PluginsInfo = json11::Json::parse(JsonText, JsonError); + if (!JsonError.empty()) + { + ZEN_WARN("failed parsing plugins config file '{}'. Reason: '{}'", Path, JsonError); + return; + } + for (const json11::Json& PluginInfo : PluginsInfo.array_items()) + { + if (!PluginInfo.is_object()) + { + ZEN_WARN("the json file '{}' does not contain a valid plugin definition, object expected, got '{}'", Path, PluginInfo.dump()); + continue; + } + + HttpServerPluginConfig Config = {}; + + bool bNeedsPort = true; + + for (const std::pair& Items : PluginInfo.object_items()) + { + if (!Items.second.is_string()) + { + ZEN_WARN("the json file '{}' does not contain a valid plugins definition, string expected, got '{}'", + Path, + Items.second.dump()); + continue; + } + + const std::string& Name = Items.first; + const std::string& Value = Items.second.string_value(); + + if (Name == "name"sv) + Config.PluginName = Value; + else + { + Config.PluginOptions.push_back({Name, Value}); + + if (Name == "port"sv) + { + bNeedsPort = false; + } + } + } + + // add a default base port in case if json config didn't provide one + if (bNeedsPort) + { + Config.PluginOptions.push_back({"port", std::to_string(BasePort)}); + } + + ServerOptions.HttpServerConfig.PluginConfigs.push_back(Config); + } +} + +void +ZenStorageServerCmdLineOptions::AddCliOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions) +{ + options.add_options()("snapshot-dir", + "Specify a snapshot of server state to mirror into the persistence root at startup", + cxxopts::value(BaseSnapshotDir)); + options.add_options()("plugins-config", "Path to plugins config file", cxxopts::value(PluginsConfigFile)); + options.add_options()("scrub", + "Validate state at startup", + cxxopts::value(ServerOptions.ScrubOptions)->implicit_value("yes"), + "(nocas,nogc,nodelete,yes,no)*"); + + AddSecurityOptions(options, ServerOptions); + AddCacheOptions(options, ServerOptions); + AddGcOptions(options, ServerOptions); + AddObjectStoreOptions(options, ServerOptions); + AddBuildStoreOptions(options, ServerOptions); + AddWorkspacesOptions(options, ServerOptions); +} + +void +ZenStorageServerCmdLineOptions::AddSecurityOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions) +{ + options.add_option("security", + "", + "encryption-aes-key", + "256 bit AES encryption key", + cxxopts::value(ServerOptions.EncryptionKey), + ""); + + options.add_option("security", + "", + "encryption-aes-iv", + "128 bit AES encryption initialization vector", + cxxopts::value(ServerOptions.EncryptionIV), + ""); + + options.add_option("security", + "", + "openid-provider-name", + "Open ID provider name", + cxxopts::value(OpenIdProviderName), + "Default"); + + options.add_option("security", "", "openid-provider-url", "Open ID provider URL", cxxopts::value(OpenIdProviderUrl), ""); + options.add_option("security", "", "openid-client-id", "Open ID client ID", cxxopts::value(OpenIdClientId), ""); +} + +void +ZenStorageServerCmdLineOptions::AddCacheOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions) +{ + options.add_option("cache", + "", + "upstream-cache-policy", + "", + cxxopts::value(UpstreamCachePolicyOptions)->default_value(""), + "Upstream cache policy (readwrite|readonly|writeonly|disabled)"); + + options.add_option("cache", + "", + "upstream-jupiter-url", + "URL to a Jupiter instance", + cxxopts::value(ServerOptions.UpstreamCacheConfig.JupiterConfig.Url)->default_value(""), + ""); + + options.add_option("cache", + "", + "upstream-jupiter-oauth-url", + "URL to the OAuth provier", + cxxopts::value(ServerOptions.UpstreamCacheConfig.JupiterConfig.OAuthUrl)->default_value(""), + ""); + + options.add_option("cache", + "", + "upstream-jupiter-oauth-clientid", + "The OAuth client ID", + cxxopts::value(ServerOptions.UpstreamCacheConfig.JupiterConfig.OAuthClientId)->default_value(""), + ""); + + options.add_option("cache", + "", + "upstream-jupiter-oauth-clientsecret", + "The OAuth client secret", + cxxopts::value(ServerOptions.UpstreamCacheConfig.JupiterConfig.OAuthClientSecret)->default_value(""), + ""); + + options.add_option("cache", + "", + "upstream-jupiter-openid-provider", + "Name of a registered Open ID provider", + cxxopts::value(ServerOptions.UpstreamCacheConfig.JupiterConfig.OpenIdProvider)->default_value(""), + ""); + + options.add_option("cache", + "", + "upstream-jupiter-token", + "A static authentication token", + cxxopts::value(ServerOptions.UpstreamCacheConfig.JupiterConfig.AccessToken)->default_value(""), + ""); + + options.add_option("cache", + "", + "upstream-jupiter-namespace", + "The Common Blob Store API namespace", + cxxopts::value(ServerOptions.UpstreamCacheConfig.JupiterConfig.Namespace)->default_value(""), + ""); + + options.add_option("cache", + "", + "upstream-jupiter-namespace-ddc", + "The lecacy DDC namespace", + cxxopts::value(ServerOptions.UpstreamCacheConfig.JupiterConfig.DdcNamespace)->default_value(""), + ""); + + options.add_option("cache", + "", + "upstream-zen-url", + "URL to remote Zen server. Use a comma separated list to choose the one with the best latency.", + cxxopts::value>(ServerOptions.UpstreamCacheConfig.ZenConfig.Urls), + ""); + + options.add_option("cache", + "", + "upstream-zen-dns", + "DNS that resolves to one or more Zen server instance(s)", + cxxopts::value>(ServerOptions.UpstreamCacheConfig.ZenConfig.Dns), + ""); + + options.add_option("cache", + "", + "upstream-thread-count", + "Number of threads used for upstream procsssing", + cxxopts::value(ServerOptions.UpstreamCacheConfig.UpstreamThreadCount)->default_value("4"), + ""); + + options.add_option("cache", + "", + "upstream-connect-timeout-ms", + "Connect timeout in millisecond(s). Default 5000 ms.", + cxxopts::value(ServerOptions.UpstreamCacheConfig.ConnectTimeoutMilliseconds)->default_value("5000"), + ""); + + options.add_option("cache", + "", + "upstream-timeout-ms", + "Timeout in millisecond(s). Default 0 ms", + cxxopts::value(ServerOptions.UpstreamCacheConfig.TimeoutMilliseconds)->default_value("0"), + ""); + + options.add_option("cache", + "", + "cache-write-log", + "Whether cache write log is enabled", + cxxopts::value(ServerOptions.StructuredCacheConfig.WriteLogEnabled)->default_value("false"), + ""); + + options.add_option("cache", + "", + "cache-access-log", + "Whether cache access log is enabled", + cxxopts::value(ServerOptions.StructuredCacheConfig.AccessLogEnabled)->default_value("false"), + ""); + + options.add_option( + "cache", + "", + "cache-memlayer-sizethreshold", + "The largest size of a cache entry that may be cached in memory. Default set to 1024 (1 Kb). Set to 0 to disable memory " + "caching. " + "Obsolete, replaced by `--cache-bucket-memlayer-sizethreshold`", + cxxopts::value(ServerOptions.StructuredCacheConfig.BucketConfig.MemCacheSizeThreshold)->default_value("1024"), + ""); + + options.add_option("cache", + "", + "cache-memlayer-targetfootprint", + "Max allowed memory used by cache memory layer per namespace in bytes. Default set to 536870912 (512 Mb).", + cxxopts::value(ServerOptions.StructuredCacheConfig.MemTargetFootprintBytes)->default_value("536870912"), + ""); + + options.add_option("cache", + "", + "cache-memlayer-triminterval", + "Minimum time between each attempt to trim cache memory layers in seconds. Default set to 60 (1 min).", + cxxopts::value(ServerOptions.StructuredCacheConfig.MemTrimIntervalSeconds)->default_value("60"), + ""); + + options.add_option("cache", + "", + "cache-memlayer-maxage", + "Maximum age of payloads when trimming cache memory layers in seconds. Default set to 86400 (1 day).", + cxxopts::value(ServerOptions.StructuredCacheConfig.MemMaxAgeSeconds)->default_value("86400"), + ""); + + options.add_option("cache", + "", + "cache-bucket-maxblocksize", + "Max size of cache bucket blocks. Default set to 1073741824 (1GB).", + cxxopts::value(ServerOptions.StructuredCacheConfig.BucketConfig.MaxBlockSize)->default_value("1073741824"), + ""); + + options.add_option("cache", + "", + "cache-bucket-payloadalignment", + "Payload alignement for cache bucket blocks. Default set to 16.", + cxxopts::value(ServerOptions.StructuredCacheConfig.BucketConfig.PayloadAlignment)->default_value("16"), + ""); + + options.add_option( + "cache", + "", + "cache-bucket-largeobjectthreshold", + "Threshold for storing cache bucket values as loose files. Default set to 131072 (128 KB).", + cxxopts::value(ServerOptions.StructuredCacheConfig.BucketConfig.LargeObjectThreshold)->default_value("131072"), + ""); + + options.add_option( + "cache", + "", + "cache-bucket-memlayer-sizethreshold", + "The largest size of a cache entry that may be cached in memory. Default set to 1024 (1 Kb). Set to 0 to disable memory " + "caching.", + cxxopts::value(ServerOptions.StructuredCacheConfig.BucketConfig.MemCacheSizeThreshold)->default_value("1024"), + ""); + + options.add_option("cache", + "", + "cache-bucket-limit-overwrites", + "Whether to require policy flag pattern before allowing overwrites in cache bucket", + cxxopts::value(ServerOptions.StructuredCacheConfig.BucketConfig.LimitOverwrites)->default_value("false"), + ""); +} + +void +ZenStorageServerCmdLineOptions::AddGcOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions) +{ + options.add_option("gc", + "", + "gc-cache-attachment-store", + "Enable storing attachments referenced by a cache record in block store meta data.", + cxxopts::value(ServerOptions.GcConfig.StoreCacheAttachmentMetaData)->default_value("false"), + ""); + + options.add_option("gc", + "", + "gc-projectstore-attachment-store", + "Enable storing attachments referenced by project oplogs in meta data.", + cxxopts::value(ServerOptions.GcConfig.StoreProjectAttachmentMetaData)->default_value("false"), + ""); + + options.add_option("gc", + "", + "gc-validation", + "Enable validation of references after full GC.", + cxxopts::value(ServerOptions.GcConfig.EnableValidation)->default_value("true"), + ""); + + options.add_option("gc", + "", + "gc-enabled", + "Whether garbage collection is enabled or not.", + cxxopts::value(ServerOptions.GcConfig.Enabled)->default_value("true"), + ""); + + options.add_option("gc", + "", + "gc-v2", + "Use V2 of GC implementation or not.", + cxxopts::value(ServerOptions.GcConfig.UseGCV2)->default_value("true"), + ""); + + options.add_option("gc", + "", + "gc-small-objects", + "Whether garbage collection of small objects is enabled or not.", + cxxopts::value(ServerOptions.GcConfig.CollectSmallObjects)->default_value("true"), + ""); + + options.add_option("gc", + "", + "gc-interval-seconds", + "Garbage collection interval in seconds. Default set to 3600 (1 hour).", + cxxopts::value(ServerOptions.GcConfig.IntervalSeconds)->default_value("3600"), + ""); + + options.add_option("gc", + "", + "gc-lightweight-interval-seconds", + "Lightweight garbage collection interval in seconds. Default set to 900 (30 min).", + cxxopts::value(ServerOptions.GcConfig.LightweightIntervalSeconds)->default_value("900"), + ""); + + options.add_option("gc", + "", + "gc-cache-duration-seconds", + "Max duration in seconds before Z$ entries get evicted. Default set to 1209600 (2 weeks)", + cxxopts::value(ServerOptions.GcConfig.Cache.MaxDurationSeconds)->default_value("1209600"), + ""); + + options.add_option("gc", + "", + "gc-projectstore-duration-seconds", + "Max duration in seconds before project store entries get evicted. Default set to 1209600 (2 weeks)", + cxxopts::value(ServerOptions.GcConfig.ProjectStore.MaxDurationSeconds)->default_value("1209600"), + ""); + + options.add_option("gc", + "", + "gc-buildstore-duration-seconds", + "Max duration in seconds before build store entries get evicted. Default set to 604800 (1 week)", + cxxopts::value(ServerOptions.GcConfig.BuildStore.MaxDurationSeconds)->default_value("604800"), + ""); + + options.add_option("gc", + "", + "disk-reserve-size", + "Size of gc disk reserve in bytes. Default set to 268435456 (256 Mb). Set to zero to disable.", + cxxopts::value(ServerOptions.GcConfig.DiskReserveSize)->default_value("268435456"), + ""); + + options.add_option("gc", + "", + "gc-monitor-interval-seconds", + "Garbage collection monitoring interval in seconds. Default set to 30 (30 seconds)", + cxxopts::value(ServerOptions.GcConfig.MonitorIntervalSeconds)->default_value("30"), + ""); + + options.add_option("gc", + "", + "gc-low-diskspace-threshold", + "Minimum free space on disk to allow writes to disk. Default set to 268435456 (256 Mb). Set to zero to disable.", + cxxopts::value(ServerOptions.GcConfig.MinimumFreeDiskSpaceToAllowWrites)->default_value("268435456"), + ""); + + options.add_option("gc", + "", + "gc-disksize-softlimit", + "Garbage collection disk usage soft limit. Default set to 0 (Off).", + cxxopts::value(ServerOptions.GcConfig.DiskSizeSoftLimit)->default_value("0"), + ""); + + options.add_option("gc", + "", + "gc-compactblock-threshold", + "Garbage collection - how much of a compact block should be used to skip compacting the block. 0 - compact only " + "empty eligible blocks, 100 - compact all non-full eligible blocks.", + cxxopts::value(ServerOptions.GcConfig.CompactBlockUsageThresholdPercent)->default_value("60"), + ""); + + options.add_option("gc", + "", + "gc-verbose", + "Enable verbose logging for GC.", + cxxopts::value(ServerOptions.GcConfig.Verbose)->default_value("false"), + ""); + + options.add_option("gc", + "", + "gc-single-threaded", + "Force GC to run single threaded.", + cxxopts::value(ServerOptions.GcConfig.SingleThreaded)->default_value("false"), + ""); + + options.add_option("gc", + "", + "gc-attachment-passes", + "Limit the range of unreferenced attachments included in GC check by breaking it into passes. Default is one pass " + "which includes all the attachments.", + cxxopts::value(ServerOptions.GcConfig.AttachmentPassCount)->default_value("1"), + ""); +} + +void +ZenStorageServerCmdLineOptions::AddObjectStoreOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions) +{ + options.add_option("objectstore", + "", + "objectstore-enabled", + "Whether the object store is enabled or not.", + cxxopts::value(ServerOptions.ObjectStoreEnabled)->default_value("false"), + ""); + + options.add_option("objectstore", + "", + "objectstore-bucket", + "Object store bucket mappings.", + cxxopts::value>(BucketConfigs), + ""); +} + +void +ZenStorageServerCmdLineOptions::AddBuildStoreOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions) +{ + options.add_option("buildstore", + "", + "buildstore-enabled", + "Whether the builds store is enabled or not.", + cxxopts::value(ServerOptions.BuildStoreConfig.Enabled)->default_value("false"), + ""); + options.add_option("buildstore", + "", + "buildstore-disksizelimit", + "Max number of bytes before build store entries get evicted. Default set to 1099511627776 (1TB week)", + cxxopts::value(ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit)->default_value("1099511627776"), + ""); +} + +void +ZenStorageServerCmdLineOptions::AddWorkspacesOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions) +{ + options.add_option("workspaces", + "", + "workspaces-enabled", + "", + cxxopts::value(ServerOptions.WorksSpacesConfig.Enabled)->default_value("true"), + "Enable workspaces support with folder sharing"); + + options.add_option("workspaces", + "", + "workspaces-allow-changes", + "", + cxxopts::value(ServerOptions.WorksSpacesConfig.AllowConfigurationChanges)->default_value("false"), + "Allow adding/modifying/deleting of workspace and shares via http endpoint"); +} + +void +ZenStorageServerCmdLineOptions::ApplyOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions) +{ + ServerOptions.BaseSnapshotDir = MakeSafeAbsolutePath(BaseSnapshotDir); + ServerOptions.PluginsConfigFile = MakeSafeAbsolutePath(PluginsConfigFile); + ServerOptions.UpstreamCacheConfig.CachePolicy = ParseUpstreamCachePolicy(UpstreamCachePolicyOptions); + + if (!BaseSnapshotDir.empty()) + { + if (ServerOptions.DataDir.empty()) + throw OptionParseException("'--snapshot-dir' requires '--data-dir'", options.help()); + + if (!IsDir(ServerOptions.BaseSnapshotDir)) + throw std::runtime_error(fmt::format("'--snapshot-dir' ('{}') must be a directory", ServerOptions.BaseSnapshotDir)); + } + + if (OpenIdProviderUrl.empty() == false) + { + if (OpenIdClientId.empty()) + { + throw OptionParseException("'--openid-provider-url' requires '--openid-client-id'", options.help()); + } + + ServerOptions.AuthConfig.OpenIdProviders.push_back( + {.Name = OpenIdProviderName, .Url = OpenIdProviderUrl, .ClientId = OpenIdClientId}); + } + + ServerOptions.ObjectStoreConfig = ParseBucketConfigs(BucketConfigs); +} + +} // namespace zen diff --git a/src/zenserver/storage/storageconfig.h b/src/zenserver/storage/storageconfig.h new file mode 100644 index 000000000..ca0cf4135 --- /dev/null +++ b/src/zenserver/storage/storageconfig.h @@ -0,0 +1,203 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "config/config.h" + +namespace zen { + +struct ZenUpstreamJupiterConfig +{ + std::string Name; + std::string Url; + std::string OAuthUrl; + std::string OAuthClientId; + std::string OAuthClientSecret; + std::string OpenIdProvider; + std::string AccessToken; + std::string Namespace; + std::string DdcNamespace; +}; + +struct ZenUpstreamZenConfig +{ + std::string Name; + std::vector Urls; + std::vector Dns; +}; + +enum class UpstreamCachePolicy : uint8_t +{ + Disabled = 0, + Read = 1 << 0, + Write = 1 << 1, + ReadWrite = Read | Write +}; + +struct ZenUpstreamCacheConfig +{ + ZenUpstreamJupiterConfig JupiterConfig; + ZenUpstreamZenConfig ZenConfig; + int32_t UpstreamThreadCount = 4; + int32_t ConnectTimeoutMilliseconds = 5000; + int32_t TimeoutMilliseconds = 0; + UpstreamCachePolicy CachePolicy = UpstreamCachePolicy::ReadWrite; +}; + +struct ZenCacheEvictionPolicy +{ + int32_t MaxDurationSeconds = 24 * 60 * 60; +}; + +struct ZenProjectStoreEvictionPolicy +{ + int32_t MaxDurationSeconds = 7 * 24 * 60 * 60; +}; + +struct ZenBuildStoreEvictionPolicy +{ + int32_t MaxDurationSeconds = 3 * 24 * 60 * 60; +}; + +struct ZenGcConfig +{ + // ZenCasEvictionPolicy Cas; + ZenCacheEvictionPolicy Cache; + ZenProjectStoreEvictionPolicy ProjectStore; + ZenBuildStoreEvictionPolicy BuildStore; + int32_t MonitorIntervalSeconds = 30; + int32_t IntervalSeconds = 0; + bool CollectSmallObjects = true; + bool Enabled = true; + uint64_t DiskReserveSize = 1ul << 28; + uint64_t DiskSizeSoftLimit = 0; + int32_t LightweightIntervalSeconds = 0; + uint64_t MinimumFreeDiskSpaceToAllowWrites = 1ul << 28; + bool UseGCV2 = false; + uint32_t CompactBlockUsageThresholdPercent = 90; + bool Verbose = false; + bool SingleThreaded = false; + static constexpr uint16_t GcMaxAttachmentPassCount = 256; + uint16_t AttachmentPassCount = 1; + bool StoreCacheAttachmentMetaData = false; + bool StoreProjectAttachmentMetaData = false; + bool EnableValidation = true; +}; + +struct ZenOpenIdProviderConfig +{ + std::string Name; + std::string Url; + std::string ClientId; +}; + +struct ZenAuthConfig +{ + std::vector OpenIdProviders; +}; + +struct ZenObjectStoreConfig +{ + struct BucketConfig + { + std::string Name; + std::filesystem::path Directory; + }; + + std::vector Buckets; +}; + +struct ZenStructuredCacheBucketConfig +{ + uint64_t MaxBlockSize = 1ull << 30; + uint32_t PayloadAlignment = 1u << 4; + uint64_t MemCacheSizeThreshold = 1 * 1024; + uint64_t LargeObjectThreshold = 128 * 1024; + bool LimitOverwrites = false; +}; + +struct ZenStructuredCacheConfig +{ + bool Enabled = true; + bool WriteLogEnabled = false; + bool AccessLogEnabled = false; + std::vector> PerBucketConfigs; + ZenStructuredCacheBucketConfig BucketConfig; + uint64_t MemTargetFootprintBytes = 512 * 1024 * 1024; + uint64_t MemTrimIntervalSeconds = 60; + uint64_t MemMaxAgeSeconds = gsl::narrow(std::chrono::seconds(std::chrono::days(1)).count()); +}; + +struct ZenProjectStoreConfig +{ + bool StoreCacheAttachmentMetaData = false; + bool StoreProjectAttachmentMetaData = false; +}; + +struct ZenBuildStoreConfig +{ + bool Enabled = false; + uint64_t MaxDiskSpaceLimit = 1u * 1024u * 1024u * 1024u * 1024u; // 1TB +}; + +struct ZenWorkspacesConfig +{ + bool Enabled = false; + bool AllowConfigurationChanges = false; +}; + +struct ZenStorageServerOptions : public ZenServerOptions +{ + ZenUpstreamCacheConfig UpstreamCacheConfig; + ZenGcConfig GcConfig; + ZenAuthConfig AuthConfig; + ZenObjectStoreConfig ObjectStoreConfig; + ZenStructuredCacheConfig StructuredCacheConfig; + ZenProjectStoreConfig ProjectStoreConfig; + ZenBuildStoreConfig BuildStoreConfig; + ZenWorkspacesConfig WorksSpacesConfig; + std::filesystem::path PluginsConfigFile; // Path to plugins config file + std::filesystem::path BaseSnapshotDir; // Path to server state snapshot (will be copied into data dir on start) + bool ObjectStoreEnabled = false; + std::string ScrubOptions; +}; + +void ParseConfigFile(const std::filesystem::path& Path, + ZenStorageServerOptions& ServerOptions, + const cxxopts::ParseResult& CmdLineResult, + std::string_view OutputConfigFile); + +void ParsePluginsConfigFile(const std::filesystem::path& Path, ZenStorageServerOptions& ServerOptions, int BasePort); +void ValidateOptions(ZenStorageServerOptions& ServerOptions); + +struct ZenStorageServerCmdLineOptions +{ + // Note to those adding future options; std::filesystem::path-type options + // must be read into a std::string first. As of cxxopts-3.0.0 it uses a >> + // stream operator to convert argv value into the options type. std::fs::path + // expects paths in streams to be quoted but argv paths are unquoted. By + // going into a std::string first, paths with whitespace parse correctly. + std::string PluginsConfigFile; + std::string BaseSnapshotDir; + + void AddCliOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions); + void ApplyOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions); + + std::string OpenIdProviderName; + std::string OpenIdProviderUrl; + std::string OpenIdClientId; + + void AddSecurityOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions); + + std::string UpstreamCachePolicyOptions; + + void AddCacheOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions); + + void AddGcOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions); + + std::vector BucketConfigs; + + void AddObjectStoreOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions); + void AddBuildStoreOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions); + void AddWorkspacesOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions); +}; + +} // namespace zen diff --git a/src/zenserver/storage/upstream/upstream.h b/src/zenserver/storage/upstream/upstream.h new file mode 100644 index 000000000..b4fd03983 --- /dev/null +++ b/src/zenserver/storage/upstream/upstream.h @@ -0,0 +1,7 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "upstreamcache.h" +#include "upstreamservice.h" +#include "zen.h" diff --git a/src/zenserver/storage/upstream/upstreamcache.cpp b/src/zenserver/storage/upstream/upstreamcache.cpp new file mode 100644 index 000000000..f7ae5f973 --- /dev/null +++ b/src/zenserver/storage/upstream/upstreamcache.cpp @@ -0,0 +1,2134 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "upstreamcache.h" +#include "zen.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include + +#include +#include + +#include "diag/logging.h" +#include "storage/cache/httpstructuredcache.h" + +#include + +#include +#include +#include +#include + +namespace zen { + +using namespace std::literals; + +namespace detail { + + class UpstreamStatus + { + public: + UpstreamEndpointState EndpointState() const { return static_cast(m_State.load(std::memory_order_relaxed)); } + + UpstreamEndpointStatus EndpointStatus() const + { + const UpstreamEndpointState State = EndpointState(); + { + std::unique_lock _(m_Mutex); + return {.Reason = m_ErrorText, .State = State}; + } + } + + void Set(UpstreamEndpointState NewState) + { + m_State.store(static_cast(NewState), std::memory_order_relaxed); + { + std::unique_lock _(m_Mutex); + m_ErrorText.clear(); + } + } + + void Set(UpstreamEndpointState NewState, std::string ErrorText) + { + m_State.store(static_cast(NewState), std::memory_order_relaxed); + { + std::unique_lock _(m_Mutex); + m_ErrorText = std::move(ErrorText); + } + } + + void SetFromErrorCode(int32_t ErrorCode, std::string_view ErrorText) + { + if (ErrorCode != 0) + { + Set(ErrorCode == 401 ? UpstreamEndpointState::kUnauthorized : UpstreamEndpointState::kError, std::string(ErrorText)); + } + } + + private: + mutable std::mutex m_Mutex; + std::string m_ErrorText; + std::atomic_uint32_t m_State; + }; + + class JupiterUpstreamEndpoint final : public UpstreamEndpoint + { + public: + JupiterUpstreamEndpoint(const JupiterClientOptions& Options, const UpstreamAuthConfig& AuthConfig, AuthMgr& Mgr) + : m_AuthMgr(Mgr) + , m_Log(zen::logging::Get("upstream")) + { + ZEN_ASSERT(!Options.Name.empty()); + m_Info.Name = Options.Name; + m_Info.Url = Options.ServiceUrl; + + std::function TokenProvider; + + if (AuthConfig.OAuthUrl.empty() == false) + { + TokenProvider = httpclientauth::CreateFromOAuthClientCredentials( + {.Url = AuthConfig.OAuthUrl, .ClientId = AuthConfig.OAuthClientId, .ClientSecret = AuthConfig.OAuthClientSecret}); + } + else if (!AuthConfig.OpenIdProvider.empty()) + { + TokenProvider = httpclientauth::CreateFromOpenIdProvider(m_AuthMgr, AuthConfig.OpenIdProvider); + } + else if (!AuthConfig.AccessToken.empty()) + { + TokenProvider = httpclientauth::CreateFromStaticToken(AuthConfig.AccessToken); + } + else + { + TokenProvider = httpclientauth::CreateFromDefaultOpenIdProvider(m_AuthMgr); + } + + m_Client = new JupiterClient(Options, std::move(TokenProvider)); + } + + virtual ~JupiterUpstreamEndpoint() {} + + virtual const UpstreamEndpointInfo& GetEndpointInfo() const override { return m_Info; } + + virtual UpstreamEndpointStatus Initialize() override + { + ZEN_TRACE_CPU("Upstream::Jupiter::Initialize"); + + try + { + if (m_Status.EndpointState() == UpstreamEndpointState::kOk) + { + return {.State = UpstreamEndpointState::kOk}; + } + + JupiterSession Session(m_Client->Logger(), m_Client->Client(), m_AllowRedirect); + const JupiterResult Result = Session.Authenticate(); + + if (Result.Success) + { + m_Status.Set(UpstreamEndpointState::kOk); + } + else if (Result.ErrorCode != 0) + { + m_Status.SetFromErrorCode(Result.ErrorCode, Result.Reason); + } + else + { + m_Status.Set(UpstreamEndpointState::kUnauthorized); + } + + return m_Status.EndpointStatus(); + } + catch (const std::exception& Err) + { + m_Status.Set(UpstreamEndpointState::kError, Err.what()); + + return {.Reason = Err.what(), .State = GetState()}; + } + } + + std::string_view GetActualBlobStoreNamespace(std::string_view Namespace) + { + if (Namespace == ZenCacheStore::DefaultNamespace) + { + return m_Client->DefaultBlobStoreNamespace(); + } + return Namespace; + } + + virtual UpstreamEndpointState GetState() override { return m_Status.EndpointState(); } + + virtual UpstreamEndpointStatus GetStatus() override { return m_Status.EndpointStatus(); } + + virtual GetUpstreamCacheSingleResult GetCacheRecord(std::string_view Namespace, + const CacheKey& CacheKey, + ZenContentType Type) override + { + ZEN_TRACE_CPU("Upstream::Jupiter::GetSingleCacheRecord"); + + try + { + JupiterSession Session(m_Client->Logger(), m_Client->Client(), m_AllowRedirect); + JupiterResult Result; + + std::string_view BlobStoreNamespace = GetActualBlobStoreNamespace(Namespace); + + if (Type == ZenContentType::kCompressedBinary) + { + Result = Session.GetRef(BlobStoreNamespace, CacheKey.Bucket, CacheKey.Hash, ZenContentType::kCbObject); + + if (Result.Success) + { + const CbValidateError ValidationResult = ValidateCompactBinary(Result.Response, CbValidateMode::All); + if (Result.Success = ValidationResult == CbValidateError::None; Result.Success) + { + CbObject CacheRecord = LoadCompactBinaryObject(Result.Response); + IoBuffer ContentBuffer; + int NumAttachments = 0; + + CacheRecord.IterateAttachments([&](CbFieldView AttachmentHash) { + JupiterResult AttachmentResult = Session.GetCompressedBlob(BlobStoreNamespace, AttachmentHash.AsHash()); + Result.ReceivedBytes += AttachmentResult.ReceivedBytes; + Result.SentBytes += AttachmentResult.SentBytes; + Result.ElapsedSeconds += AttachmentResult.ElapsedSeconds; + Result.ErrorCode = AttachmentResult.ErrorCode; + + IoHash RawHash; + uint64_t RawSize; + if (CompressedBuffer::ValidateCompressedHeader(AttachmentResult.Response, RawHash, RawSize)) + { + Result.Response = AttachmentResult.Response; + ++NumAttachments; + } + else + { + Result.Success = false; + } + }); + if (NumAttachments != 1) + { + Result.Success = false; + } + } + } + } + else + { + const ZenContentType AcceptType = Type == ZenContentType::kCbPackage ? ZenContentType::kCbObject : Type; + Result = Session.GetRef(BlobStoreNamespace, CacheKey.Bucket, CacheKey.Hash, AcceptType); + + if (Result.Success && Type == ZenContentType::kCbPackage) + { + CbPackage Package; + + const CbValidateError ValidationResult = ValidateCompactBinary(Result.Response, CbValidateMode::All); + if (Result.Success = ValidationResult == CbValidateError::None; Result.Success) + { + CbObject CacheRecord = LoadCompactBinaryObject(Result.Response); + + CacheRecord.IterateAttachments([&](CbFieldView AttachmentHash) { + JupiterResult AttachmentResult = Session.GetCompressedBlob(BlobStoreNamespace, AttachmentHash.AsHash()); + Result.ReceivedBytes += AttachmentResult.ReceivedBytes; + Result.SentBytes += AttachmentResult.SentBytes; + Result.ElapsedSeconds += AttachmentResult.ElapsedSeconds; + Result.ErrorCode = AttachmentResult.ErrorCode; + + IoHash RawHash; + uint64_t RawSize; + if (CompressedBuffer Chunk = + CompressedBuffer::FromCompressed(SharedBuffer(AttachmentResult.Response), RawHash, RawSize)) + { + Package.AddAttachment(CbAttachment(Chunk, AttachmentHash.AsHash())); + } + else + { + Result.Success = false; + } + }); + + Package.SetObject(CacheRecord); + } + + if (Result.Success) + { + BinaryWriter MemStream; + Package.Save(MemStream); + + Result.Response = IoBuffer(IoBuffer::Clone, MemStream.Data(), MemStream.Size()); + } + } + } + + m_Status.SetFromErrorCode(Result.ErrorCode, Result.Reason); + + if (Result.ErrorCode == 0) + { + return {.Status = {.Bytes = gsl::narrow(Result.ReceivedBytes), + .ElapsedSeconds = Result.ElapsedSeconds, + .Success = Result.Success}, + .Value = Result.Response, + .Source = &m_Info}; + } + else + { + return {.Status = {.Error{.ErrorCode = Result.ErrorCode, .Reason = std::move(Result.Reason)}}}; + } + } + catch (const std::exception& Err) + { + m_Status.Set(UpstreamEndpointState::kError, Err.what()); + + return {.Status = {.Error{.ErrorCode = -1, .Reason = Err.what()}}}; + } + } + + virtual GetUpstreamCacheResult GetCacheRecords(std::string_view Namespace, + std::span Requests, + OnCacheRecordGetComplete&& OnComplete) override + { + ZEN_TRACE_CPU("Upstream::Jupiter::GetCacheRecords"); + + JupiterSession Session(m_Client->Logger(), m_Client->Client(), m_AllowRedirect); + GetUpstreamCacheResult Result; + + for (CacheKeyRequest* Request : Requests) + { + const CacheKey& CacheKey = Request->Key; + CbPackage Package; + CbObject Record; + + double ElapsedSeconds = 0.0; + if (!Result.Error) + { + std::string_view BlobStoreNamespace = GetActualBlobStoreNamespace(Namespace); + JupiterResult RefResult = Session.GetRef(BlobStoreNamespace, CacheKey.Bucket, CacheKey.Hash, ZenContentType::kCbObject); + AppendResult(RefResult, Result); + ElapsedSeconds = RefResult.ElapsedSeconds; + + m_Status.SetFromErrorCode(RefResult.ErrorCode, RefResult.Reason); + + if (RefResult.ErrorCode == 0) + { + const CbValidateError ValidationResult = ValidateCompactBinary(RefResult.Response, CbValidateMode::All); + if (ValidationResult == CbValidateError::None) + { + Record = LoadCompactBinaryObject(RefResult.Response); + Record.IterateAttachments([&](CbFieldView AttachmentHash) { + JupiterResult BlobResult = Session.GetCompressedBlob(BlobStoreNamespace, AttachmentHash.AsHash()); + AppendResult(BlobResult, Result); + + m_Status.SetFromErrorCode(BlobResult.ErrorCode, BlobResult.Reason); + + if (BlobResult.ErrorCode == 0) + { + IoHash RawHash; + uint64_t RawSize; + if (CompressedBuffer Chunk = + CompressedBuffer::FromCompressed(SharedBuffer(BlobResult.Response), RawHash, RawSize)) + { + if (RawHash == AttachmentHash.AsHash()) + { + Package.AddAttachment(CbAttachment(Chunk, RawHash)); + } + } + } + }); + } + } + } + + OnComplete( + {.Request = *Request, .Record = Record, .Package = Package, .ElapsedSeconds = ElapsedSeconds, .Source = &m_Info}); + } + + return Result; + } + + virtual GetUpstreamCacheSingleResult GetCacheChunk(std::string_view Namespace, + const CacheKey&, + const IoHash& ValueContentId) override + { + ZEN_TRACE_CPU("Upstream::Jupiter::GetSingleCacheChunk"); + + try + { + JupiterSession Session(m_Client->Logger(), m_Client->Client(), m_AllowRedirect); + std::string_view BlobStoreNamespace = GetActualBlobStoreNamespace(Namespace); + const JupiterResult Result = Session.GetCompressedBlob(BlobStoreNamespace, ValueContentId); + + m_Status.SetFromErrorCode(Result.ErrorCode, Result.Reason); + + if (Result.ErrorCode == 0) + { + return {.Status = {.Bytes = gsl::narrow(Result.ReceivedBytes), + .ElapsedSeconds = Result.ElapsedSeconds, + .Success = Result.Success}, + .Value = Result.Response, + .Source = &m_Info}; + } + else + { + return {.Status = {.Error{.ErrorCode = Result.ErrorCode, .Reason = std::move(Result.Reason)}}}; + } + } + catch (const std::exception& Err) + { + m_Status.Set(UpstreamEndpointState::kError, Err.what()); + + return {.Status = {.Error{.ErrorCode = -1, .Reason = Err.what()}}}; + } + } + + virtual GetUpstreamCacheResult GetCacheChunks(std::string_view Namespace, + std::span CacheChunkRequests, + OnCacheChunksGetComplete&& OnComplete) override final + { + ZEN_TRACE_CPU("Upstream::Jupiter::GetCacheChunks"); + + JupiterSession Session(m_Client->Logger(), m_Client->Client(), m_AllowRedirect); + GetUpstreamCacheResult Result; + + for (CacheChunkRequest* RequestPtr : CacheChunkRequests) + { + CacheChunkRequest& Request = *RequestPtr; + IoBuffer Payload; + IoHash RawHash = IoHash::Zero; + uint64_t RawSize = 0; + + double ElapsedSeconds = 0.0; + bool IsCompressed = false; + if (!Result.Error) + { + std::string_view BlobStoreNamespace = GetActualBlobStoreNamespace(Namespace); + const JupiterResult BlobResult = + Request.ChunkId == IoHash::Zero + ? Session.GetInlineBlob(BlobStoreNamespace, Request.Key.Bucket, Request.Key.Hash, Request.ChunkId) + : Session.GetCompressedBlob(BlobStoreNamespace, Request.ChunkId); + ElapsedSeconds = BlobResult.ElapsedSeconds; + Payload = BlobResult.Response; + + AppendResult(BlobResult, Result); + + m_Status.SetFromErrorCode(BlobResult.ErrorCode, BlobResult.Reason); + if (Payload && IsCompressedBinary(Payload.GetContentType())) + { + IsCompressed = CompressedBuffer::ValidateCompressedHeader(Payload, RawHash, RawSize); + } + } + + if (IsCompressed) + { + OnComplete({.Request = Request, + .RawHash = RawHash, + .RawSize = RawSize, + .Value = Payload, + .ElapsedSeconds = ElapsedSeconds, + .Source = &m_Info}); + } + else + { + OnComplete({.Request = Request, .RawHash = IoHash::Zero, .RawSize = 0, .Value = IoBuffer()}); + } + } + + return Result; + } + + virtual GetUpstreamCacheResult GetCacheValues(std::string_view Namespace, + std::span CacheValueRequests, + OnCacheValueGetComplete&& OnComplete) override final + { + ZEN_TRACE_CPU("Upstream::Jupiter::GetCacheValues"); + + JupiterSession Session(m_Client->Logger(), m_Client->Client(), m_AllowRedirect); + GetUpstreamCacheResult Result; + + for (CacheValueRequest* RequestPtr : CacheValueRequests) + { + CacheValueRequest& Request = *RequestPtr; + IoBuffer Payload; + IoHash RawHash = IoHash::Zero; + uint64_t RawSize = 0; + + double ElapsedSeconds = 0.0; + bool IsCompressed = false; + if (!Result.Error) + { + std::string_view BlobStoreNamespace = GetActualBlobStoreNamespace(Namespace); + IoHash PayloadHash; + const JupiterResult BlobResult = + Session.GetInlineBlob(BlobStoreNamespace, Request.Key.Bucket, Request.Key.Hash, PayloadHash); + ElapsedSeconds = BlobResult.ElapsedSeconds; + Payload = BlobResult.Response; + + AppendResult(BlobResult, Result); + + m_Status.SetFromErrorCode(BlobResult.ErrorCode, BlobResult.Reason); + if (Payload) + { + if (IsCompressedBinary(Payload.GetContentType())) + { + IsCompressed = CompressedBuffer::ValidateCompressedHeader(Payload, RawHash, RawSize) && RawHash != PayloadHash; + } + else + { + CompressedBuffer Compressed = CompressedBuffer::Compress(SharedBuffer(Payload)); + RawHash = Compressed.DecodeRawHash(); + if (RawHash == PayloadHash) + { + IsCompressed = true; + } + else + { + ZEN_WARN("Jupiter request for inline payload of {}/{}/{} has hash {}, expected hash {} from header", + Namespace, + Request.Key.Bucket, + Request.Key.Hash.ToHexString(), + RawHash.ToHexString(), + PayloadHash.ToHexString()); + } + } + } + } + + if (IsCompressed) + { + OnComplete({.Request = Request, + .RawHash = RawHash, + .RawSize = RawSize, + .Value = Payload, + .ElapsedSeconds = ElapsedSeconds, + .Source = &m_Info}); + } + else + { + OnComplete({.Request = Request, .RawHash = IoHash::Zero, .RawSize = 0, .Value = IoBuffer()}); + } + } + + return Result; + } + + virtual PutUpstreamCacheResult PutCacheRecord(const UpstreamCacheRecord& CacheRecord, + IoBuffer RecordValue, + std::span Values) override + { + ZEN_TRACE_CPU("Upstream::Jupiter::PutCacheRecord"); + + ZEN_ASSERT(CacheRecord.ValueContentIds.size() == Values.size()); + const int32_t MaxAttempts = 3; + + try + { + JupiterSession Session(m_Client->Logger(), m_Client->Client(), m_AllowRedirect); + + if (CacheRecord.Type == ZenContentType::kBinary) + { + JupiterResult Result; + for (uint32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) + { + std::string_view BlobStoreNamespace = GetActualBlobStoreNamespace(CacheRecord.Namespace); + Result = Session.PutRef(BlobStoreNamespace, + CacheRecord.Key.Bucket, + CacheRecord.Key.Hash, + RecordValue, + ZenContentType::kBinary); + } + + m_Status.SetFromErrorCode(Result.ErrorCode, Result.Reason); + + return {.Reason = std::move(Result.Reason), + .Bytes = gsl::narrow(Result.ReceivedBytes), + .ElapsedSeconds = Result.ElapsedSeconds, + .Success = Result.Success}; + } + else if (CacheRecord.Type == ZenContentType::kCompressedBinary) + { + IoHash RawHash; + uint64_t RawSize; + if (!CompressedBuffer::ValidateCompressedHeader(RecordValue, RawHash, RawSize)) + { + return {.Reason = std::string("Invalid compressed value buffer"), .Success = false}; + } + + CbObjectWriter ReferencingObject; + ReferencingObject.AddBinaryAttachment("RawHash", RawHash); + ReferencingObject.AddInteger("RawSize", RawSize); + + return PerformStructuredPut( + Session, + CacheRecord.Namespace, + CacheRecord.Key, + ReferencingObject.Save().GetBuffer().AsIoBuffer(), + MaxAttempts, + [&](const IoHash& ValueContentId, IoBuffer& OutBuffer, std::string& OutReason) { + if (ValueContentId != RawHash) + { + OutReason = + fmt::format("Value '{}' MISMATCHED from compressed buffer raw hash {}", ValueContentId, RawHash); + return false; + } + + OutBuffer = RecordValue; + return true; + }); + } + else + { + return PerformStructuredPut( + Session, + CacheRecord.Namespace, + CacheRecord.Key, + RecordValue, + MaxAttempts, + [&](const IoHash& ValueContentId, IoBuffer& OutBuffer, std::string& OutReason) { + const auto It = + std::find(std::begin(CacheRecord.ValueContentIds), std::end(CacheRecord.ValueContentIds), ValueContentId); + + if (It == std::end(CacheRecord.ValueContentIds)) + { + OutReason = fmt::format("value '{}' MISSING from local cache", ValueContentId); + return false; + } + + const size_t Idx = std::distance(std::begin(CacheRecord.ValueContentIds), It); + + OutBuffer = Values[Idx]; + return true; + }); + } + } + catch (const std::exception& Err) + { + m_Status.Set(UpstreamEndpointState::kError, Err.what()); + + return {.Reason = std::string(Err.what()), .Success = false}; + } + } + + virtual UpstreamEndpointStats& Stats() override { return m_Stats; } + + private: + static void AppendResult(const JupiterResult& Result, GetUpstreamCacheResult& Out) + { + Out.Success &= Result.Success; + Out.Bytes += gsl::narrow(Result.ReceivedBytes); + Out.ElapsedSeconds += Result.ElapsedSeconds; + + if (Result.ErrorCode) + { + Out.Error = {.ErrorCode = Result.ErrorCode, .Reason = std::move(Result.Reason)}; + } + }; + + PutUpstreamCacheResult PerformStructuredPut( + JupiterSession& Session, + std::string_view Namespace, + const CacheKey& Key, + IoBuffer ObjectBuffer, + const int32_t MaxAttempts, + std::function&& BlobFetchFn) + { + int64_t TotalBytes = 0ull; + double TotalElapsedSeconds = 0.0; + + std::string_view BlobStoreNamespace = GetActualBlobStoreNamespace(Namespace); + const auto PutBlobs = [&](std::span ValueContentIds, std::string& OutReason) -> bool { + for (const IoHash& ValueContentId : ValueContentIds) + { + IoBuffer BlobBuffer; + if (!BlobFetchFn(ValueContentId, BlobBuffer, OutReason)) + { + return false; + } + + JupiterResult BlobResult; + for (int32_t Attempt = 0; Attempt < MaxAttempts && !BlobResult.Success; Attempt++) + { + BlobResult = Session.PutCompressedBlob(BlobStoreNamespace, ValueContentId, BlobBuffer); + } + + m_Status.SetFromErrorCode(BlobResult.ErrorCode, BlobResult.Reason); + + if (!BlobResult.Success) + { + OutReason = fmt::format("upload value '{}' FAILED, reason '{}'", ValueContentId, BlobResult.Reason); + return false; + } + + TotalBytes += gsl::narrow(BlobResult.ReceivedBytes); + TotalElapsedSeconds += BlobResult.ElapsedSeconds; + } + + return true; + }; + + PutRefResult RefResult; + for (int32_t Attempt = 0; Attempt < MaxAttempts && !RefResult.Success; Attempt++) + { + RefResult = Session.PutRef(BlobStoreNamespace, Key.Bucket, Key.Hash, ObjectBuffer, ZenContentType::kCbObject); + } + + m_Status.SetFromErrorCode(RefResult.ErrorCode, RefResult.Reason); + + if (!RefResult.Success) + { + return {.Reason = fmt::format("upload cache record '{}/{}' FAILED, reason '{}'", Key.Bucket, Key.Hash, RefResult.Reason), + .Success = false}; + } + + TotalBytes += gsl::narrow(RefResult.ReceivedBytes); + TotalElapsedSeconds += RefResult.ElapsedSeconds; + + std::string Reason; + if (!PutBlobs(RefResult.Needs, Reason)) + { + return {.Reason = std::move(Reason), .Success = false}; + } + + const IoHash RefHash = IoHash::HashBuffer(ObjectBuffer); + FinalizeRefResult FinalizeResult = Session.FinalizeRef(BlobStoreNamespace, Key.Bucket, Key.Hash, RefHash); + + m_Status.SetFromErrorCode(FinalizeResult.ErrorCode, FinalizeResult.Reason); + + if (!FinalizeResult.Success) + { + return { + .Reason = fmt::format("finalize cache record '{}/{}' FAILED, reason '{}'", Key.Bucket, Key.Hash, FinalizeResult.Reason), + .Success = false}; + } + + if (!FinalizeResult.Needs.empty()) + { + if (!PutBlobs(FinalizeResult.Needs, Reason)) + { + return {.Reason = std::move(Reason), .Success = false}; + } + + FinalizeResult = Session.FinalizeRef(BlobStoreNamespace, Key.Bucket, Key.Hash, RefHash); + + m_Status.SetFromErrorCode(FinalizeResult.ErrorCode, FinalizeResult.Reason); + + if (!FinalizeResult.Success) + { + return {.Reason = fmt::format("finalize '{}/{}' FAILED, reason '{}'", Key.Bucket, Key.Hash, FinalizeResult.Reason), + .Success = false}; + } + + if (!FinalizeResult.Needs.empty()) + { + ExtendableStringBuilder<256> Sb; + for (const IoHash& MissingHash : FinalizeResult.Needs) + { + Sb << MissingHash.ToHexString() << ","; + } + + return { + .Reason = fmt::format("finalize '{}/{}' FAILED, still needs value(s) '{}'", Key.Bucket, Key.Hash, Sb.ToString()), + .Success = false}; + } + } + + TotalBytes += gsl::narrow(FinalizeResult.ReceivedBytes); + TotalElapsedSeconds += FinalizeResult.ElapsedSeconds; + + return {.Bytes = TotalBytes, .ElapsedSeconds = TotalElapsedSeconds, .Success = true}; + } + + LoggerRef Log() { return m_Log; } + + AuthMgr& m_AuthMgr; + LoggerRef m_Log; + UpstreamEndpointInfo m_Info; + UpstreamStatus m_Status; + UpstreamEndpointStats m_Stats; + RefPtr m_Client; + const bool m_AllowRedirect = false; + }; + + class ZenUpstreamEndpoint final : public UpstreamEndpoint + { + struct ZenEndpoint + { + std::string Url; + std::string Reason; + double Latency{}; + bool Ok = false; + + bool operator<(const ZenEndpoint& RHS) const { return Ok && RHS.Ok ? Latency < RHS.Latency : Ok; } + }; + + public: + ZenUpstreamEndpoint(const ZenStructuredCacheClientOptions& Options) + : m_Log(zen::logging::Get("upstream")) + , m_ConnectTimeout(Options.ConnectTimeout) + , m_Timeout(Options.Timeout) + { + ZEN_ASSERT(!Options.Name.empty()); + m_Info.Name = Options.Name; + + for (const auto& Url : Options.Urls) + { + m_Endpoints.push_back({.Url = Url}); + } + } + + ~ZenUpstreamEndpoint() {} + + virtual const UpstreamEndpointInfo& GetEndpointInfo() const override { return m_Info; } + + virtual UpstreamEndpointStatus Initialize() override + { + ZEN_TRACE_CPU("Upstream::Zen::Initialize"); + + try + { + if (m_Status.EndpointState() == UpstreamEndpointState::kOk) + { + return {.State = UpstreamEndpointState::kOk}; + } + + const ZenEndpoint& Ep = GetEndpoint(); + + if (m_Info.Url != Ep.Url) + { + ZEN_INFO("Setting Zen upstream URL to '{}'", Ep.Url); + m_Info.Url = Ep.Url; + } + + if (Ep.Ok) + { + RwLock::ExclusiveLockScope _(m_ClientLock); + m_Client = new ZenStructuredCacheClient({.Url = m_Info.Url, .ConnectTimeout = m_ConnectTimeout, .Timeout = m_Timeout}); + m_Status.Set(UpstreamEndpointState::kOk); + } + else + { + m_Status.Set(UpstreamEndpointState::kError, Ep.Reason); + } + + return m_Status.EndpointStatus(); + } + catch (const std::exception& Err) + { + m_Status.Set(UpstreamEndpointState::kError, Err.what()); + + return {.Reason = Err.what(), .State = GetState()}; + } + } + + virtual UpstreamEndpointState GetState() override { return m_Status.EndpointState(); } + + virtual UpstreamEndpointStatus GetStatus() override { return m_Status.EndpointStatus(); } + + virtual GetUpstreamCacheSingleResult GetCacheRecord(std::string_view Namespace, + const CacheKey& CacheKey, + ZenContentType Type) override + { + ZEN_TRACE_CPU("Upstream::Zen::GetSingleCacheRecord"); + + try + { + ZenStructuredCacheSession Session(GetClientRef()); + const ZenCacheResult Result = Session.GetCacheRecord(Namespace, CacheKey.Bucket, CacheKey.Hash, Type); + + m_Status.SetFromErrorCode(Result.ErrorCode, Result.Reason); + + if (Result.ErrorCode == 0) + { + return {.Status = {.Bytes = Result.Bytes, .ElapsedSeconds = Result.ElapsedSeconds, .Success = Result.Success}, + .Value = Result.Response, + .Source = &m_Info}; + } + else + { + return {.Status = {.Error{.ErrorCode = Result.ErrorCode, .Reason = std::move(Result.Reason)}}}; + } + } + catch (const std::exception& Err) + { + m_Status.Set(UpstreamEndpointState::kError, Err.what()); + + return {.Status = {.Error{.ErrorCode = -1, .Reason = Err.what()}}}; + } + } + + virtual GetUpstreamCacheResult GetCacheRecords(std::string_view Namespace, + std::span Requests, + OnCacheRecordGetComplete&& OnComplete) override + { + ZEN_TRACE_CPU("Upstream::Zen::GetCacheRecords"); + ZEN_ASSERT(Requests.size() > 0); + + CbObjectWriter BatchRequest; + BatchRequest << "Method"sv + << "GetCacheRecords"sv; + BatchRequest << "Accept"sv << kCbPkgMagic; + + BatchRequest.BeginObject("Params"sv); + { + CachePolicy DefaultPolicy = Requests[0]->Policy.GetRecordPolicy(); + BatchRequest << "DefaultPolicy"sv << WriteToString<128>(DefaultPolicy); + + BatchRequest << "Namespace"sv << Namespace; + + BatchRequest.BeginArray("Requests"sv); + for (CacheKeyRequest* Request : Requests) + { + BatchRequest.BeginObject(); + { + const CacheKey& Key = Request->Key; + BatchRequest.BeginObject("Key"sv); + { + BatchRequest << "Bucket"sv << Key.Bucket; + BatchRequest << "Hash"sv << Key.Hash; + } + BatchRequest.EndObject(); + if (!Request->Policy.IsUniform() || Request->Policy.GetRecordPolicy() != DefaultPolicy) + { + BatchRequest.SetName("Policy"sv); + Request->Policy.Save(BatchRequest); + } + } + BatchRequest.EndObject(); + } + BatchRequest.EndArray(); + } + BatchRequest.EndObject(); + + ZenCacheResult Result; + + { + ZenStructuredCacheSession Session(GetClientRef()); + Result = Session.InvokeRpc(BatchRequest.Save()); + } + + m_Status.SetFromErrorCode(Result.ErrorCode, Result.Reason); + + if (Result.Success) + { + CbPackage BatchResponse; + if (ParsePackageMessageWithLegacyFallback(Result.Response, BatchResponse)) + { + CbArrayView Results = BatchResponse.GetObject()["Result"sv].AsArrayView(); + if (Results.Num() != Requests.size()) + { + ZEN_WARN("Upstream::Zen::GetCacheRecords invalid number of Response results from Upstream."); + } + else + { + for (size_t Index = 0; CbFieldView Record : Results) + { + CacheKeyRequest* Request = Requests[Index++]; + OnComplete({.Request = *Request, + .Record = Record.AsObjectView(), + .Package = BatchResponse, + .ElapsedSeconds = Result.ElapsedSeconds, + .Source = &m_Info}); + } + + return {.Bytes = Result.Bytes, .ElapsedSeconds = Result.ElapsedSeconds, .Success = true}; + } + } + else + { + ZEN_WARN("Upstream::Zen::GetCacheRecords invalid Response from Upstream."); + } + } + + for (CacheKeyRequest* Request : Requests) + { + OnComplete({.Request = *Request, .Record = CbObjectView(), .Package = CbPackage()}); + } + + return {.Error{.ErrorCode = Result.ErrorCode, .Reason = std::move(Result.Reason)}}; + } + + virtual GetUpstreamCacheSingleResult GetCacheChunk(std::string_view Namespace, + const CacheKey& CacheKey, + const IoHash& ValueContentId) override + { + ZEN_TRACE_CPU("Upstream::Zen::GetCacheChunk"); + + try + { + ZenStructuredCacheSession Session(GetClientRef()); + const ZenCacheResult Result = Session.GetCacheChunk(Namespace, CacheKey.Bucket, CacheKey.Hash, ValueContentId); + + m_Status.SetFromErrorCode(Result.ErrorCode, Result.Reason); + + if (Result.ErrorCode == 0) + { + return {.Status = {.Bytes = Result.Bytes, .ElapsedSeconds = Result.ElapsedSeconds, .Success = Result.Success}, + .Value = Result.Response, + .Source = &m_Info}; + } + else + { + return {.Status = {.Error{.ErrorCode = Result.ErrorCode, .Reason = std::move(Result.Reason)}}}; + } + } + catch (const std::exception& Err) + { + m_Status.Set(UpstreamEndpointState::kError, Err.what()); + + return {.Status = {.Error{.ErrorCode = -1, .Reason = Err.what()}}}; + } + } + + virtual GetUpstreamCacheResult GetCacheValues(std::string_view Namespace, + std::span CacheValueRequests, + OnCacheValueGetComplete&& OnComplete) override final + { + ZEN_TRACE_CPU("Upstream::Zen::GetCacheValues"); + ZEN_ASSERT(!CacheValueRequests.empty()); + + CbObjectWriter BatchRequest; + BatchRequest << "Method"sv + << "GetCacheValues"sv; + BatchRequest << "Accept"sv << kCbPkgMagic; + + BatchRequest.BeginObject("Params"sv); + { + CachePolicy DefaultPolicy = CacheValueRequests[0]->Policy; + BatchRequest << "DefaultPolicy"sv << WriteToString<128>(DefaultPolicy).ToView(); + BatchRequest << "Namespace"sv << Namespace; + + BatchRequest.BeginArray("Requests"sv); + { + for (CacheValueRequest* RequestPtr : CacheValueRequests) + { + const CacheValueRequest& Request = *RequestPtr; + + BatchRequest.BeginObject(); + { + BatchRequest.BeginObject("Key"sv); + BatchRequest << "Bucket"sv << Request.Key.Bucket; + BatchRequest << "Hash"sv << Request.Key.Hash; + BatchRequest.EndObject(); + if (Request.Policy != DefaultPolicy) + { + BatchRequest << "Policy"sv << WriteToString<128>(Request.Policy).ToView(); + } + } + BatchRequest.EndObject(); + } + } + BatchRequest.EndArray(); + } + BatchRequest.EndObject(); + + ZenCacheResult Result; + + { + ZenStructuredCacheSession Session(GetClientRef()); + Result = Session.InvokeRpc(BatchRequest.Save()); + } + + m_Status.SetFromErrorCode(Result.ErrorCode, Result.Reason); + + if (Result.Success) + { + CbPackage BatchResponse; + if (ParsePackageMessageWithLegacyFallback(Result.Response, BatchResponse)) + { + CbArrayView Results = BatchResponse.GetObject()["Result"sv].AsArrayView(); + if (CacheValueRequests.size() != Results.Num()) + { + ZEN_WARN("Upstream::Zen::GetCacheValues invalid number of Response results from Upstream."); + } + else + { + for (size_t RequestIndex = 0; CbFieldView ChunkField : Results) + { + CacheValueRequest& Request = *CacheValueRequests[RequestIndex++]; + CbObjectView ChunkObject = ChunkField.AsObjectView(); + IoHash RawHash = ChunkObject["RawHash"sv].AsHash(); + IoBuffer Payload; + uint64_t RawSize = 0; + if (RawHash != IoHash::Zero) + { + bool Success = false; + const CbAttachment* Attachment = BatchResponse.FindAttachment(RawHash); + if (Attachment) + { + if (const CompressedBuffer& Compressed = Attachment->AsCompressedBinary()) + { + Payload = Compressed.GetCompressed().Flatten().AsIoBuffer(); + Payload.SetContentType(ZenContentType::kCompressedBinary); + RawSize = Compressed.DecodeRawSize(); + Success = true; + } + } + if (!Success) + { + CbFieldView RawSizeField = ChunkObject["RawSize"sv]; + RawSize = RawSizeField.AsUInt64(); + Success = !RawSizeField.HasError(); + } + if (!Success) + { + RawHash = IoHash::Zero; + } + } + OnComplete({.Request = Request, + .RawHash = RawHash, + .RawSize = RawSize, + .Value = std::move(Payload), + .ElapsedSeconds = Result.ElapsedSeconds, + .Source = &m_Info}); + } + + return {.Bytes = Result.Bytes, .ElapsedSeconds = Result.ElapsedSeconds, .Success = true}; + } + } + else + { + ZEN_WARN("Upstream::Zen::GetCacheValues invalid Response from Upstream."); + } + } + + for (CacheValueRequest* RequestPtr : CacheValueRequests) + { + OnComplete({.Request = *RequestPtr, .RawHash = IoHash::Zero, .RawSize = 0, .Value = IoBuffer()}); + } + + return {.Error{.ErrorCode = Result.ErrorCode, .Reason = std::move(Result.Reason)}}; + } + + virtual GetUpstreamCacheResult GetCacheChunks(std::string_view Namespace, + std::span CacheChunkRequests, + OnCacheChunksGetComplete&& OnComplete) override final + { + ZEN_TRACE_CPU("Upstream::Zen::GetCacheChunks"); + ZEN_ASSERT(!CacheChunkRequests.empty()); + + CbObjectWriter BatchRequest; + BatchRequest << "Method"sv + << "GetCacheChunks"sv; + BatchRequest << "Accept"sv << kCbPkgMagic; + + BatchRequest.BeginObject("Params"sv); + { + CachePolicy DefaultPolicy = CacheChunkRequests[0]->Policy; + BatchRequest << "DefaultPolicy"sv << WriteToString<128>(DefaultPolicy).ToView(); + BatchRequest << "Namespace"sv << Namespace; + + BatchRequest.BeginArray("ChunkRequests"sv); + { + for (CacheChunkRequest* RequestPtr : CacheChunkRequests) + { + const CacheChunkRequest& Request = *RequestPtr; + + BatchRequest.BeginObject(); + { + BatchRequest.BeginObject("Key"sv); + BatchRequest << "Bucket"sv << Request.Key.Bucket; + BatchRequest << "Hash"sv << Request.Key.Hash; + BatchRequest.EndObject(); + if (Request.ValueId) + { + BatchRequest.AddObjectId("ValueId"sv, Request.ValueId); + } + if (Request.ChunkId != Request.ChunkId.Zero) + { + BatchRequest << "ChunkId"sv << Request.ChunkId; + } + if (Request.RawOffset != 0) + { + BatchRequest << "RawOffset"sv << Request.RawOffset; + } + if (Request.RawSize != UINT64_MAX) + { + BatchRequest << "RawSize"sv << Request.RawSize; + } + if (Request.Policy != DefaultPolicy) + { + BatchRequest << "Policy"sv << WriteToString<128>(Request.Policy).ToView(); + } + } + BatchRequest.EndObject(); + } + } + BatchRequest.EndArray(); + } + BatchRequest.EndObject(); + + ZenCacheResult Result; + + { + ZenStructuredCacheSession Session(GetClientRef()); + Result = Session.InvokeRpc(BatchRequest.Save()); + } + + m_Status.SetFromErrorCode(Result.ErrorCode, Result.Reason); + + if (Result.Success) + { + CbPackage BatchResponse; + if (ParsePackageMessageWithLegacyFallback(Result.Response, BatchResponse)) + { + CbArrayView Results = BatchResponse.GetObject()["Result"sv].AsArrayView(); + if (CacheChunkRequests.size() != Results.Num()) + { + ZEN_WARN("Upstream::Zen::GetCacheChunks invalid number of Response results from Upstream."); + } + else + { + for (size_t RequestIndex = 0; CbFieldView ChunkField : Results) + { + CacheChunkRequest& Request = *CacheChunkRequests[RequestIndex++]; + CbObjectView ChunkObject = ChunkField.AsObjectView(); + IoHash RawHash = ChunkObject["RawHash"sv].AsHash(); + IoBuffer Payload; + uint64_t RawSize = 0; + if (RawHash != IoHash::Zero) + { + bool Success = false; + const CbAttachment* Attachment = BatchResponse.FindAttachment(RawHash); + if (Attachment) + { + if (const CompressedBuffer& Compressed = Attachment->AsCompressedBinary()) + { + Payload = Compressed.GetCompressed().Flatten().AsIoBuffer(); + Payload.SetContentType(ZenContentType::kCompressedBinary); + RawSize = Compressed.DecodeRawSize(); + Success = true; + } + } + if (!Success) + { + CbFieldView RawSizeField = ChunkObject["RawSize"sv]; + RawSize = RawSizeField.AsUInt64(); + Success = !RawSizeField.HasError(); + } + if (!Success) + { + RawHash = IoHash::Zero; + } + } + OnComplete({.Request = Request, + .RawHash = RawHash, + .RawSize = RawSize, + .Value = std::move(Payload), + .ElapsedSeconds = Result.ElapsedSeconds, + .Source = &m_Info}); + } + + return {.Bytes = Result.Bytes, .ElapsedSeconds = Result.ElapsedSeconds, .Success = true}; + } + } + else + { + ZEN_WARN("Upstream::Zen::GetCacheChunks invalid Response from Upstream."); + } + } + + for (CacheChunkRequest* RequestPtr : CacheChunkRequests) + { + OnComplete({.Request = *RequestPtr, .RawHash = IoHash::Zero, .RawSize = 0, .Value = IoBuffer()}); + } + + return {.Error{.ErrorCode = Result.ErrorCode, .Reason = std::move(Result.Reason)}}; + } + + virtual PutUpstreamCacheResult PutCacheRecord(const UpstreamCacheRecord& CacheRecord, + IoBuffer RecordValue, + std::span Values) override + { + ZEN_TRACE_CPU("Upstream::Zen::PutCacheRecord"); + + ZEN_ASSERT(CacheRecord.ValueContentIds.size() == Values.size()); + const int32_t MaxAttempts = 3; + + try + { + ZenStructuredCacheSession Session(GetClientRef()); + ZenCacheResult Result; + int64_t TotalBytes = 0ull; + double TotalElapsedSeconds = 0.0; + + if (CacheRecord.Type == ZenContentType::kCbPackage) + { + CbPackage Package; + Package.SetObject(CbObject(SharedBuffer(RecordValue))); + + for (const IoBuffer& Value : Values) + { + IoHash RawHash; + uint64_t RawSize; + if (CompressedBuffer AttachmentBuffer = CompressedBuffer::FromCompressed(SharedBuffer(Value), RawHash, RawSize)) + { + Package.AddAttachment(CbAttachment(AttachmentBuffer, RawHash)); + } + else + { + return {.Reason = std::string("Invalid value buffer"), .Success = false}; + } + } + + BinaryWriter MemStream; + Package.Save(MemStream); + IoBuffer PackagePayload(IoBuffer::Wrap, MemStream.Data(), MemStream.Size()); + + for (uint32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) + { + Result = Session.PutCacheRecord(CacheRecord.Namespace, + CacheRecord.Key.Bucket, + CacheRecord.Key.Hash, + PackagePayload, + CacheRecord.Type); + } + + m_Status.SetFromErrorCode(Result.ErrorCode, Result.Reason); + + TotalBytes = Result.Bytes; + TotalElapsedSeconds = Result.ElapsedSeconds; + } + else if (CacheRecord.Type == ZenContentType::kCompressedBinary) + { + IoHash RawHash; + uint64_t RawSize; + CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(RecordValue), RawHash, RawSize); + if (!Compressed) + { + return {.Reason = std::string("Invalid value compressed buffer"), .Success = false}; + } + + CbPackage BatchPackage; + CbObjectWriter BatchWriter; + BatchWriter << "Method"sv + << "PutCacheValues"sv; + BatchWriter << "Accept"sv << kCbPkgMagic; + + BatchWriter.BeginObject("Params"sv); + { + // DefaultPolicy unspecified and expected to be Default + + BatchWriter << "Namespace"sv << CacheRecord.Namespace; + + BatchWriter.BeginArray("Requests"sv); + { + BatchWriter.BeginObject(); + { + const CacheKey& Key = CacheRecord.Key; + BatchWriter.BeginObject("Key"sv); + { + BatchWriter << "Bucket"sv << Key.Bucket; + BatchWriter << "Hash"sv << Key.Hash; + } + BatchWriter.EndObject(); + // Policy unspecified and expected to be Default + BatchWriter.AddBinaryAttachment("RawHash"sv, RawHash); + BatchPackage.AddAttachment(CbAttachment(Compressed, RawHash)); + } + BatchWriter.EndObject(); + } + BatchWriter.EndArray(); + } + BatchWriter.EndObject(); + BatchPackage.SetObject(BatchWriter.Save()); + + Result.Success = false; + for (uint32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) + { + Result = Session.InvokeRpc(BatchPackage); + } + + m_Status.SetFromErrorCode(Result.ErrorCode, Result.Reason); + + TotalBytes += Result.Bytes; + TotalElapsedSeconds += Result.ElapsedSeconds; + } + else + { + for (size_t Idx = 0, Count = Values.size(); Idx < Count; Idx++) + { + Result.Success = false; + for (uint32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) + { + Result = Session.PutCacheValue(CacheRecord.Namespace, + CacheRecord.Key.Bucket, + CacheRecord.Key.Hash, + CacheRecord.ValueContentIds[Idx], + Values[Idx]); + } + + m_Status.SetFromErrorCode(Result.ErrorCode, Result.Reason); + + TotalBytes += Result.Bytes; + TotalElapsedSeconds += Result.ElapsedSeconds; + + if (!Result.Success) + { + return {.Reason = "Failed to upload value", + .Bytes = TotalBytes, + .ElapsedSeconds = TotalElapsedSeconds, + .Success = false}; + } + } + + Result.Success = false; + for (uint32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) + { + Result = Session.PutCacheRecord(CacheRecord.Namespace, + CacheRecord.Key.Bucket, + CacheRecord.Key.Hash, + RecordValue, + CacheRecord.Type); + } + + m_Status.SetFromErrorCode(Result.ErrorCode, Result.Reason); + + TotalBytes += Result.Bytes; + TotalElapsedSeconds += Result.ElapsedSeconds; + } + + return {.Reason = std::move(Result.Reason), + .Bytes = TotalBytes, + .ElapsedSeconds = TotalElapsedSeconds, + .Success = Result.Success}; + } + catch (const std::exception& Err) + { + m_Status.Set(UpstreamEndpointState::kError, Err.what()); + + return {.Reason = std::string(Err.what()), .Success = false}; + } + } + + virtual UpstreamEndpointStats& Stats() override { return m_Stats; } + + private: + Ref GetClientRef() + { + // m_Client can be modified at any time by a different thread. + // Make sure we safely bump the refcount inside a scope lock + RwLock::SharedLockScope _(m_ClientLock); + ZEN_ASSERT(m_Client); + Ref ClientRef(m_Client); + _.ReleaseNow(); + return ClientRef; + } + + const ZenEndpoint& GetEndpoint() + { + for (ZenEndpoint& Ep : m_Endpoints) + { + Ref Client( + new ZenStructuredCacheClient({.Url = Ep.Url, .ConnectTimeout = std::chrono::milliseconds(1000)})); + ZenStructuredCacheSession Session(std::move(Client)); + const int32_t SampleCount = 2; + + Ep.Ok = false; + Ep.Latency = {}; + + for (int32_t Sample = 0; Sample < SampleCount; ++Sample) + { + ZenCacheResult Result = Session.CheckHealth(); + Ep.Ok = Result.Success; + Ep.Reason = std::move(Result.Reason); + Ep.Latency += Result.ElapsedSeconds; + } + Ep.Latency /= double(SampleCount); + } + + std::sort(std::begin(m_Endpoints), std::end(m_Endpoints)); + + for (const auto& Ep : m_Endpoints) + { + ZEN_INFO("ping 'Zen' endpoint '{}' latency '{:.3}s' {}", Ep.Url, Ep.Latency, Ep.Ok ? "OK" : Ep.Reason); + } + + return m_Endpoints.front(); + } + + LoggerRef Log() { return m_Log; } + + LoggerRef m_Log; + UpstreamEndpointInfo m_Info; + UpstreamStatus m_Status; + UpstreamEndpointStats m_Stats; + std::vector m_Endpoints; + std::chrono::milliseconds m_ConnectTimeout; + std::chrono::milliseconds m_Timeout; + RwLock m_ClientLock; + RefPtr m_Client; + }; + +} // namespace detail + +////////////////////////////////////////////////////////////////////////// + +class UpstreamCacheImpl final : public UpstreamCache +{ +public: + UpstreamCacheImpl(const UpstreamCacheOptions& Options, ZenCacheStore& CacheStore, CidStore& CidStore) + : m_Log(logging::Get("upstream")) + , m_Options(Options) + , m_CacheStore(CacheStore) + , m_CidStore(CidStore) + { + } + + virtual ~UpstreamCacheImpl() { Shutdown(); } + + virtual void Initialize() override + { + ZEN_TRACE_CPU("Upstream::Initialize"); + + m_RunState.IsRunning = true; + } + + virtual bool IsActive() override + { + std::shared_lock _(m_EndpointsMutex); + return !m_Endpoints.empty(); + } + + virtual void RegisterEndpoint(std::unique_ptr Endpoint) override + { + ZEN_TRACE_CPU("Upstream::RegisterEndpoint"); + + const UpstreamEndpointStatus Status = Endpoint->Initialize(); + const UpstreamEndpointInfo& Info = Endpoint->GetEndpointInfo(); + + if (Status.State == UpstreamEndpointState::kOk) + { + ZEN_INFO("register endpoint '{} - {}' {}", Info.Name, Info.Url, ToString(Status.State)); + } + else + { + ZEN_WARN("register endpoint '{} - {}' {}", Info.Name, Info.Url, ToString(Status.State)); + } + + // Register endpoint even if it fails, the health monitor thread will probe failing endpoint(s) + std::unique_lock _(m_EndpointsMutex); + if (m_Endpoints.empty()) + { + for (uint32_t Idx = 0; Idx < m_Options.ThreadCount; Idx++) + { + m_UpstreamThreads.emplace_back(&UpstreamCacheImpl::ProcessUpstreamQueue, this, Idx + 1); + } + + m_EndpointMonitorThread = std::thread(&UpstreamCacheImpl::MonitorEndpoints, this); + } + m_Endpoints.emplace_back(std::move(Endpoint)); + } + + virtual void IterateEndpoints(std::function&& Fn) override + { + std::shared_lock _(m_EndpointsMutex); + + for (auto& Ep : m_Endpoints) + { + if (!Fn(*Ep)) + { + break; + } + } + } + + virtual GetUpstreamCacheSingleResult GetCacheRecord(std::string_view Namespace, const CacheKey& CacheKey, ZenContentType Type) override + { + ZEN_TRACE_CPU("Upstream::GetCacheRecord"); + + std::shared_lock _(m_EndpointsMutex); + + if (m_Options.ReadUpstream) + { + for (auto& Endpoint : m_Endpoints) + { + if (Endpoint->GetState() != UpstreamEndpointState::kOk) + { + continue; + } + + UpstreamEndpointStats& Stats = Endpoint->Stats(); + metrics::OperationTiming::Scope Scope(Stats.CacheGetRequestTiming); + GetUpstreamCacheSingleResult Result = Endpoint->GetCacheRecord(Namespace, CacheKey, Type); + Scope.Stop(); + + Stats.CacheGetCount.Increment(1); + Stats.CacheGetTotalBytes.Increment(Result.Status.Bytes); + + if (Result.Status.Success) + { + Stats.CacheHitCount.Increment(1); + + return Result; + } + + if (Result.Status.Error) + { + Stats.CacheErrorCount.Increment(1); + + ZEN_WARN("get cache record FAILED, endpoint '{}', reason '{}', error code '{}'", + Endpoint->GetEndpointInfo().Url, + Result.Status.Error.Reason, + Result.Status.Error.ErrorCode); + } + } + } + + return {}; + } + + virtual void GetCacheRecords(std::string_view Namespace, + std::span Requests, + OnCacheRecordGetComplete&& OnComplete) override final + { + ZEN_TRACE_CPU("Upstream::GetCacheRecords"); + + std::shared_lock _(m_EndpointsMutex); + + std::vector RemainingKeys(Requests.begin(), Requests.end()); + + if (m_Options.ReadUpstream) + { + for (auto& Endpoint : m_Endpoints) + { + if (RemainingKeys.empty()) + { + break; + } + + if (Endpoint->GetState() != UpstreamEndpointState::kOk) + { + continue; + } + + UpstreamEndpointStats& Stats = Endpoint->Stats(); + std::vector Missing; + GetUpstreamCacheResult Result; + { + metrics::OperationTiming::Scope Scope(Stats.CacheGetRequestTiming); + + Result = Endpoint->GetCacheRecords(Namespace, RemainingKeys, [&](CacheRecordGetCompleteParams&& Params) { + if (Params.Record) + { + OnComplete(std::forward(Params)); + + Stats.CacheHitCount.Increment(1); + } + else + { + Missing.push_back(&Params.Request); + } + }); + } + + Stats.CacheGetCount.Increment(int64_t(RemainingKeys.size())); + Stats.CacheGetTotalBytes.Increment(Result.Bytes); + + if (Result.Error) + { + Stats.CacheErrorCount.Increment(1); + + ZEN_WARN("get cache record(s) (rpc) FAILED, endpoint '{}', reason '{}', error code '{}'", + Endpoint->GetEndpointInfo().Url, + Result.Error.Reason, + Result.Error.ErrorCode); + } + + RemainingKeys = std::move(Missing); + } + } + + const UpstreamEndpointInfo Info; + for (CacheKeyRequest* Request : RemainingKeys) + { + OnComplete({.Request = *Request, .Record = CbObjectView(), .Package = CbPackage()}); + } + } + + virtual void GetCacheChunks(std::string_view Namespace, + std::span CacheChunkRequests, + OnCacheChunksGetComplete&& OnComplete) override final + { + ZEN_TRACE_CPU("Upstream::GetCacheChunks"); + + std::shared_lock _(m_EndpointsMutex); + + std::vector RemainingKeys(CacheChunkRequests.begin(), CacheChunkRequests.end()); + + if (m_Options.ReadUpstream) + { + for (auto& Endpoint : m_Endpoints) + { + if (RemainingKeys.empty()) + { + break; + } + + if (Endpoint->GetState() != UpstreamEndpointState::kOk) + { + continue; + } + + UpstreamEndpointStats& Stats = Endpoint->Stats(); + std::vector Missing; + GetUpstreamCacheResult Result; + { + metrics::OperationTiming::Scope Scope(Endpoint->Stats().CacheGetRequestTiming); + + Result = Endpoint->GetCacheChunks(Namespace, RemainingKeys, [&](CacheChunkGetCompleteParams&& Params) { + if (Params.RawHash != Params.RawHash.Zero) + { + OnComplete(std::forward(Params)); + + Stats.CacheHitCount.Increment(1); + } + else + { + Missing.push_back(&Params.Request); + } + }); + } + + Stats.CacheGetCount.Increment(int64_t(RemainingKeys.size())); + Stats.CacheGetTotalBytes.Increment(Result.Bytes); + + if (Result.Error) + { + Stats.CacheErrorCount.Increment(1); + + ZEN_WARN("get cache chunks(s) (rpc) FAILED, endpoint '{}', reason '{}', error code '{}'", + Endpoint->GetEndpointInfo().Url, + Result.Error.Reason, + Result.Error.ErrorCode); + } + + RemainingKeys = std::move(Missing); + } + } + + const UpstreamEndpointInfo Info; + for (CacheChunkRequest* RequestPtr : RemainingKeys) + { + OnComplete({.Request = *RequestPtr, .RawHash = IoHash::Zero, .RawSize = 0, .Value = IoBuffer()}); + } + } + + virtual GetUpstreamCacheSingleResult GetCacheChunk(std::string_view Namespace, + const CacheKey& CacheKey, + const IoHash& ValueContentId) override + { + ZEN_TRACE_CPU("Upstream::GetCacheChunk"); + + if (m_Options.ReadUpstream) + { + for (auto& Endpoint : m_Endpoints) + { + if (Endpoint->GetState() != UpstreamEndpointState::kOk) + { + continue; + } + + UpstreamEndpointStats& Stats = Endpoint->Stats(); + metrics::OperationTiming::Scope Scope(Stats.CacheGetRequestTiming); + GetUpstreamCacheSingleResult Result = Endpoint->GetCacheChunk(Namespace, CacheKey, ValueContentId); + Scope.Stop(); + + Stats.CacheGetCount.Increment(1); + Stats.CacheGetTotalBytes.Increment(Result.Status.Bytes); + + if (Result.Status.Success) + { + Stats.CacheHitCount.Increment(1); + + return Result; + } + + if (Result.Status.Error) + { + Stats.CacheErrorCount.Increment(1); + + ZEN_WARN("get cache chunk FAILED, endpoint '{}', reason '{}', error code '{}'", + Endpoint->GetEndpointInfo().Url, + Result.Status.Error.Reason, + Result.Status.Error.ErrorCode); + } + } + } + + return {}; + } + + virtual void GetCacheValues(std::string_view Namespace, + std::span CacheValueRequests, + OnCacheValueGetComplete&& OnComplete) override final + { + ZEN_TRACE_CPU("Upstream::GetCacheValues"); + + std::shared_lock _(m_EndpointsMutex); + + std::vector RemainingKeys(CacheValueRequests.begin(), CacheValueRequests.end()); + + if (m_Options.ReadUpstream) + { + for (auto& Endpoint : m_Endpoints) + { + if (RemainingKeys.empty()) + { + break; + } + + if (Endpoint->GetState() != UpstreamEndpointState::kOk) + { + continue; + } + + UpstreamEndpointStats& Stats = Endpoint->Stats(); + std::vector Missing; + GetUpstreamCacheResult Result; + { + metrics::OperationTiming::Scope Scope(Endpoint->Stats().CacheGetRequestTiming); + + Result = Endpoint->GetCacheValues(Namespace, RemainingKeys, [&](CacheValueGetCompleteParams&& Params) { + if (Params.RawHash != Params.RawHash.Zero) + { + OnComplete(std::forward(Params)); + + Stats.CacheHitCount.Increment(1); + } + else + { + Missing.push_back(&Params.Request); + } + }); + } + + Stats.CacheGetCount.Increment(int64_t(RemainingKeys.size())); + Stats.CacheGetTotalBytes.Increment(Result.Bytes); + + if (Result.Error) + { + Stats.CacheErrorCount.Increment(1); + + ZEN_WARN("get cache values(s) (rpc) FAILED, endpoint '{}', reason '{}', error code '{}'", + Endpoint->GetEndpointInfo().Url, + Result.Error.Reason, + Result.Error.ErrorCode); + } + + RemainingKeys = std::move(Missing); + } + } + + const UpstreamEndpointInfo Info; + for (CacheValueRequest* RequestPtr : RemainingKeys) + { + OnComplete({.Request = *RequestPtr, .RawHash = IoHash::Zero, .RawSize = 0, .Value = IoBuffer()}); + } + } + + virtual void EnqueueUpstream(UpstreamCacheRecord CacheRecord) override + { + if (m_RunState.IsRunning && m_Options.WriteUpstream && m_Endpoints.size() > 0) + { + if (!m_UpstreamThreads.empty()) + { + m_UpstreamQueue.Enqueue(std::move(CacheRecord)); + } + else + { + ProcessCacheRecord(std::move(CacheRecord)); + } + } + } + + virtual void GetStatus(CbObjectWriter& Status) override + { + ZEN_TRACE_CPU("Upstream::GetStatus"); + + Status << "active" << IsActive(); + Status << "reading" << m_Options.ReadUpstream; + Status << "writing" << m_Options.WriteUpstream; + Status << "worker_threads" << m_UpstreamThreads.size(); + Status << "queue_count" << m_UpstreamQueue.Size(); + + Status.BeginArray("endpoints"); + for (const auto& Ep : m_Endpoints) + { + const UpstreamEndpointInfo& EpInfo = Ep->GetEndpointInfo(); + const UpstreamEndpointStatus EpStatus = Ep->GetStatus(); + UpstreamEndpointStats& EpStats = Ep->Stats(); + + Status.BeginObject(); + Status << "name" << EpInfo.Name; + Status << "url" << EpInfo.Url; + Status << "state" << ToString(EpStatus.State); + Status << "reason" << EpStatus.Reason; + + Status.BeginObject("cache"sv); + { + const int64_t GetCount = EpStats.CacheGetCount.Value(); + const int64_t HitCount = EpStats.CacheHitCount.Value(); + const int64_t ErrorCount = EpStats.CacheErrorCount.Value(); + const double HitRatio = GetCount > 0 ? double(HitCount) / double(GetCount) : 0.0; + const double ErrorRatio = GetCount > 0 ? double(ErrorCount) / double(GetCount) : 0.0; + + metrics::EmitSnapshot("get_requests"sv, EpStats.CacheGetRequestTiming, Status); + Status << "get_bytes" << EpStats.CacheGetTotalBytes.Value(); + Status << "get_count" << GetCount; + Status << "hit_count" << HitCount; + Status << "hit_ratio" << HitRatio; + Status << "error_count" << ErrorCount; + Status << "error_ratio" << ErrorRatio; + metrics::EmitSnapshot("put_requests"sv, EpStats.CachePutRequestTiming, Status); + Status << "put_bytes" << EpStats.CachePutTotalBytes.Value(); + } + Status.EndObject(); + + Status.EndObject(); + } + Status.EndArray(); + } + +private: + void ProcessCacheRecord(UpstreamCacheRecord CacheRecord) + { + ZEN_TRACE_CPU("Upstream::ProcessCacheRecord"); + + ZenCacheValue CacheValue; + std::vector Payloads; + + if (!m_CacheStore.Get(CacheRecord.Context, CacheRecord.Namespace, CacheRecord.Key.Bucket, CacheRecord.Key.Hash, CacheValue)) + { + ZEN_WARN("process upstream FAILED, '{}/{}/{}', cache record doesn't exist", + CacheRecord.Namespace, + CacheRecord.Key.Bucket, + CacheRecord.Key.Hash); + return; + } + + for (const IoHash& ValueContentId : CacheRecord.ValueContentIds) + { + if (IoBuffer Payload = m_CidStore.FindChunkByCid(ValueContentId)) + { + Payloads.push_back(Payload); + } + else + { + ZEN_WARN("process upstream FAILED, '{}/{}/{}/{}', ValueContentId doesn't exist in CAS", + CacheRecord.Namespace, + CacheRecord.Key.Bucket, + CacheRecord.Key.Hash, + ValueContentId); + return; + } + } + + std::shared_lock _(m_EndpointsMutex); + + for (auto& Endpoint : m_Endpoints) + { + if (Endpoint->GetState() != UpstreamEndpointState::kOk) + { + continue; + } + + UpstreamEndpointStats& Stats = Endpoint->Stats(); + PutUpstreamCacheResult Result; + { + metrics::OperationTiming::Scope Scope(Stats.CachePutRequestTiming); + Result = Endpoint->PutCacheRecord(CacheRecord, CacheValue.Value, std::span(Payloads)); + } + + Stats.CachePutTotalBytes.Increment(Result.Bytes); + + if (!Result.Success) + { + ZEN_WARN("upload cache record '{}/{}/{}' FAILED, endpoint '{}', reason '{}'", + CacheRecord.Namespace, + CacheRecord.Key.Bucket, + CacheRecord.Key.Hash, + Endpoint->GetEndpointInfo().Url, + Result.Reason); + } + } + } + + void ProcessUpstreamQueue(int ThreadIndex) + { + std::string ThreadName = fmt::format("upstream_{}", ThreadIndex); + SetCurrentThreadName(ThreadName); + + for (;;) + { + UpstreamCacheRecord CacheRecord; + if (m_UpstreamQueue.WaitAndDequeue(CacheRecord)) + { + try + { + ProcessCacheRecord(std::move(CacheRecord)); + } + catch (const std::exception& Err) + { + ZEN_ERROR("upload cache record '{}/{}/{}' FAILED, reason '{}'", + CacheRecord.Namespace, + CacheRecord.Key.Bucket, + CacheRecord.Key.Hash, + Err.what()); + } + } + + if (!m_RunState.IsRunning) + { + break; + } + } + } + + void MonitorEndpoints() + { + SetCurrentThreadName("upstream_monitor"); + + for (;;) + { + { + std::unique_lock lk(m_RunState.Mutex); + if (m_RunState.ExitSignal.wait_for(lk, m_Options.HealthCheckInterval, [this]() { return !m_RunState.IsRunning.load(); })) + { + break; + } + } + + try + { + std::vector Endpoints; + + { + std::shared_lock _(m_EndpointsMutex); + + for (auto& Endpoint : m_Endpoints) + { + UpstreamEndpointState State = Endpoint->GetState(); + if (State == UpstreamEndpointState::kError) + { + Endpoints.push_back(Endpoint.get()); + ZEN_WARN("HEALTH - endpoint '{} - {}' is in error state '{}'", + Endpoint->GetEndpointInfo().Name, + Endpoint->GetEndpointInfo().Url, + Endpoint->GetStatus().Reason); + } + if (State == UpstreamEndpointState::kUnauthorized) + { + Endpoints.push_back(Endpoint.get()); + } + } + } + + for (auto& Endpoint : Endpoints) + { + const UpstreamEndpointInfo& Info = Endpoint->GetEndpointInfo(); + const UpstreamEndpointStatus Status = Endpoint->Initialize(); + + if (Status.State == UpstreamEndpointState::kOk) + { + ZEN_INFO("HEALTH - endpoint '{} - {}' Ok", Info.Name, Info.Url); + } + else + { + const std::string Reason = Status.Reason.empty() ? "" : fmt::format(", reason '{}'", Status.Reason); + ZEN_WARN("HEALTH - endpoint '{} - {}' {} {}", Info.Name, Info.Url, ToString(Status.State), Reason); + } + } + } + catch (const std::exception& Err) + { + ZEN_ERROR("check endpoint(s) health FAILED, reason '{}'", Err.what()); + } + } + } + + void Shutdown() + { + if (m_RunState.Stop()) + { + m_UpstreamQueue.CompleteAdding(); + for (std::thread& Thread : m_UpstreamThreads) + { + Thread.join(); + } + if (m_EndpointMonitorThread.joinable()) + { + m_EndpointMonitorThread.join(); + } + m_UpstreamThreads.clear(); + m_Endpoints.clear(); + } + } + + LoggerRef Log() { return m_Log; } + + using UpstreamQueue = BlockingQueue; + + struct RunState + { + std::mutex Mutex; + std::condition_variable ExitSignal; + std::atomic_bool IsRunning{false}; + + bool Stop() + { + bool Stopped = false; + { + std::lock_guard _(Mutex); + Stopped = IsRunning.exchange(false); + } + if (Stopped) + { + ExitSignal.notify_all(); + } + return Stopped; + } + }; + + LoggerRef m_Log; + UpstreamCacheOptions m_Options; + ZenCacheStore& m_CacheStore; + CidStore& m_CidStore; + UpstreamQueue m_UpstreamQueue; + std::shared_mutex m_EndpointsMutex; + std::vector> m_Endpoints; + std::vector m_UpstreamThreads; + std::thread m_EndpointMonitorThread; + RunState m_RunState; +}; + +////////////////////////////////////////////////////////////////////////// + +std::unique_ptr +UpstreamEndpoint::CreateZenEndpoint(const ZenStructuredCacheClientOptions& Options) +{ + return std::make_unique(Options); +} + +std::unique_ptr +UpstreamEndpoint::CreateJupiterEndpoint(const JupiterClientOptions& Options, const UpstreamAuthConfig& AuthConfig, AuthMgr& Mgr) +{ + return std::make_unique(Options, AuthConfig, Mgr); +} + +std::unique_ptr +CreateUpstreamCache(const UpstreamCacheOptions& Options, ZenCacheStore& CacheStore, CidStore& CidStore) +{ + return std::make_unique(Options, CacheStore, CidStore); +} + +} // namespace zen diff --git a/src/zenserver/storage/upstream/upstreamcache.h b/src/zenserver/storage/upstream/upstreamcache.h new file mode 100644 index 000000000..d5d61c8d9 --- /dev/null +++ b/src/zenserver/storage/upstream/upstreamcache.h @@ -0,0 +1,167 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace zen { + +class CbObjectView; +class AuthMgr; +class CbObjectView; +class CbPackage; +class CbObjectWriter; +class CidStore; +class ZenCacheStore; +struct JupiterClientOptions; +class JupiterAccessTokenProvider; +struct ZenStructuredCacheClientOptions; + +struct UpstreamEndpointStats +{ + metrics::OperationTiming CacheGetRequestTiming; + metrics::OperationTiming CachePutRequestTiming; + metrics::Counter CacheGetTotalBytes; + metrics::Counter CachePutTotalBytes; + metrics::Counter CacheGetCount; + metrics::Counter CacheHitCount; + metrics::Counter CacheErrorCount; +}; + +enum class UpstreamEndpointState : uint32_t +{ + kDisabled, + kUnauthorized, + kError, + kOk +}; + +inline std::string_view +ToString(UpstreamEndpointState State) +{ + using namespace std::literals; + + switch (State) + { + case UpstreamEndpointState::kDisabled: + return "Disabled"sv; + case UpstreamEndpointState::kUnauthorized: + return "Unauthorized"sv; + case UpstreamEndpointState::kError: + return "Error"sv; + case UpstreamEndpointState::kOk: + return "Ok"sv; + default: + return "Unknown"sv; + } +} + +struct UpstreamAuthConfig +{ + std::string_view OAuthUrl; + std::string_view OAuthClientId; + std::string_view OAuthClientSecret; + std::string_view OpenIdProvider; + std::string_view AccessToken; +}; + +struct UpstreamEndpointStatus +{ + std::string Reason; + UpstreamEndpointState State; +}; + +struct GetUpstreamCacheSingleResult +{ + GetUpstreamCacheResult Status; + IoBuffer Value; + const UpstreamEndpointInfo* Source = nullptr; +}; + +/** + * The upstream endpoint is responsible for handling upload/downloading of cache records. + */ +class UpstreamEndpoint +{ +public: + virtual ~UpstreamEndpoint() = default; + + virtual UpstreamEndpointStatus Initialize() = 0; + + virtual const UpstreamEndpointInfo& GetEndpointInfo() const = 0; + + virtual UpstreamEndpointState GetState() = 0; + virtual UpstreamEndpointStatus GetStatus() = 0; + + virtual GetUpstreamCacheSingleResult GetCacheRecord(std::string_view Namespace, const CacheKey& CacheKey, ZenContentType Type) = 0; + virtual GetUpstreamCacheResult GetCacheRecords(std::string_view Namespace, + std::span Requests, + OnCacheRecordGetComplete&& OnComplete) = 0; + + virtual GetUpstreamCacheResult GetCacheValues(std::string_view Namespace, + std::span CacheValueRequests, + OnCacheValueGetComplete&& OnComplete) = 0; + + virtual GetUpstreamCacheSingleResult GetCacheChunk(std::string_view Namespace, const CacheKey& CacheKey, const IoHash& PayloadId) = 0; + virtual GetUpstreamCacheResult GetCacheChunks(std::string_view Namespace, + std::span CacheChunkRequests, + OnCacheChunksGetComplete&& OnComplete) = 0; + + virtual PutUpstreamCacheResult PutCacheRecord(const UpstreamCacheRecord& CacheRecord, + IoBuffer RecordValue, + std::span Payloads) = 0; + + virtual UpstreamEndpointStats& Stats() = 0; + + static std::unique_ptr CreateZenEndpoint(const ZenStructuredCacheClientOptions& Options); + + static std::unique_ptr CreateJupiterEndpoint(const JupiterClientOptions& Options, + const UpstreamAuthConfig& AuthConfig, + AuthMgr& Mgr); +}; + +/** + * Manages one or more upstream cache endpoints. + */ + +class UpstreamCache : public UpstreamCacheClient +{ +public: + virtual void Initialize() = 0; + + virtual void RegisterEndpoint(std::unique_ptr Endpoint) = 0; + virtual void IterateEndpoints(std::function&& Fn) = 0; + + virtual GetUpstreamCacheSingleResult GetCacheRecord(std::string_view Namespace, const CacheKey& CacheKey, ZenContentType Type) = 0; + + virtual GetUpstreamCacheSingleResult GetCacheChunk(std::string_view Namespace, + const CacheKey& CacheKey, + const IoHash& ValueContentId) = 0; + + virtual void GetStatus(CbObjectWriter& CbO) = 0; +}; + +struct UpstreamCacheOptions +{ + std::chrono::seconds HealthCheckInterval{5}; + uint32_t ThreadCount = 4; + bool ReadUpstream = true; + bool WriteUpstream = true; +}; + +std::unique_ptr CreateUpstreamCache(const UpstreamCacheOptions& Options, ZenCacheStore& CacheStore, CidStore& CidStore); + +} // namespace zen diff --git a/src/zenserver/storage/upstream/upstreamservice.cpp b/src/zenserver/storage/upstream/upstreamservice.cpp new file mode 100644 index 000000000..e7f7d2d5c --- /dev/null +++ b/src/zenserver/storage/upstream/upstreamservice.cpp @@ -0,0 +1,55 @@ +// Copyright Epic Games, Inc. All Rights Reserved. +#include "upstreamservice.h" + +#include "upstreamcache.h" + +#include +#include + +namespace zen { + +using namespace std::literals; + +HttpUpstreamService::HttpUpstreamService(UpstreamCache& Upstream, AuthMgr& Mgr) : m_Upstream(Upstream), m_AuthMgr(Mgr) +{ + m_Router.RegisterRoute( + "endpoints", + [this](HttpRouterRequest& Req) { + CbObjectWriter Writer; + Writer.BeginArray("Endpoints"sv); + m_Upstream.IterateEndpoints([&Writer](UpstreamEndpoint& Ep) { + UpstreamEndpointInfo Info = Ep.GetEndpointInfo(); + UpstreamEndpointStatus Status = Ep.GetStatus(); + + Writer.BeginObject(); + Writer << "Name"sv << Info.Name; + Writer << "Url"sv << Info.Url; + Writer << "State"sv << ToString(Status.State); + Writer << "Reason"sv << Status.Reason; + Writer.EndObject(); + + return true; + }); + Writer.EndArray(); + Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Writer.Save()); + }, + HttpVerb::kGet); +} + +HttpUpstreamService::~HttpUpstreamService() +{ +} + +const char* +HttpUpstreamService::BaseUri() const +{ + return "/upstream/"; +} + +void +HttpUpstreamService::HandleRequest(zen::HttpServerRequest& Request) +{ + m_Router.HandleRequest(Request); +} + +} // namespace zen diff --git a/src/zenserver/storage/upstream/upstreamservice.h b/src/zenserver/storage/upstream/upstreamservice.h new file mode 100644 index 000000000..f1da03c8c --- /dev/null +++ b/src/zenserver/storage/upstream/upstreamservice.h @@ -0,0 +1,27 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +namespace zen { + +class AuthMgr; +class UpstreamCache; + +class HttpUpstreamService final : public zen::HttpService +{ +public: + HttpUpstreamService(UpstreamCache& Upstream, AuthMgr& Mgr); + virtual ~HttpUpstreamService(); + + virtual const char* BaseUri() const override; + virtual void HandleRequest(zen::HttpServerRequest& Request) override; + +private: + UpstreamCache& m_Upstream; + AuthMgr& m_AuthMgr; + HttpRequestRouter m_Router; +}; + +} // namespace zen diff --git a/src/zenserver/storage/upstream/zen.cpp b/src/zenserver/storage/upstream/zen.cpp new file mode 100644 index 000000000..25fd3a3bb --- /dev/null +++ b/src/zenserver/storage/upstream/zen.cpp @@ -0,0 +1,251 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "zen.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "diag/logging.h" + +#include +#include + +namespace zen { + +////////////////////////////////////////////////////////////////////////// + +ZenStructuredCacheClient::ZenStructuredCacheClient(const ZenStructuredCacheClientOptions& Options) +: m_Log(logging::Get(std::string_view("zenclient"))) +, m_ServiceUrl(Options.Url) +, m_ConnectTimeout(Options.ConnectTimeout) +, m_Timeout(Options.Timeout) +{ +} + +ZenStructuredCacheClient::~ZenStructuredCacheClient() +{ +} + +////////////////////////////////////////////////////////////////////////// + +using namespace std::literals; + +ZenStructuredCacheSession::ZenStructuredCacheSession(Ref&& OuterClient) +: m_Log(OuterClient->Log()) +, m_Client(std::move(OuterClient)) +{ +} + +ZenStructuredCacheSession::~ZenStructuredCacheSession() +{ +} + +ZenCacheResult +ZenStructuredCacheSession::CheckHealth() +{ + HttpClient Http{m_Client->ServiceUrl()}; + + HttpClient::Response Response = Http.Get("/health/check"sv); + + if (auto& Error = Response.Error; Error) + { + return {.ErrorCode = static_cast(Error->ErrorCode), .Reason = std::move(Error->ErrorMessage)}; + } + + return {.Bytes = Response.DownloadedBytes, + .ElapsedSeconds = Response.ElapsedSeconds, + .Success = Response.StatusCode == HttpResponseCode::OK}; +} + +ZenCacheResult +ZenStructuredCacheSession::GetCacheRecord(std::string_view Namespace, std::string_view BucketId, const IoHash& Key, ZenContentType Type) +{ + HttpClient Http{m_Client->ServiceUrl()}; + + ExtendableStringBuilder<256> Uri; + Uri << "/z$/"; + if (Namespace != ZenCacheStore::DefaultNamespace) + { + Uri << Namespace << "/"; + } + Uri << BucketId << "/" << Key.ToHexString(); + + HttpClient::Response Response = Http.Get(Uri, {{"Accept", std::string{MapContentTypeToString(Type)}}}); + ZEN_DEBUG("GET {}", Response); + + if (auto& Error = Response.Error; Error) + { + return {.ErrorCode = static_cast(Error->ErrorCode), .Reason = std::move(Error->ErrorMessage)}; + } + + const bool Success = Response.StatusCode == HttpResponseCode::OK; + const IoBuffer Buffer = Success ? Response.ResponsePayload : IoBuffer{}; + + return {.Response = Buffer, .Bytes = Response.DownloadedBytes, .ElapsedSeconds = Response.ElapsedSeconds, .Success = Success}; +} + +ZenCacheResult +ZenStructuredCacheSession::GetCacheChunk(std::string_view Namespace, + std::string_view BucketId, + const IoHash& Key, + const IoHash& ValueContentId) +{ + HttpClient Http{m_Client->ServiceUrl()}; + + ExtendableStringBuilder<256> Uri; + Uri << "/z$/"; + if (Namespace != ZenCacheStore::DefaultNamespace) + { + Uri << Namespace << "/"; + } + Uri << BucketId << "/" << Key.ToHexString() << "/" << ValueContentId.ToHexString(); + + HttpClient::Response Response = Http.Get(Uri, {{"Accept", "application/x-ue-comp"}}); + ZEN_DEBUG("GET {}", Response); + + if (auto& Error = Response.Error; Error) + { + return {.ErrorCode = static_cast(Error->ErrorCode), .Reason = std::move(Error->ErrorMessage)}; + } + + const bool Success = Response.StatusCode == HttpResponseCode::OK; + const IoBuffer Buffer = Success ? Response.ResponsePayload : IoBuffer{}; + + return {.Response = Buffer, .Bytes = Response.DownloadedBytes, .ElapsedSeconds = Response.ElapsedSeconds, .Success = Success}; +} + +ZenCacheResult +ZenStructuredCacheSession::PutCacheRecord(std::string_view Namespace, + std::string_view BucketId, + const IoHash& Key, + IoBuffer Value, + ZenContentType Type) +{ + HttpClient Http{m_Client->ServiceUrl()}; + + ExtendableStringBuilder<256> Uri; + Uri << "/z$/"; + if (Namespace != ZenCacheStore::DefaultNamespace) + { + Uri << Namespace << "/"; + } + Uri << BucketId << "/" << Key.ToHexString(); + + Value.SetContentType(Type); + + HttpClient::Response Response = Http.Put(Uri, Value); + ZEN_DEBUG("PUT {}", Response); + + if (auto& Error = Response.Error; Error) + { + return {.ErrorCode = static_cast(Error->ErrorCode), .Reason = std::move(Error->ErrorMessage)}; + } + + const bool Success = Response.StatusCode == HttpResponseCode::OK || Response.StatusCode == HttpResponseCode::Created; + + return {.Bytes = Response.DownloadedBytes, .ElapsedSeconds = Response.ElapsedSeconds, .Success = Success}; +} + +ZenCacheResult +ZenStructuredCacheSession::PutCacheValue(std::string_view Namespace, + std::string_view BucketId, + const IoHash& Key, + const IoHash& ValueContentId, + IoBuffer Payload) +{ + HttpClient Http{m_Client->ServiceUrl()}; + + ExtendableStringBuilder<256> Uri; + Uri << "/z$/"; + if (Namespace != ZenCacheStore::DefaultNamespace) + { + Uri << Namespace << "/"; + } + Uri << BucketId << "/" << Key.ToHexString() << "/" << ValueContentId.ToHexString(); + + Payload.SetContentType(HttpContentType::kCompressedBinary); + + HttpClient::Response Response = Http.Put(Uri, Payload); + ZEN_DEBUG("PUT {}", Response); + + if (auto& Error = Response.Error; Error) + { + return {.ErrorCode = static_cast(Error->ErrorCode), .Reason = std::move(Error->ErrorMessage)}; + } + + const bool Success = Response.StatusCode == HttpResponseCode::OK || Response.StatusCode == HttpResponseCode::Created; + + return {.Bytes = Response.DownloadedBytes, .ElapsedSeconds = Response.ElapsedSeconds, .Success = Success}; +} + +ZenCacheResult +ZenStructuredCacheSession::InvokeRpc(const CbObjectView& Request) +{ + HttpClient Http{m_Client->ServiceUrl()}; + + ExtendableStringBuilder<256> Uri; + Uri << "/z$/$rpc"; + + // TODO: this seems redundant, we should be able to send the data more directly, without the BinaryWriter + + BinaryWriter BodyWriter; + Request.CopyTo(BodyWriter); + + IoBuffer Body{IoBuffer::Wrap, BodyWriter.GetData(), BodyWriter.GetSize()}; + Body.SetContentType(HttpContentType::kCbObject); + + HttpClient::Response Response = Http.Post(Uri, Body, {{"Accept", "application/x-ue-cbpkg"}}); + ZEN_DEBUG("POST {}", Response); + + if (auto& Error = Response.Error; Error) + { + return {.ErrorCode = static_cast(Error->ErrorCode), .Reason = std::move(Error->ErrorMessage)}; + } + + const bool Success = Response.StatusCode == HttpResponseCode::OK; + const IoBuffer Buffer = Success ? Response.ResponsePayload : IoBuffer{}; + + return {.Response = std::move(Buffer), + .Bytes = Response.DownloadedBytes, + .ElapsedSeconds = Response.ElapsedSeconds, + .Success = Success}; +} + +ZenCacheResult +ZenStructuredCacheSession::InvokeRpc(const CbPackage& Request) +{ + HttpClient Http{m_Client->ServiceUrl()}; + + ExtendableStringBuilder<256> Uri; + Uri << "/z$/$rpc"; + + IoBuffer Message = FormatPackageMessageBuffer(Request).Flatten().AsIoBuffer(); + Message.SetContentType(HttpContentType::kCbPackage); + + HttpClient::Response Response = Http.Post(Uri, Message, {{"Accept", "application/x-ue-cbpkg"}}); + ZEN_DEBUG("POST {}", Response); + + if (auto& Error = Response.Error; Error) + { + return {.ErrorCode = static_cast(Error->ErrorCode), .Reason = std::move(Error->ErrorMessage)}; + } + + const bool Success = Response.StatusCode == HttpResponseCode::OK; + const IoBuffer Buffer = Success ? Response.ResponsePayload : IoBuffer{}; + + return {.Response = std::move(Buffer), + .Bytes = Response.DownloadedBytes, + .ElapsedSeconds = Response.ElapsedSeconds, + .Success = Success}; +} + +} // namespace zen diff --git a/src/zenserver/storage/upstream/zen.h b/src/zenserver/storage/upstream/zen.h new file mode 100644 index 000000000..6321b46b1 --- /dev/null +++ b/src/zenserver/storage/upstream/zen.h @@ -0,0 +1,103 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include + +struct ZenCacheValue; + +namespace zen { + +class CbObjectWriter; +class CbObjectView; +class CbPackage; +class ZenStructuredCacheClient; + +////////////////////////////////////////////////////////////////////////// + +struct ZenCacheResult +{ + IoBuffer Response; + int64_t Bytes = {}; + double ElapsedSeconds = {}; + int32_t ErrorCode = {}; + std::string Reason; + bool Success = false; +}; + +struct ZenStructuredCacheClientOptions +{ + std::string_view Name; + std::string_view Url; + std::span Urls; + std::chrono::milliseconds ConnectTimeout{}; + std::chrono::milliseconds Timeout{}; +}; + +/** Zen Structured Cache session + * + * This provides a context in which cache queries can be performed + * + * These are currently all synchronous. Will need to be made asynchronous + */ +class ZenStructuredCacheSession +{ +public: + ZenStructuredCacheSession(Ref&& OuterClient); + ~ZenStructuredCacheSession(); + + ZenCacheResult CheckHealth(); + ZenCacheResult GetCacheRecord(std::string_view Namespace, std::string_view BucketId, const IoHash& Key, ZenContentType Type); + ZenCacheResult GetCacheChunk(std::string_view Namespace, std::string_view BucketId, const IoHash& Key, const IoHash& ValueContentId); + ZenCacheResult PutCacheRecord(std::string_view Namespace, + std::string_view BucketId, + const IoHash& Key, + IoBuffer Value, + ZenContentType Type); + ZenCacheResult PutCacheValue(std::string_view Namespace, + std::string_view BucketId, + const IoHash& Key, + const IoHash& ValueContentId, + IoBuffer Payload); + ZenCacheResult InvokeRpc(const CbObjectView& Request); + ZenCacheResult InvokeRpc(const CbPackage& Package); + +private: + inline LoggerRef Log() { return m_Log; } + + LoggerRef m_Log; + Ref m_Client; +}; + +/** Zen Structured Cache client + * + * This represents an endpoint to query -- actual queries should be done via + * ZenStructuredCacheSession + */ +class ZenStructuredCacheClient : public RefCounted +{ +public: + ZenStructuredCacheClient(const ZenStructuredCacheClientOptions& Options); + ~ZenStructuredCacheClient(); + + std::string_view ServiceUrl() const { return m_ServiceUrl; } + + inline LoggerRef Log() { return m_Log; } + +private: + LoggerRef m_Log; + std::string m_ServiceUrl; + std::chrono::milliseconds m_ConnectTimeout; + std::chrono::milliseconds m_Timeout; + + friend class ZenStructuredCacheSession; +}; + +} // namespace zen diff --git a/src/zenserver/storage/vfs/vfsservice.cpp b/src/zenserver/storage/vfs/vfsservice.cpp new file mode 100644 index 000000000..863ec348a --- /dev/null +++ b/src/zenserver/storage/vfs/vfsservice.cpp @@ -0,0 +1,179 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "vfsservice.h" + +#include + +#include + +namespace zen { + +using namespace std::literals; + +#if ZEN_WITH_VFS + +////////////////////////////////////////////////////////////////////////// + +bool +GetContentAsCbObject(HttpServerRequest& HttpReq, CbObject& Cb) +{ + IoBuffer Payload = HttpReq.ReadPayload(); + HttpContentType PayloadContentType = HttpReq.RequestContentType(); + + switch (PayloadContentType) + { + case HttpContentType::kJSON: + case HttpContentType::kUnknownContentType: + case HttpContentType::kText: + { + std::string JsonText(reinterpret_cast(Payload.GetData()), Payload.GetSize()); + Cb = LoadCompactBinaryFromJson(JsonText).AsObject(); + if (!Cb) + { + HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Content format not supported, expected JSON format"); + return false; + } + } + break; + case HttpContentType::kCbObject: + Cb = LoadCompactBinaryObject(Payload); + if (!Cb) + { + HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Content format not supported, expected compact binary format"); + return false; + } + break; + default: + HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid request content type"); + return false; + } + + return true; +} + +////////////////////////////////////////////////////////////////////////// +// +// to test: +// +// echo {"method": "mount", "params": {"path": "d:\\VFS_ROOT"}} | curl.exe http://localhost:8558/vfs --data-binary @- +// echo {"method": "unmount"} | curl.exe http://localhost:8558/vfs --data-binary @- + +VfsService::VfsService(HttpStatusService& StatusService, VfsServiceImpl* ServiceImpl) : m_StatusService(StatusService), m_Impl(ServiceImpl) +{ + m_Router.RegisterRoute( + "info", + [&](HttpRouterRequest& Request) { + CbObjectWriter Cbo; + Cbo << "running" << m_Impl->IsVfsRunning(); + Cbo << "rootpath" << m_Impl->GetMountpointPath(); + + Request.ServerRequest().WriteResponse(HttpResponseCode::OK, Cbo.Save()); + }, + HttpVerb::kGet | HttpVerb::kHead); + + m_Router.RegisterRoute( + "", + [&](HttpRouterRequest& Req) { + CbObject Payload; + + if (!GetContentAsCbObject(Req.ServerRequest(), Payload)) + return; + + std::string_view RpcName = Payload["method"sv].AsString(); + + if (RpcName == "mount"sv) + { + CbObjectView Params = Payload["params"sv].AsObjectView(); + std::string_view Mountpath = Params["path"sv].AsString(); + + if (Mountpath.empty()) + { + return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "no path specified"); + } + + if (m_Impl->IsVfsRunning()) + { + return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "VFS already mounted"); + } + + try + { + m_Impl->Mount(Mountpath); + } + catch (const std::exception& Ex) + { + return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, Ex.what()); + } + + Req.ServerRequest().WriteResponse(HttpResponseCode::OK); + } + else if (RpcName == "unmount"sv) + { + if (!m_Impl->IsVfsRunning()) + { + return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "VFS not active"); + } + + try + { + m_Impl->Unmount(); + } + catch (const std::exception& Ex) + { + return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, Ex.what()); + } + + Req.ServerRequest().WriteResponse(HttpResponseCode::OK); + } + else + { + Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "unknown RPC"sv); + } + }, + HttpVerb::kPost); + m_StatusService.RegisterHandler("vfs", *this); +} + +VfsService::~VfsService() +{ + m_StatusService.UnregisterHandler("vfs", *this); +} + +#else + +VfsService::VfsService(HttpStatusService& StatusService, VfsServiceImpl* ServiceImpl) : m_StatusService(StatusService) +{ + ZEN_UNUSED(ServiceImpl); +} + +VfsService::~VfsService() +{ +} + +#endif + +const char* +VfsService::BaseUri() const +{ + return "/vfs/"; +} + +void +VfsService::HandleStatusRequest(HttpServerRequest& Request) +{ + CbObjectWriter Cbo; + Cbo << "ok" << true; + Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + +void +VfsService::HandleRequest(HttpServerRequest& HttpServiceRequest) +{ + m_Router.HandleRequest(HttpServiceRequest); +} + +} // namespace zen diff --git a/src/zenserver/storage/vfs/vfsservice.h b/src/zenserver/storage/vfs/vfsservice.h new file mode 100644 index 000000000..4e06da878 --- /dev/null +++ b/src/zenserver/storage/vfs/vfsservice.h @@ -0,0 +1,48 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include +#include + +#include + +namespace zen { + +class ProjectStore; +class ZenCacheStore; +struct VfsServiceImpl; + +/** Virtual File System service + + Implements support for exposing data via a virtual file system interface. Currently + this is primarily used to surface various data stored in the local storage service + to users for debugging and exploration purposes. + + Currently, it surfaces information from the structured cache service and from the + project store. + + */ + +class VfsService : public HttpService, public IHttpStatusProvider +{ +public: + explicit VfsService(HttpStatusService& StatusService, VfsServiceImpl* ServiceImpl); + ~VfsService(); + +protected: + virtual const char* BaseUri() const override; + virtual void HandleRequest(HttpServerRequest& HttpServiceRequest) override; + virtual void HandleStatusRequest(HttpServerRequest& Request) override; + +private: + VfsServiceImpl* m_Impl = nullptr; + + HttpStatusService& m_StatusService; + HttpRequestRouter m_Router; + + friend struct VfsServiceDataSource; +}; + +} // namespace zen diff --git a/src/zenserver/storage/workspaces/httpworkspaces.cpp b/src/zenserver/storage/workspaces/httpworkspaces.cpp new file mode 100644 index 000000000..3fea46b2f --- /dev/null +++ b/src/zenserver/storage/workspaces/httpworkspaces.cpp @@ -0,0 +1,1211 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "httpworkspaces.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace zen { +using namespace std::literals; + +ZEN_DEFINE_LOG_CATEGORY_STATIC(LogFs, "fs"sv); + +namespace { + + std::filesystem::path GetPathParameter(HttpServerRequest& ServerRequest, std::string_view Name) + { + if (std::string_view Value = ServerRequest.GetQueryParams().GetValue(Name); !Value.empty()) + { + return std::filesystem::path(HttpServerRequest::Decode(Value)); + } + return {}; + } + + void WriteWorkspaceConfig(CbWriter& Writer, const Workspaces::WorkspaceConfiguration& Config) + { + Writer << "id" << Config.Id; + Writer << "root_path" << Config.RootPath.string(); // utf8? + Writer << "allow_share_creation_from_http" << Config.AllowShareCreationFromHttp; + }; + + void WriteWorkspaceShareConfig(CbWriter& Writer, const Workspaces::WorkspaceShareConfiguration& Config) + { + Writer << "id" << Config.Id; + Writer << "share_path" << Config.SharePath.string(); // utf8? + if (!Config.Alias.empty()) + { + Writer << "alias" << Config.Alias; + } + }; + + void WriteWorkspaceAndSharesConfig(CbWriter& Writer, Workspaces& Workspaces, const Workspaces::WorkspaceConfiguration& WorkspaceConfig) + { + WriteWorkspaceConfig(Writer, WorkspaceConfig); + if (std::optional> ShareIds = Workspaces.GetWorkspaceShares(WorkspaceConfig.Id); ShareIds) + { + Writer.BeginArray("shares"); + { + for (const Oid& ShareId : *ShareIds) + { + if (std::optional WorkspaceShareConfig = + Workspaces.GetWorkspaceShareConfiguration(WorkspaceConfig.Id, ShareId); + WorkspaceShareConfig) + { + Writer.BeginObject(); + { + WriteWorkspaceShareConfig(Writer, *WorkspaceShareConfig); + } + Writer.EndObject(); + } + } + } + Writer.EndArray(); + } + } + +} // namespace + +HttpWorkspacesService::HttpWorkspacesService(HttpStatusService& StatusService, + HttpStatsService& StatsService, + const WorkspacesServeConfig& Cfg, + Workspaces& Workspaces) +: m_Log(logging::Get("workspaces")) +, m_StatusService(StatusService) +, m_StatsService(StatsService) +, m_Config(Cfg) +, m_Workspaces(Workspaces) +{ + Initialize(); +} + +HttpWorkspacesService::~HttpWorkspacesService() +{ + m_StatsService.UnregisterHandler("ws", *this); + m_StatusService.UnregisterHandler("ws", *this); +} + +const char* +HttpWorkspacesService::BaseUri() const +{ + return "/ws/"; +} + +void +HttpWorkspacesService::HandleRequest(HttpServerRequest& Request) +{ + metrics::OperationTiming::Scope $(m_HttpRequests); + + if (m_Router.HandleRequest(Request) == false) + { + ZEN_LOG_WARN(LogFs, "No route found for {0}", Request.RelativeUri()); + return Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Not found"sv); + } +} + +void +HttpWorkspacesService::HandleStatsRequest(HttpServerRequest& HttpReq) +{ + ZEN_TRACE_CPU("WorkspacesService::Stats"); + CbObjectWriter Cbo; + + EmitSnapshot("requests", m_HttpRequests, Cbo); + + Cbo.BeginObject("workspaces"); + { + Cbo.BeginObject("workspace"); + { + Cbo << "readcount" << m_WorkspacesStats.WorkspaceReadCount << "writecount" << m_WorkspacesStats.WorkspaceWriteCount + << "deletecount" << m_WorkspacesStats.WorkspaceDeleteCount; + } + Cbo.EndObject(); + + Cbo.BeginObject("workspaceshare"); + { + Cbo << "readcount" << m_WorkspacesStats.WorkspaceShareReadCount << "writecount" << m_WorkspacesStats.WorkspaceShareWriteCount + << "deletecount" << m_WorkspacesStats.WorkspaceShareDeleteCount; + } + Cbo.EndObject(); + + Cbo.BeginObject("chunk"); + { + Cbo << "hitcount" << m_WorkspacesStats.WorkspaceShareChunkHitCount << "misscount" + << m_WorkspacesStats.WorkspaceShareChunkMissCount; + } + Cbo.EndObject(); + + Cbo << "filescount" << m_WorkspacesStats.WorkspaceShareFilesReadCount; + Cbo << "entriescount" << m_WorkspacesStats.WorkspaceShareEntriesReadCount; + Cbo << "batchcount" << m_WorkspacesStats.WorkspaceShareBatchReadCount; + + Cbo << "requestcount" << m_WorkspacesStats.RequestCount; + Cbo << "badrequestcount" << m_WorkspacesStats.BadRequestCount; + } + Cbo.EndObject(); + + return HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + +void +HttpWorkspacesService::HandleStatusRequest(HttpServerRequest& Request) +{ + ZEN_TRACE_CPU("HttpWorkspacesService::Status"); + CbObjectWriter Cbo; + Cbo << "ok" << true; + Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + +void +HttpWorkspacesService::Initialize() +{ + using namespace std::literals; + + ZEN_LOG_INFO(LogFs, "Initializing Workspaces Service"); + + m_Router.AddPattern("workspace_id", "([[:xdigit:]]{24})"); + m_Router.AddPattern("share_id", "([[:xdigit:]]{24})"); + m_Router.AddPattern("chunk", "([[:xdigit:]]{24})"); + m_Router.AddPattern("share_alias", "([[:alnum:]_.\\+\\-\\[\\]]+)"); + + m_Router.RegisterRoute( + "{workspace_id}/{share_id}/files", + [this](HttpRouterRequest& Req) { FilesRequest(Req); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "{workspace_id}/{share_id}/{chunk}/info", + [this](HttpRouterRequest& Req) { ChunkInfoRequest(Req); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "{workspace_id}/{share_id}/batch", + [this](HttpRouterRequest& Req) { BatchRequest(Req); }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "{workspace_id}/{share_id}/entries", + [this](HttpRouterRequest& Req) { EntriesRequest(Req); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "{workspace_id}/{share_id}/{chunk}", + [this](HttpRouterRequest& Req) { ChunkRequest(Req); }, + HttpVerb::kGet | HttpVerb::kHead); + + m_Router.RegisterRoute( + "share/{share_alias}/files", + [this](HttpRouterRequest& Req) { ShareAliasFilesRequest(Req); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "share/{share_alias}/{chunk}/info", + [this](HttpRouterRequest& Req) { ShareAliasChunkInfoRequest(Req); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "share/{share_alias}/batch", + [this](HttpRouterRequest& Req) { ShareAliasBatchRequest(Req); }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "share/{share_alias}/entries", + [this](HttpRouterRequest& Req) { ShareAliasEntriesRequest(Req); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "share/{share_alias}/{chunk}", + [this](HttpRouterRequest& Req) { ShareAliasChunkRequest(Req); }, + HttpVerb::kGet | HttpVerb::kHead); + + m_Router.RegisterRoute( + "share/{share_alias}", + [this](HttpRouterRequest& Req) { ShareAliasRequest(Req); }, + HttpVerb::kPut | HttpVerb::kGet | HttpVerb::kDelete); + + m_Router.RegisterRoute( + "{workspace_id}/{share_id}", + [this](HttpRouterRequest& Req) { ShareRequest(Req); }, + HttpVerb::kPut | HttpVerb::kGet | HttpVerb::kDelete); + + m_Router.RegisterRoute( + "{workspace_id}", + [this](HttpRouterRequest& Req) { WorkspaceRequest(Req); }, + HttpVerb::kPut | HttpVerb::kGet | HttpVerb::kDelete); + + m_Router.RegisterRoute( + "refresh", + [this](HttpRouterRequest& Req) { RefreshRequest(Req); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "", + [this](HttpRouterRequest& Req) { WorkspacesRequest(Req); }, + HttpVerb::kGet); + + RefreshState(); + + m_StatsService.RegisterHandler("ws", *this); + m_StatusService.RegisterHandler("ws", *this); +} + +std::filesystem::path +HttpWorkspacesService::GetStatePath() const +{ + return m_Config.SystemRootDir / "workspaces"; +} + +void +HttpWorkspacesService::RefreshState() +{ + if (!m_Config.SystemRootDir.empty()) + { + m_Workspaces.RefreshState(GetStatePath()); + } +} + +bool +HttpWorkspacesService::MayChangeConfiguration(const HttpServerRequest& Req) const +{ + ZEN_UNUSED(Req); + return m_Config.AllowConfigurationChanges; +} + +void +HttpWorkspacesService::RefreshRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + RefreshState(); + return ServerRequest.WriteResponse(HttpResponseCode::OK); +} + +void +HttpWorkspacesService::WorkspacesRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + + std::vector WorkspaceIds = m_Workspaces.GetWorkspaces(); + CbObjectWriter Response; + Response.BeginArray("workspaces"); + for (const Oid& WorkspaceId : WorkspaceIds) + { + if (std::optional WorkspaceConfig = m_Workspaces.GetWorkspaceConfiguration(WorkspaceId); + WorkspaceConfig) + { + Response.BeginObject(); + { + WriteWorkspaceAndSharesConfig(Response, m_Workspaces, *WorkspaceConfig); + } + Response.EndObject(); + } + } + Response.EndArray(); + + return ServerRequest.WriteResponse(HttpResponseCode::OK, Response.Save()); +} + +void +HttpWorkspacesService::FilesRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + const Oid WorkspaceId = Oid::TryFromHexString(Req.GetCapture(1)); + if (WorkspaceId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); + } + const Oid ShareId = Oid::TryFromHexString(Req.GetCapture(2)); + if (ShareId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid share id '{}'", Req.GetCapture(2))); + } + FilesRequest(Req, WorkspaceId, ShareId); +} + +void +HttpWorkspacesService::ChunkInfoRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + const Oid WorkspaceId = Oid::TryFromHexString(Req.GetCapture(1)); + if (WorkspaceId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); + } + const Oid ShareId = Oid::TryFromHexString(Req.GetCapture(2)); + if (ShareId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid share id '{}'", Req.GetCapture(2))); + } + const Oid ChunkId = Oid::TryFromHexString(Req.GetCapture(3)); + if (ChunkId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid chunk id '{}'", Req.GetCapture(3))); + } + ChunkInfoRequest(Req, WorkspaceId, ShareId, ChunkId); +} + +void +HttpWorkspacesService::BatchRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + const Oid WorkspaceId = Oid::TryFromHexString(Req.GetCapture(1)); + if (WorkspaceId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); + } + const Oid ShareId = Oid::TryFromHexString(Req.GetCapture(2)); + if (ShareId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid share id '{}'", Req.GetCapture(2))); + } + BatchRequest(Req, WorkspaceId, ShareId); +} + +void +HttpWorkspacesService::EntriesRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + const Oid WorkspaceId = Oid::TryFromHexString(Req.GetCapture(1)); + if (WorkspaceId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); + } + const Oid ShareId = Oid::TryFromHexString(Req.GetCapture(2)); + if (ShareId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid share id '{}'", Req.GetCapture(2))); + } + EntriesRequest(Req, WorkspaceId, ShareId); +} + +void +HttpWorkspacesService::ChunkRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + const Oid WorkspaceId = Oid::TryFromHexString(Req.GetCapture(1)); + if (WorkspaceId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); + } + const Oid ShareId = Oid::TryFromHexString(Req.GetCapture(2)); + if (ShareId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid share id '{}'", Req.GetCapture(2))); + } + const Oid ChunkId = Oid::TryFromHexString(Req.GetCapture(3)); + if (ChunkId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid chunk id '{}'", Req.GetCapture(3))); + } + ChunkRequest(Req, WorkspaceId, ShareId, ChunkId); +} + +void +HttpWorkspacesService::ShareRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + const Oid WorkspaceId = Oid::TryFromHexString(Req.GetCapture(1)); + if (WorkspaceId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); + } + Oid ShareId = Oid::Zero; + if (Req.GetCapture(2) != Oid::Zero.ToString()) + { + ShareId = Oid::TryFromHexString(Req.GetCapture(2)); + if (ShareId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid share id '{}'", Req.GetCapture(2))); + } + } + ShareRequest(Req, WorkspaceId, ShareId); +} + +void +HttpWorkspacesService::WorkspaceRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + Oid WorkspaceId = Oid::TryFromHexString(Req.GetCapture(1)); + switch (ServerRequest.RequestVerb()) + { + case HttpVerb::kPut: + { + std::filesystem::path WorkspacePath = GetPathParameter(ServerRequest, "root_path"sv); + if (WorkspacePath.empty()) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Invalid 'root_path' parameter"); + } + + if (Req.GetCapture(1) == Oid::Zero.ToString()) + { + // Synthesize Id + WorkspaceId = Workspaces::PathToId(WorkspacePath); + ZEN_INFO("Generated workspace id from path '{}': {}", WorkspacePath, WorkspaceId); + } + else if (WorkspaceId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); + } + + if (!MayChangeConfiguration(ServerRequest)) + { + return ServerRequest.WriteResponse(HttpResponseCode::Unauthorized, + HttpContentType::kText, + fmt::format("Adding workspace {} is not allowed", WorkspaceId)); + } + bool AllowShareCreationFromHttp = false; + if (std::string_view Value = ServerRequest.GetQueryParams().GetValue("allow_share_creation_from_http"); Value == "true"sv) + { + AllowShareCreationFromHttp = true; + } + + m_WorkspacesStats.WorkspaceWriteCount++; + Workspaces::WorkspaceConfiguration OldConfig = Workspaces::FindWorkspace(Log(), GetStatePath(), WorkspaceId); + Workspaces::WorkspaceConfiguration NewConfig = {.Id = WorkspaceId, + .RootPath = WorkspacePath, + .AllowShareCreationFromHttp = AllowShareCreationFromHttp}; + if (OldConfig.Id == WorkspaceId && (OldConfig != NewConfig)) + { + return ServerRequest.WriteResponse( + HttpResponseCode::Conflict, + HttpContentType::kText, + fmt::format("Workspace {} already exists with root path '{}'", WorkspaceId, OldConfig.RootPath)); + } + else if (OldConfig.Id == Oid::Zero) + { + if (Workspaces::WorkspaceConfiguration ConfigWithSameRoot = + Workspaces::FindWorkspace(Log(), GetStatePath(), WorkspacePath); + ConfigWithSameRoot.Id != Oid::Zero) + { + return ServerRequest.WriteResponse( + HttpResponseCode::Conflict, + HttpContentType::kText, + fmt::format("Workspace {} already exists with same root path '{}'", ConfigWithSameRoot.Id, WorkspacePath)); + } + } + + bool Created = Workspaces::AddWorkspace(Log(), GetStatePath(), NewConfig); + if (Created) + { + ZEN_ASSERT(OldConfig.Id == Oid::Zero); + RefreshState(); + return ServerRequest.WriteResponse(HttpResponseCode::Created, HttpContentType::kText, fmt::format("{}", WorkspaceId)); + } + else + { + return ServerRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, fmt::format("{}", WorkspaceId)); + } + } + case HttpVerb::kGet: + { + if (WorkspaceId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); + } + m_WorkspacesStats.WorkspaceReadCount++; + std::optional Workspace = m_Workspaces.GetWorkspaceConfiguration(WorkspaceId); + if (Workspace) + { + CbObjectWriter Response; + WriteWorkspaceAndSharesConfig(Response, m_Workspaces, *Workspace); + return ServerRequest.WriteResponse(HttpResponseCode::OK, Response.Save()); + } + else + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + } + case HttpVerb::kDelete: + { + if (WorkspaceId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); + } + + if (!MayChangeConfiguration(ServerRequest)) + { + return ServerRequest.WriteResponse(HttpResponseCode::Unauthorized, + HttpContentType::kText, + fmt::format("Removing workspace {} is not allowed", WorkspaceId)); + } + + m_WorkspacesStats.WorkspaceDeleteCount++; + bool Deleted = Workspaces::RemoveWorkspace(Log(), GetStatePath(), WorkspaceId); + if (Deleted) + { + RefreshState(); + return ServerRequest.WriteResponse(HttpResponseCode::OK); + } + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + } +} + +void +HttpWorkspacesService::ShareAliasFilesRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + std::string_view Alias = Req.GetCapture(1); + if (Alias.empty()) + { + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid alias '{}'", Req.GetCapture(1))); + } + std::optional WorkspaceAndShareId = m_Workspaces.GetShareAlias(Alias); + if (!WorkspaceAndShareId.has_value()) + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + FilesRequest(Req, WorkspaceAndShareId.value().WorkspaceId, WorkspaceAndShareId.value().ShareId); +} + +void +HttpWorkspacesService::ShareAliasChunkInfoRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + std::string_view Alias = Req.GetCapture(1); + if (Alias.empty()) + { + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid alias '{}'", Req.GetCapture(1))); + } + std::optional WorkspaceAndShareId = m_Workspaces.GetShareAlias(Alias); + if (!WorkspaceAndShareId.has_value()) + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + const Oid ChunkId = Oid::TryFromHexString(Req.GetCapture(2)); + if (ChunkId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid chunk id '{}'", Req.GetCapture(2))); + } + ChunkInfoRequest(Req, WorkspaceAndShareId.value().WorkspaceId, WorkspaceAndShareId.value().ShareId, ChunkId); +} + +void +HttpWorkspacesService::ShareAliasBatchRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + std::string_view Alias = Req.GetCapture(1); + if (Alias.empty()) + { + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid alias '{}'", Req.GetCapture(1))); + } + std::optional WorkspaceAndShareId = m_Workspaces.GetShareAlias(Alias); + if (!WorkspaceAndShareId.has_value()) + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + BatchRequest(Req, WorkspaceAndShareId.value().WorkspaceId, WorkspaceAndShareId.value().ShareId); +} + +void +HttpWorkspacesService::ShareAliasEntriesRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + std::string_view Alias = Req.GetCapture(1); + if (Alias.empty()) + { + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid alias '{}'", Req.GetCapture(1))); + } + std::optional WorkspaceAndShareId = m_Workspaces.GetShareAlias(Alias); + if (!WorkspaceAndShareId.has_value()) + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + EntriesRequest(Req, WorkspaceAndShareId.value().WorkspaceId, WorkspaceAndShareId.value().ShareId); +} + +void +HttpWorkspacesService::ShareAliasChunkRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + std::string_view Alias = Req.GetCapture(1); + if (Alias.empty()) + { + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid alias '{}'", Req.GetCapture(1))); + } + std::optional WorkspaceAndShareId = m_Workspaces.GetShareAlias(Alias); + if (!WorkspaceAndShareId.has_value()) + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + const Oid ChunkId = Oid::TryFromHexString(Req.GetCapture(2)); + if (ChunkId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid chunk id '{}'", Req.GetCapture(2))); + } + ChunkRequest(Req, WorkspaceAndShareId.value().WorkspaceId, WorkspaceAndShareId.value().ShareId, ChunkId); +} + +void +HttpWorkspacesService::ShareAliasRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + std::string_view Alias = Req.GetCapture(1); + if (Alias.empty()) + { + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid alias '{}'", Req.GetCapture(1))); + } + std::optional WorkspaceAndShareId = m_Workspaces.GetShareAlias(Alias); + if (!WorkspaceAndShareId.has_value()) + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + ShareRequest(Req, WorkspaceAndShareId.value().WorkspaceId, WorkspaceAndShareId.value().ShareId); +} + +void +HttpWorkspacesService::FilesRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& ShareId) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + + m_WorkspacesStats.WorkspaceShareFilesReadCount++; + + std::unordered_set WantedFieldNames; + if (auto FieldFilter = HttpServerRequest::Decode(ServerRequest.GetQueryParams().GetValue("fieldnames")); !FieldFilter.empty()) + { + if (FieldFilter != "*") // Get all - empty FieldFilter equal getting all fields + { + ForEachStrTok(FieldFilter, ',', [&](std::string_view FieldName) { + WantedFieldNames.insert(std::string(FieldName)); + return true; + }); + } + } + else + { + const bool FilterClient = ServerRequest.GetQueryParams().GetValue("filter"sv) == "client"sv; + WantedFieldNames.insert("id"); + WantedFieldNames.insert("clientpath"); + if (!FilterClient) + { + WantedFieldNames.insert("serverpath"); + } + } + + bool Refresh = false; + if (auto RefreshStr = ServerRequest.GetQueryParams().GetValue("refresh"); !RefreshStr.empty()) + { + Refresh = StrCaseCompare(std::string(RefreshStr).c_str(), "true") == 0; + } + + const bool WantsAllFields = WantedFieldNames.empty(); + + const bool WantsIdField = WantsAllFields || WantedFieldNames.contains("id"); + const bool WantsClientPathField = WantsAllFields || WantedFieldNames.contains("clientpath"); + const bool WantsServerPathField = WantsAllFields || WantedFieldNames.contains("serverpath"); + const bool WantsRawSizeField = WantsAllFields || WantedFieldNames.contains("rawsize"); + const bool WantsSizeField = WantsAllFields || WantedFieldNames.contains("size"); + + std::optional> Files = + m_Workspaces.GetWorkspaceShareFiles(WorkspaceId, ShareId, Refresh, GetSmallWorkerPool(EWorkloadType::Burst)); + if (!Files.has_value()) + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + + CbObjectWriter Response; + Response.BeginArray("files"sv); + { + for (const Workspaces::ShareFile& Entry : Files.value()) + { + Response.BeginObject(); + if (WantsIdField) + { + Response << "id"sv << Entry.Id; + } + if (WantsServerPathField) + { + Response << "serverpath"sv << Entry.RelativePath; + } + if (WantsClientPathField) + { + Response << "clientpath"sv << Entry.RelativePath; + } + if (WantsSizeField) + { + Response << "size"sv << Entry.Size; + } + if (WantsRawSizeField) + { + Response << "rawsize"sv << Entry.Size; + } + Response.EndObject(); + } + } + Response.EndArray(); + + return ServerRequest.WriteResponse(HttpResponseCode::OK, Response.Save()); +} + +void +HttpWorkspacesService::ChunkInfoRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& ShareId, const Oid& ChunkId) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + Workspaces::ShareFile File = + m_Workspaces.GetWorkspaceShareChunkInfo(WorkspaceId, ShareId, ChunkId, GetSmallWorkerPool(EWorkloadType::Burst)); + if (File.Id != Oid::Zero) + { + CbObjectWriter Response; + Response << "size"sv << File.Size; + m_WorkspacesStats.WorkspaceShareChunkHitCount++; + return ServerRequest.WriteResponse(HttpResponseCode::OK, Response.Save()); + } + m_WorkspacesStats.WorkspaceShareChunkMissCount++; + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); +} + +void +HttpWorkspacesService::BatchRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& ShareId) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + IoBuffer Payload = ServerRequest.ReadPayload(); + std::optional> ChunkRequests = ParseChunkBatchRequest(Payload); + if (!ChunkRequests.has_value()) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "batch payload malformed"); + } + m_WorkspacesStats.WorkspaceShareBatchReadCount++; + std::vector Requests; + Requests.reserve(ChunkRequests.value().size()); + std::transform(ChunkRequests.value().begin(), + ChunkRequests.value().end(), + std::back_inserter(Requests), + [](const RequestChunkEntry& Entry) { + return Workspaces::ChunkRequest{.ChunkId = Entry.ChunkId, .Offset = Entry.Offset, .Size = Entry.RequestBytes}; + }); + std::vector Chunks = + m_Workspaces.GetWorkspaceShareChunks(WorkspaceId, ShareId, Requests, GetSmallWorkerPool(EWorkloadType::Burst)); + if (Chunks.empty()) + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + for (const IoBuffer& Buffer : Chunks) + { + if (Buffer) + { + m_WorkspacesStats.WorkspaceShareChunkHitCount++; + } + else + { + m_WorkspacesStats.WorkspaceShareChunkMissCount++; + } + } + std::vector Response = BuildChunkBatchResponse(ChunkRequests.value(), Chunks); + if (!Response.empty()) + { + return ServerRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, Response); + } + return ServerRequest.WriteResponse(HttpResponseCode::InternalServerError, + HttpContentType::kText, + fmt::format("failed formatting response for batch of {} chunks", Chunks.size())); +} + +void +HttpWorkspacesService::EntriesRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& ShareId) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + std::string_view OpKey = ServerRequest.GetQueryParams().GetValue("opkey"sv); + if (!OpKey.empty() && OpKey != "file_manifest") + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + std::unordered_set WantedFieldNames; + if (auto FieldFilter = HttpServerRequest::Decode(ServerRequest.GetQueryParams().GetValue("fieldfilter")); !FieldFilter.empty()) + { + if (FieldFilter != "*") // Get all - empty FieldFilter equal getting all fields + { + ForEachStrTok(FieldFilter, ',', [&](std::string_view FieldName) { + WantedFieldNames.insert(std::string(FieldName)); + return true; + }); + } + } + + bool Refresh = false; + if (auto RefreshStr = ServerRequest.GetQueryParams().GetValue("refresh"); !RefreshStr.empty()) + { + Refresh = StrCaseCompare(std::string(RefreshStr).c_str(), "true") == 0; + } + + m_WorkspacesStats.WorkspaceShareEntriesReadCount++; + std::optional> Files = + m_Workspaces.GetWorkspaceShareFiles(WorkspaceId, ShareId, Refresh, GetSmallWorkerPool(EWorkloadType::Burst)); + if (!Files.has_value()) + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + const bool WantsAllFields = WantedFieldNames.empty(); + + const bool WantsIdField = WantsAllFields || WantedFieldNames.contains("id"); + const bool WantsClientPathField = WantsAllFields || WantedFieldNames.contains("clientpath"); + const bool WantsServerPathField = WantsAllFields || WantedFieldNames.contains("serverpath"); + + CbObjectWriter Response; + + if (OpKey.empty()) + { + Response.BeginArray("entries"sv); + Response.BeginObject(); + } + else + { + Response.BeginObject("entry"sv); + } + { + // Synthesize a fake op + Response << "key" + << "file_manifest"; + + Response.BeginArray("files"); + { + for (const Workspaces::ShareFile& Entry : Files.value()) + { + Response.BeginObject(); + { + if (WantsIdField) + { + Response << "id"sv << Entry.Id; + } + if (WantsServerPathField) + { + Response << "serverpath"sv << Entry.RelativePath; + } + if (WantsClientPathField) + { + Response << "clientpath"sv << Entry.RelativePath; + } + } + Response.EndObject(); + } + } + Response.EndArray(); + } + + if (OpKey.empty()) + { + Response.EndObject(); + Response.EndArray(); + } + else + { + Response.EndObject(); + } + + return ServerRequest.WriteResponse(HttpResponseCode::OK, Response.Save()); +} + +void +HttpWorkspacesService::ChunkRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& ShareId, const Oid& ChunkId) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + + uint64_t Offset = 0; + uint64_t Size = ~(0ull); + if (auto OffsetParm = ServerRequest.GetQueryParams().GetValue("offset"); OffsetParm.empty() == false) + { + if (auto OffsetVal = ParseInt(OffsetParm)) + { + Offset = OffsetVal.value(); + } + else + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid offset parameter '{}'", OffsetParm)); + } + } + + if (auto SizeParm = ServerRequest.GetQueryParams().GetValue("size"); SizeParm.empty() == false) + { + if (auto SizeVal = ParseInt(SizeParm)) + { + Size = SizeVal.value(); + } + else + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid size parameter '{}'", SizeParm)); + } + } + + std::vector Response = m_Workspaces.GetWorkspaceShareChunks( + WorkspaceId, + ShareId, + std::vector{Workspaces::ChunkRequest{.ChunkId = ChunkId, .Offset = Offset, .Size = Size}}, + GetSmallWorkerPool(EWorkloadType::Burst)); + if (!Response.empty() && Response[0]) + { + m_WorkspacesStats.WorkspaceShareChunkHitCount++; + if (Response[0].GetSize() == 0) + { + return ServerRequest.WriteResponse(HttpResponseCode::OK); + } + return ServerRequest.WriteResponse(HttpResponseCode::OK, Response[0].GetContentType(), Response); + } + m_WorkspacesStats.WorkspaceShareChunkMissCount++; + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); +} + +void +HttpWorkspacesService::ShareRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& InShareId) +{ + Oid ShareId = InShareId; + + HttpServerRequest& ServerRequest = Req.ServerRequest(); + switch (ServerRequest.RequestVerb()) + { + case HttpVerb::kPut: + { + std::filesystem::path SharePath = GetPathParameter(ServerRequest, "share_path"sv); + if (SharePath.empty()) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Invalid 'share_path' parameter"); + } + + if (ShareId == Oid::Zero) + { + // Synthesize Id + ShareId = Workspaces::PathToId(SharePath); + ZEN_INFO("Generated workspace id from path '{}': {}", SharePath, ShareId); + } + + std::string Alias = HttpServerRequest::Decode(ServerRequest.GetQueryParams().GetValue("alias"sv)); + if (!AsciiSet::HasOnly(Alias, Workspaces::ValidAliasCharactersSet)) + { + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid 'alias' parameter"); + } + + Workspaces::WorkspaceConfiguration Workspace = Workspaces::FindWorkspace(Log(), GetStatePath(), WorkspaceId); + if (Workspace.Id == Oid::Zero) + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Workspace '{}' does not exist", WorkspaceId)); + } + + if (!Workspace.AllowShareCreationFromHttp) + { + if (!MayChangeConfiguration(ServerRequest)) + { + return ServerRequest.WriteResponse( + HttpResponseCode::Unauthorized, + HttpContentType::kText, + fmt::format("Adding workspace share {} in workspace {} is not allowed", WorkspaceId, ShareId)); + } + } + + m_WorkspacesStats.WorkspaceShareWriteCount++; + + const Workspaces::WorkspaceShareConfiguration OldConfig = + Workspaces::FindWorkspaceShare(Log(), Workspace.RootPath, ShareId); + const Workspaces::WorkspaceShareConfiguration NewConfig = {.Id = ShareId, + .SharePath = SharePath, + .Alias = std::string(Alias)}; + + if (OldConfig.Id == ShareId && (OldConfig != NewConfig)) + { + return ServerRequest.WriteResponse( + HttpResponseCode::Conflict, + HttpContentType::kText, + fmt::format("Workspace share '{}' already exist in workspace '{}' with share path '{}' and alias '{}'", + ShareId, + WorkspaceId, + OldConfig.SharePath, + OldConfig.Alias)); + } + else if (OldConfig.Id == Oid::Zero) + { + if (Workspaces::WorkspaceShareConfiguration ConfigWithSamePath = + Workspaces::FindWorkspaceShare(Log(), Workspace.RootPath, SharePath); + ConfigWithSamePath.Id != Oid::Zero) + { + return ServerRequest.WriteResponse( + HttpResponseCode::Conflict, + HttpContentType::kText, + fmt::format("Workspace share '{}' already exist in workspace '{}' with same share path '{}' and alias '{}'", + ShareId, + WorkspaceId, + OldConfig.SharePath, + OldConfig.Alias)); + } + } + + if (!IsDir(Workspace.RootPath / NewConfig.SharePath)) + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("directory {} does not exist in workspace {} root '{}'", + NewConfig.SharePath, + WorkspaceId, + Workspace.RootPath)); + } + + bool Created = Workspaces::AddWorkspaceShare(Log(), Workspace.RootPath, NewConfig); + if (Created) + { + RefreshState(); + return ServerRequest.WriteResponse(HttpResponseCode::Created, HttpContentType::kText, fmt::format("{}", ShareId)); + } + return ServerRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, fmt::format("{}", ShareId)); + } + case HttpVerb::kGet: + { + if (WorkspaceId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); + } + if (ShareId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid share id '{}'", ShareId)); + } + + m_WorkspacesStats.WorkspaceShareReadCount++; + std::optional Config = + m_Workspaces.GetWorkspaceShareConfiguration(WorkspaceId, ShareId); + if (!Config) + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + + CbObjectWriter Response; + WriteWorkspaceShareConfig(Response, *Config); + return ServerRequest.WriteResponse(HttpResponseCode::OK, Response.Save()); + } + case HttpVerb::kDelete: + { + if (WorkspaceId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); + } + if (ShareId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid share id '{}'", ShareId)); + } + + Workspaces::WorkspaceConfiguration Workspace = Workspaces::FindWorkspace(Log(), GetStatePath(), WorkspaceId); + if (Workspace.Id == Oid::Zero) + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + + if (!Workspace.AllowShareCreationFromHttp) + { + if (!MayChangeConfiguration(ServerRequest)) + { + return ServerRequest.WriteResponse( + HttpResponseCode::Unauthorized, + HttpContentType::kText, + fmt::format("Removing workspace share {} in workspace {} is not allowed", WorkspaceId, ShareId)); + } + } + + m_WorkspacesStats.WorkspaceShareDeleteCount++; + bool Deleted = Workspaces::RemoveWorkspaceShare(Log(), Workspace.RootPath, ShareId); + if (Deleted) + { + RefreshState(); + return ServerRequest.WriteResponse(HttpResponseCode::OK); + } + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + } +} + +} // namespace zen diff --git a/src/zenserver/storage/workspaces/httpworkspaces.h b/src/zenserver/storage/workspaces/httpworkspaces.h new file mode 100644 index 000000000..89a8e8bdc --- /dev/null +++ b/src/zenserver/storage/workspaces/httpworkspaces.h @@ -0,0 +1,97 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include +#include +#include + +namespace zen { + +class Workspaces; + +struct WorkspacesServeConfig +{ + std::filesystem::path SystemRootDir; + bool AllowConfigurationChanges = false; +}; + +class HttpWorkspacesService final : public HttpService, public IHttpStatusProvider, public IHttpStatsProvider +{ +public: + HttpWorkspacesService(HttpStatusService& StatusService, + HttpStatsService& StatsService, + const WorkspacesServeConfig& Cfg, + Workspaces& Workspaces); + virtual ~HttpWorkspacesService(); + + virtual const char* BaseUri() const override; + virtual void HandleRequest(HttpServerRequest& Request) override; + + virtual void HandleStatsRequest(HttpServerRequest& Request) override; + virtual void HandleStatusRequest(HttpServerRequest& Request) override; + +private: + struct WorkspacesStats + { + std::atomic_uint64_t WorkspaceReadCount{}; + std::atomic_uint64_t WorkspaceWriteCount{}; + std::atomic_uint64_t WorkspaceDeleteCount{}; + std::atomic_uint64_t WorkspaceShareReadCount{}; + std::atomic_uint64_t WorkspaceShareWriteCount{}; + std::atomic_uint64_t WorkspaceShareDeleteCount{}; + std::atomic_uint64_t WorkspaceShareFilesReadCount{}; + std::atomic_uint64_t WorkspaceShareEntriesReadCount{}; + std::atomic_uint64_t WorkspaceShareBatchReadCount{}; + std::atomic_uint64_t WorkspaceShareChunkHitCount{}; + std::atomic_uint64_t WorkspaceShareChunkMissCount{}; + std::atomic_uint64_t RequestCount{}; + std::atomic_uint64_t BadRequestCount{}; + }; + + inline LoggerRef Log() { return m_Log; } + + LoggerRef m_Log; + + void Initialize(); + std::filesystem::path GetStatePath() const; + void RefreshState(); + // void WriteState(); + + bool MayChangeConfiguration(const HttpServerRequest& Req) const; + + void WorkspacesRequest(HttpRouterRequest& Req); + void RefreshRequest(HttpRouterRequest& Req); + void FilesRequest(HttpRouterRequest& Req); + void ChunkInfoRequest(HttpRouterRequest& Req); + void BatchRequest(HttpRouterRequest& Req); + void EntriesRequest(HttpRouterRequest& Req); + void ChunkRequest(HttpRouterRequest& Req); + void ShareRequest(HttpRouterRequest& Req); + void WorkspaceRequest(HttpRouterRequest& Req); + + void ShareAliasFilesRequest(HttpRouterRequest& Req); + void ShareAliasChunkInfoRequest(HttpRouterRequest& Req); + void ShareAliasBatchRequest(HttpRouterRequest& Req); + void ShareAliasEntriesRequest(HttpRouterRequest& Req); + void ShareAliasChunkRequest(HttpRouterRequest& Req); + void ShareAliasRequest(HttpRouterRequest& Req); + + void FilesRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& ShareId); + void ChunkInfoRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& ShareId, const Oid& ChunkId); + void BatchRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& ShareId); + void EntriesRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& ShareId); + void ChunkRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& ShareId, const Oid& ChunkId); + void ShareRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& InShareId); + + HttpStatusService& m_StatusService; + HttpStatsService& m_StatsService; + const WorkspacesServeConfig m_Config; + HttpRequestRouter m_Router; + Workspaces& m_Workspaces; + WorkspacesStats m_WorkspacesStats; + metrics::OperationTiming m_HttpRequests; +}; + +} // namespace zen diff --git a/src/zenserver/storage/zenstorageserver.cpp b/src/zenserver/storage/zenstorageserver.cpp new file mode 100644 index 000000000..73896512d --- /dev/null +++ b/src/zenserver/storage/zenstorageserver.cpp @@ -0,0 +1,961 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "zenstorageserver.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if ZEN_PLATFORM_WINDOWS +# include +#endif + +#if ZEN_PLATFORM_LINUX +# include +#endif + +#if ZEN_PLATFORM_MAC +# include +#endif + +ZEN_THIRD_PARTY_INCLUDES_START +#include +#include +ZEN_THIRD_PARTY_INCLUDES_END + +#include +#include + +////////////////////////////////////////////////////////////////////////// + +#include "diag/logging.h" +#include "storageconfig.h" + +#include + +namespace zen { + +namespace utils { + asio::error_code ResolveHostname(asio::io_context& Ctx, + std::string_view Host, + std::string_view DefaultPort, + std::vector& OutEndpoints) + { + std::string_view Port = DefaultPort; + + if (const size_t Idx = Host.find(":"); Idx != std::string_view::npos) + { + Port = Host.substr(Idx + 1); + Host = Host.substr(0, Idx); + } + + asio::ip::tcp::resolver Resolver(Ctx); + + asio::error_code ErrorCode; + asio::ip::tcp::resolver::results_type Endpoints = Resolver.resolve(Host, Port, ErrorCode); + + if (!ErrorCode) + { + for (const asio::ip::tcp::endpoint Ep : Endpoints) + { + OutEndpoints.push_back(fmt::format("http://{}:{}", Ep.address().to_string(), Ep.port())); + } + } + + return ErrorCode; + } +} // namespace utils + +using namespace std::literals; + +ZenStorageServer::ZenStorageServer() +{ +} + +ZenStorageServer::~ZenStorageServer() +{ +} + +int +ZenStorageServer::Initialize(const ZenStorageServerOptions& ServerOptions, ZenServerState::ZenServerEntry* ServerEntry) +{ + ZEN_TRACE_CPU("ZenStorageServer::Initialize"); + ZEN_MEMSCOPE(GetZenserverTag()); + + const int EffectiveBasePort = ZenServerBase::Initialize(ServerOptions, ServerEntry); + if (EffectiveBasePort < 0) + { + return EffectiveBasePort; + } + + m_DebugOptionForcedCrash = ServerOptions.ShouldCrash; + m_StartupScrubOptions = ServerOptions.ScrubOptions; + + InitializeState(ServerOptions); + InitializeServices(ServerOptions); + RegisterServices(); + + ZenServerBase::Finalize(); + + return EffectiveBasePort; +} + +void +ZenStorageServer::RegisterServices() +{ + m_Http->RegisterService(*m_AuthService); + m_Http->RegisterService(m_StatsService); + m_Http->RegisterService(m_TestService); // NOTE: this is intentionally not limited to test mode as it's useful for diagnostics + +#if ZEN_WITH_TESTS + m_Http->RegisterService(m_TestingService); +#endif + + if (m_StructuredCacheService) + { + m_Http->RegisterService(*m_StructuredCacheService); + } + + if (m_UpstreamService) + { + m_Http->RegisterService(*m_UpstreamService); + } + + if (m_HttpProjectService) + { + m_Http->RegisterService(*m_HttpProjectService); + } + + if (m_HttpWorkspacesService) + { + m_Http->RegisterService(*m_HttpWorkspacesService); + } + + m_FrontendService = std::make_unique(m_ContentRoot, m_StatusService); + + if (m_FrontendService) + { + m_Http->RegisterService(*m_FrontendService); + } + + if (m_ObjStoreService) + { + m_Http->RegisterService(*m_ObjStoreService); + } + + if (m_BuildStoreService) + { + m_Http->RegisterService(*m_BuildStoreService); + } + +#if ZEN_WITH_VFS + m_Http->RegisterService(*m_VfsService); +#endif // ZEN_WITH_VFS + + m_Http->RegisterService(*m_AdminService); +} + +void +ZenStorageServer::InitializeServices(const ZenStorageServerOptions& ServerOptions) +{ + InitializeAuthentication(ServerOptions); + + ZEN_INFO("initializing storage"); + + CidStoreConfiguration Config; + Config.RootDirectory = m_DataRoot / "cas"; + + m_CidStore = std::make_unique(m_GcManager); + m_CidStore->Initialize(Config); + + ZEN_INFO("instantiating project service"); + + m_JobQueue = MakeJobQueue(8, "bgjobs"); + + m_ProjectStore = new ProjectStore(*m_CidStore, m_DataRoot / "projects", m_GcManager, ProjectStore::Configuration{}); + m_HttpProjectService.reset( + new HttpProjectService{*m_CidStore, m_ProjectStore, m_StatusService, m_StatsService, *m_AuthMgr, *m_OpenProcessCache, *m_JobQueue}); + + if (ServerOptions.WorksSpacesConfig.Enabled) + { + m_Workspaces.reset(new Workspaces()); + m_HttpWorkspacesService.reset( + new HttpWorkspacesService(m_StatusService, + m_StatsService, + {.SystemRootDir = ServerOptions.SystemRootDir, + .AllowConfigurationChanges = ServerOptions.WorksSpacesConfig.AllowConfigurationChanges}, + *m_Workspaces)); + } + + if (ServerOptions.BuildStoreConfig.Enabled) + { + CidStoreConfiguration BuildCidConfig; + BuildCidConfig.RootDirectory = m_DataRoot / "builds_cas"; + m_BuildCidStore = std::make_unique(m_GcManager); + m_BuildCidStore->Initialize(BuildCidConfig); + + BuildStoreConfig BuildsCfg; + BuildsCfg.RootDirectory = m_DataRoot / "builds"; + BuildsCfg.MaxDiskSpaceLimit = ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit; + m_BuildStore = std::make_unique(std::move(BuildsCfg), m_GcManager, *m_BuildCidStore); + } + + if (ServerOptions.StructuredCacheConfig.Enabled) + { + InitializeStructuredCache(ServerOptions); + } + else + { + ZEN_INFO("NOT instantiating structured cache service"); + } + + if (ServerOptions.ObjectStoreEnabled) + { + ObjectStoreConfig ObjCfg; + ObjCfg.RootDirectory = m_DataRoot / "obj"; + + for (const auto& Bucket : ServerOptions.ObjectStoreConfig.Buckets) + { + ObjectStoreConfig::BucketConfig NewBucket{.Name = Bucket.Name}; + NewBucket.Directory = Bucket.Directory.empty() ? (ObjCfg.RootDirectory / Bucket.Name) : Bucket.Directory; + ObjCfg.Buckets.push_back(std::move(NewBucket)); + } + + m_ObjStoreService = std::make_unique(m_StatusService, std::move(ObjCfg)); + } + + if (ServerOptions.BuildStoreConfig.Enabled) + { + m_BuildStoreService = std::make_unique(m_StatusService, m_StatsService, *m_BuildStore); + } + +#if ZEN_WITH_VFS + m_VfsServiceImpl = std::make_unique(); + m_VfsServiceImpl->AddService(Ref(m_ProjectStore)); + m_VfsServiceImpl->AddService(Ref(m_CacheStore)); + + m_VfsService = std::make_unique(m_StatusService, m_VfsServiceImpl.get()); +#endif // ZEN_WITH_VFS + + ZEN_INFO("initializing GC, enabled '{}', interval {}, lightweight interval {}", + ServerOptions.GcConfig.Enabled, + NiceTimeSpanMs(ServerOptions.GcConfig.IntervalSeconds * 1000ull), + NiceTimeSpanMs(ServerOptions.GcConfig.LightweightIntervalSeconds * 1000ull)); + + GcSchedulerConfig GcConfig{.RootDirectory = m_DataRoot / "gc", + .MonitorInterval = std::chrono::seconds(ServerOptions.GcConfig.MonitorIntervalSeconds), + .Interval = std::chrono::seconds(ServerOptions.GcConfig.IntervalSeconds), + .MaxCacheDuration = std::chrono::seconds(ServerOptions.GcConfig.Cache.MaxDurationSeconds), + .MaxProjectStoreDuration = std::chrono::seconds(ServerOptions.GcConfig.ProjectStore.MaxDurationSeconds), + .MaxBuildStoreDuration = std::chrono::seconds(ServerOptions.GcConfig.BuildStore.MaxDurationSeconds), + .CollectSmallObjects = ServerOptions.GcConfig.CollectSmallObjects, + .Enabled = ServerOptions.GcConfig.Enabled, + .DiskReserveSize = ServerOptions.GcConfig.DiskReserveSize, + .DiskSizeSoftLimit = ServerOptions.GcConfig.DiskSizeSoftLimit, + .MinimumFreeDiskSpaceToAllowWrites = ServerOptions.GcConfig.MinimumFreeDiskSpaceToAllowWrites, + .LightweightInterval = std::chrono::seconds(ServerOptions.GcConfig.LightweightIntervalSeconds), + .UseGCVersion = ServerOptions.GcConfig.UseGCV2 ? GcVersion::kV2 : GcVersion::kV1_Deprecated, + .CompactBlockUsageThresholdPercent = ServerOptions.GcConfig.CompactBlockUsageThresholdPercent, + .Verbose = ServerOptions.GcConfig.Verbose, + .SingleThreaded = ServerOptions.GcConfig.SingleThreaded, + .AttachmentPassCount = ServerOptions.GcConfig.AttachmentPassCount}; + m_GcScheduler.Initialize(GcConfig); + + // Create and register admin interface last to make sure all is properly initialized + m_AdminService = std::make_unique( + m_GcScheduler, + *m_JobQueue, + m_CacheStore.Get(), + [this]() { Flush(); }, + HttpAdminService::LogPaths{.AbsLogPath = ServerOptions.AbsLogFile, + .HttpLogPath = ServerOptions.DataDir / "logs" / "http.log", + .CacheLogPath = ServerOptions.DataDir / "logs" / "z$.log"}, + ServerOptions); +} + +void +ZenStorageServer::InitializeAuthentication(const ZenStorageServerOptions& ServerOptions) +{ + // Setup authentication manager + { + ZEN_TRACE_CPU("ZenStorageServer::InitAuth"); + std::string EncryptionKey = ServerOptions.EncryptionKey; + + if (EncryptionKey.empty()) + { + EncryptionKey = "abcdefghijklmnopqrstuvxyz0123456"; + + if (ServerOptions.IsDedicated) + { + ZEN_WARN("Using default encryption key for authentication state"); + } + } + + std::string EncryptionIV = ServerOptions.EncryptionIV; + + if (EncryptionIV.empty()) + { + EncryptionIV = "0123456789abcdef"; + + if (ServerOptions.IsDedicated) + { + ZEN_WARN("Using default encryption initialization vector for authentication state"); + } + } + + m_AuthMgr = AuthMgr::Create({.RootDirectory = m_DataRoot / "auth", + .EncryptionKey = AesKey256Bit::FromString(EncryptionKey), + .EncryptionIV = AesIV128Bit::FromString(EncryptionIV)}); + + for (const ZenOpenIdProviderConfig& OpenIdProvider : ServerOptions.AuthConfig.OpenIdProviders) + { + m_AuthMgr->AddOpenIdProvider({.Name = OpenIdProvider.Name, .Url = OpenIdProvider.Url, .ClientId = OpenIdProvider.ClientId}); + } + } + + m_AuthService = std::make_unique(*m_AuthMgr); +} + +void +ZenStorageServer::InitializeState(const ZenStorageServerOptions& ServerOptions) +{ + ZEN_TRACE_CPU("ZenStorageServer::InitializeState"); + + // Check root manifest to deal with schema versioning + + bool WipeState = false; + std::string WipeReason = "Unspecified"; + + if (ServerOptions.IsCleanStart) + { + WipeState = true; + WipeReason = "clean start requested"; + } + + bool UpdateManifest = false; + std::filesystem::path ManifestPath = m_DataRoot / "root_manifest"; + Oid StateId = Oid::Zero; + DateTime CreatedWhen{0}; + + if (!WipeState) + { + FileContents ManifestData = ReadFile(ManifestPath); + + if (ManifestData.ErrorCode) + { + if (ServerOptions.IsFirstRun) + { + ZEN_INFO("Initializing state at '{}'", m_DataRoot); + + UpdateManifest = true; + } + else + { + WipeState = true; + WipeReason = fmt::format("No manifest present at '{}'", ManifestPath); + } + } + else + { + IoBuffer Manifest = ManifestData.Flatten(); + + if (CbValidateError ValidationResult = ValidateCompactBinary(Manifest, CbValidateMode::All); + ValidationResult != CbValidateError::None) + { + ZEN_WARN("Manifest validation failed: {}, state will be wiped", zen::ToString(ValidationResult)); + + WipeState = true; + WipeReason = fmt::format("Validation of manifest at '{}' failed: {}", ManifestPath, zen::ToString(ValidationResult)); + } + else + { + m_RootManifest = LoadCompactBinaryObject(Manifest); + + const int32_t ManifestVersion = m_RootManifest["schema_version"].AsInt32(0); + StateId = m_RootManifest["state_id"].AsObjectId(); + CreatedWhen = m_RootManifest["created"].AsDateTime(); + + if (ManifestVersion != ZEN_CFG_SCHEMA_VERSION) + { + std::filesystem::path ManifestSkipSchemaChangePath = m_DataRoot / "root_manifest.ignore_schema_mismatch"; + if (ManifestVersion != 0 && IsFile(ManifestSkipSchemaChangePath)) + { + ZEN_INFO( + "Schema version {} found in '{}' does not match {}, ignoring mismatch due to existance of '{}' and updating " + "schema version", + ManifestVersion, + ManifestPath, + ZEN_CFG_SCHEMA_VERSION, + ManifestSkipSchemaChangePath); + UpdateManifest = true; + } + else + { + WipeState = true; + WipeReason = + fmt::format("Manifest schema version: {}, differs from required: {}", ManifestVersion, ZEN_CFG_SCHEMA_VERSION); + } + } + } + } + } + + if (StateId == Oid::Zero) + { + StateId = Oid::NewOid(); + UpdateManifest = true; + } + + const DateTime Now = DateTime::Now(); + + if (CreatedWhen.GetTicks() == 0) + { + CreatedWhen = Now; + UpdateManifest = true; + } + + // Handle any state wipe + + if (WipeState) + { + ZEN_WARN("Wiping state at '{}' - reason: '{}'", m_DataRoot, WipeReason); + + std::error_code Ec; + for (const std::filesystem::directory_entry& DirEntry : std::filesystem::directory_iterator{m_DataRoot, Ec}) + { + if (DirEntry.is_directory() && (DirEntry.path().filename() != "logs")) + { + ZEN_INFO("Deleting '{}'", DirEntry.path()); + + DeleteDirectories(DirEntry.path(), Ec); + + if (Ec) + { + ZEN_WARN("Delete of '{}' returned error: '{}'", DirEntry.path(), Ec.message()); + } + } + } + + ZEN_INFO("Wiped all directories in data root"); + + UpdateManifest = true; + } + + // Write manifest + + { + CbObjectWriter Cbo; + Cbo << "schema_version" << ZEN_CFG_SCHEMA_VERSION << "created" << CreatedWhen << "updated" << Now << "state_id" << StateId; + + m_RootManifest = Cbo.Save(); + + if (UpdateManifest) + { + TemporaryFile::SafeWriteFile(ManifestPath, m_RootManifest.GetBuffer().GetView()); + } + + if (!ServerOptions.IsTest) + { + try + { + EmitCentralManifest(ServerOptions.SystemRootDir, StateId, m_RootManifest, ManifestPath); + } + catch (const std::exception& Ex) + { + ZEN_WARN("Unable to emit central manifest: ", Ex.what()); + } + } + } + + // Write state marker + + { + std::filesystem::path StateMarkerPath = m_DataRoot / "state_marker"; + static const std::string_view StateMarkerContent = "deleting this file will cause " ZEN_APP_NAME " to exit"sv; + WriteFile(StateMarkerPath, IoBuffer(IoBuffer::Wrap, StateMarkerContent.data(), StateMarkerContent.size())); + + EnqueueStateMarkerTimer(); + } + + EnqueueStateExitFlagTimer(); +} + +void +ZenStorageServer::InitializeStructuredCache(const ZenStorageServerOptions& ServerOptions) +{ + ZEN_TRACE_CPU("ZenStorageServer::InitializeStructuredCache"); + + using namespace std::literals; + + ZEN_INFO("instantiating structured cache service"); + ZenCacheStore::Configuration Config; + Config.AllowAutomaticCreationOfNamespaces = true; + Config.Logging = {.EnableWriteLog = ServerOptions.StructuredCacheConfig.WriteLogEnabled, + .EnableAccessLog = ServerOptions.StructuredCacheConfig.AccessLogEnabled}; + + for (const auto& It : ServerOptions.StructuredCacheConfig.PerBucketConfigs) + { + const std::string& BucketName = It.first; + const ZenStructuredCacheBucketConfig& ZenBucketConfig = It.second; + ZenCacheDiskLayer::BucketConfiguration BucketConfig = {.MaxBlockSize = ZenBucketConfig.MaxBlockSize, + .PayloadAlignment = ZenBucketConfig.PayloadAlignment, + .MemCacheSizeThreshold = ZenBucketConfig.MemCacheSizeThreshold, + .LargeObjectThreshold = ZenBucketConfig.LargeObjectThreshold, + .LimitOverwrites = ZenBucketConfig.LimitOverwrites}; + Config.NamespaceConfig.DiskLayerConfig.BucketConfigMap.insert_or_assign(BucketName, BucketConfig); + } + Config.NamespaceConfig.DiskLayerConfig.BucketConfig.MaxBlockSize = ServerOptions.StructuredCacheConfig.BucketConfig.MaxBlockSize, + Config.NamespaceConfig.DiskLayerConfig.BucketConfig.PayloadAlignment = + ServerOptions.StructuredCacheConfig.BucketConfig.PayloadAlignment, + Config.NamespaceConfig.DiskLayerConfig.BucketConfig.MemCacheSizeThreshold = + ServerOptions.StructuredCacheConfig.BucketConfig.MemCacheSizeThreshold, + Config.NamespaceConfig.DiskLayerConfig.BucketConfig.LargeObjectThreshold = + ServerOptions.StructuredCacheConfig.BucketConfig.LargeObjectThreshold, + Config.NamespaceConfig.DiskLayerConfig.BucketConfig.LimitOverwrites = ServerOptions.StructuredCacheConfig.BucketConfig.LimitOverwrites; + Config.NamespaceConfig.DiskLayerConfig.MemCacheTargetFootprintBytes = ServerOptions.StructuredCacheConfig.MemTargetFootprintBytes; + Config.NamespaceConfig.DiskLayerConfig.MemCacheTrimIntervalSeconds = ServerOptions.StructuredCacheConfig.MemTrimIntervalSeconds; + Config.NamespaceConfig.DiskLayerConfig.MemCacheMaxAgeSeconds = ServerOptions.StructuredCacheConfig.MemMaxAgeSeconds; + + if (ServerOptions.IsDedicated) + { + Config.NamespaceConfig.DiskLayerConfig.BucketConfig.LargeObjectThreshold = 128 * 1024 * 1024; + } + + m_CacheStore = new ZenCacheStore(m_GcManager, *m_JobQueue, m_DataRoot / "cache", Config, m_GcManager.GetDiskWriteBlocker()); + m_OpenProcessCache = std::make_unique(); + + const ZenUpstreamCacheConfig& UpstreamConfig = ServerOptions.UpstreamCacheConfig; + + UpstreamCacheOptions UpstreamOptions; + UpstreamOptions.ReadUpstream = (uint8_t(ServerOptions.UpstreamCacheConfig.CachePolicy) & uint8_t(UpstreamCachePolicy::Read)) != 0; + UpstreamOptions.WriteUpstream = (uint8_t(ServerOptions.UpstreamCacheConfig.CachePolicy) & uint8_t(UpstreamCachePolicy::Write)) != 0; + + if (UpstreamConfig.UpstreamThreadCount < 32) + { + UpstreamOptions.ThreadCount = static_cast(UpstreamConfig.UpstreamThreadCount); + } + + m_UpstreamCache = CreateUpstreamCache(UpstreamOptions, *m_CacheStore, *m_CidStore); + m_UpstreamService = std::make_unique(*m_UpstreamCache, *m_AuthMgr); + m_UpstreamCache->Initialize(); + + if (ServerOptions.UpstreamCacheConfig.CachePolicy != UpstreamCachePolicy::Disabled) + { + // Zen upstream + { + std::vector ZenUrls = UpstreamConfig.ZenConfig.Urls; + if (!UpstreamConfig.ZenConfig.Dns.empty()) + { + for (const std::string& Dns : UpstreamConfig.ZenConfig.Dns) + { + if (!Dns.empty()) + { + const asio::error_code Err = utils::ResolveHostname(m_IoContext, Dns, "8558"sv, ZenUrls); + if (Err) + { + ZEN_ERROR("resolve of '{}' FAILED, reason '{}'", Dns, Err.message()); + } + } + } + } + + std::erase_if(ZenUrls, [](const auto& Url) { return Url.empty(); }); + + if (!ZenUrls.empty()) + { + const auto ZenEndpointName = UpstreamConfig.ZenConfig.Name.empty() ? "Zen"sv : UpstreamConfig.ZenConfig.Name; + + std::unique_ptr ZenEndpoint = UpstreamEndpoint::CreateZenEndpoint( + {.Name = ZenEndpointName, + .Urls = ZenUrls, + .ConnectTimeout = std::chrono::milliseconds(UpstreamConfig.ConnectTimeoutMilliseconds), + .Timeout = std::chrono::milliseconds(UpstreamConfig.TimeoutMilliseconds)}); + + m_UpstreamCache->RegisterEndpoint(std::move(ZenEndpoint)); + } + } + + // Jupiter upstream + if (UpstreamConfig.JupiterConfig.Url.empty() == false) + { + std::string_view EndpointName = UpstreamConfig.JupiterConfig.Name.empty() ? "Jupiter"sv : UpstreamConfig.JupiterConfig.Name; + + auto Options = JupiterClientOptions{.Name = EndpointName, + .ServiceUrl = UpstreamConfig.JupiterConfig.Url, + .DdcNamespace = UpstreamConfig.JupiterConfig.DdcNamespace, + .BlobStoreNamespace = UpstreamConfig.JupiterConfig.Namespace, + .ConnectTimeout = std::chrono::milliseconds(UpstreamConfig.ConnectTimeoutMilliseconds), + .Timeout = std::chrono::milliseconds(UpstreamConfig.TimeoutMilliseconds)}; + + auto AuthConfig = UpstreamAuthConfig{.OAuthUrl = UpstreamConfig.JupiterConfig.OAuthUrl, + .OAuthClientId = UpstreamConfig.JupiterConfig.OAuthClientId, + .OAuthClientSecret = UpstreamConfig.JupiterConfig.OAuthClientSecret, + .OpenIdProvider = UpstreamConfig.JupiterConfig.OpenIdProvider, + .AccessToken = UpstreamConfig.JupiterConfig.AccessToken}; + + std::unique_ptr JupiterEndpoint = UpstreamEndpoint::CreateJupiterEndpoint(Options, AuthConfig, *m_AuthMgr); + + m_UpstreamCache->RegisterEndpoint(std::move(JupiterEndpoint)); + } + } + + m_StructuredCacheService = std::make_unique(*m_CacheStore, + *m_CidStore, + m_StatsService, + m_StatusService, + *m_UpstreamCache, + m_GcManager.GetDiskWriteBlocker(), + *m_OpenProcessCache); + + m_StatsReporter.AddProvider(m_CacheStore.Get()); + m_StatsReporter.AddProvider(m_CidStore.get()); + m_StatsReporter.AddProvider(m_BuildCidStore.get()); +} + +void +ZenStorageServer::Run() +{ + if (m_ProcessMonitor.IsActive()) + { + CheckOwnerPid(); + } + + if (!m_TestMode) + { + ZEN_INFO( + "__________ _________ __ \n" + "\\____ /____ ____ / _____// |_ ___________ ____ \n" + " / // __ \\ / \\ \\_____ \\\\ __\\/ _ \\_ __ \\_/ __ \\ \n" + " / /\\ ___/| | \\ / \\| | ( <_> ) | \\/\\ ___/ \n" + "/_______ \\___ >___| / /_______ /|__| \\____/|__| \\___ >\n" + " \\/ \\/ \\/ \\/ \\/ \n"); + } + + ZEN_INFO(ZEN_APP_NAME " now running (pid: {})", GetCurrentProcessId()); + +#if ZEN_PLATFORM_WINDOWS + if (zen::windows::IsRunningOnWine()) + { + ZEN_INFO("detected Wine session - " ZEN_APP_NAME " is not formally tested on Wine and may therefore not work or perform well"); + } +#endif + +#if ZEN_USE_SENTRY + ZEN_INFO("sentry crash handler {}", m_UseSentry ? "ENABLED" : "DISABLED"); + if (m_UseSentry) + { + SentryIntegration::ClearCaches(); + } +#endif + + if (m_DebugOptionForcedCrash) + { + ZEN_DEBUG_BREAK(); + } + + const bool IsInteractiveMode = IsInteractiveSession() && !m_TestMode; + + SetNewState(kRunning); + + OnReady(); + + if (!m_StartupScrubOptions.empty()) + { + using namespace std::literals; + + ZEN_INFO("triggering scrub with settings: '{}'", m_StartupScrubOptions); + + bool DoScrub = true; + bool DoWait = false; + GcScheduler::TriggerScrubParams ScrubParams; + + ForEachStrTok(m_StartupScrubOptions, ',', [&](std::string_view Token) { + if (Token == "nocas"sv) + { + ScrubParams.SkipCas = true; + } + else if (Token == "nodelete"sv) + { + ScrubParams.SkipDelete = true; + } + else if (Token == "nogc"sv) + { + ScrubParams.SkipGc = true; + } + else if (Token == "no"sv) + { + DoScrub = false; + } + else if (Token == "wait"sv) + { + DoWait = true; + } + return true; + }); + + if (DoScrub) + { + m_GcScheduler.TriggerScrub(ScrubParams); + + if (DoWait) + { + auto State = m_GcScheduler.Status(); + + while ((State != GcSchedulerStatus::kRunning) && (State != GcSchedulerStatus::kStopped)) + { + Sleep(500); + + State = m_GcScheduler.Status(); + } + + ZEN_INFO("waiting for Scrub/GC to complete..."); + + while (State == GcSchedulerStatus::kRunning) + { + Sleep(500); + + State = m_GcScheduler.Status(); + } + + ZEN_INFO("Scrub/GC completed"); + } + } + } + + if (m_IsPowerCycle) + { + ZEN_INFO("Power cycle mode enabled -- shutting down"); + RequestExit(0); + } + + m_Http->Run(IsInteractiveMode); + + SetNewState(kShuttingDown); + + ZEN_INFO(ZEN_APP_NAME " exiting"); +} + +void +ZenStorageServer::Cleanup() +{ + ZEN_TRACE_CPU("ZenStorageServer::Cleanup"); + ZEN_INFO(ZEN_APP_NAME " cleaning up"); + try + { + m_IoContext.stop(); + if (m_IoRunner.joinable()) + { + m_IoRunner.join(); + } + + if (m_Http) + { + m_Http->Close(); + } + + if (m_JobQueue) + { + m_JobQueue->Stop(); + } + + m_StatsReporter.Shutdown(); + m_GcScheduler.Shutdown(); + + Flush(); + + m_AdminService.reset(); + m_VfsService.reset(); + m_VfsServiceImpl.reset(); + m_ObjStoreService.reset(); + m_FrontendService.reset(); + + m_BuildStoreService.reset(); + m_BuildStore = {}; + m_BuildCidStore.reset(); + + m_StructuredCacheService.reset(); + m_UpstreamService.reset(); + m_UpstreamCache.reset(); + m_CacheStore = {}; + m_OpenProcessCache.reset(); + + m_HttpWorkspacesService.reset(); + m_Workspaces.reset(); + m_HttpProjectService.reset(); + m_ProjectStore = {}; + m_CidStore.reset(); + m_AuthService.reset(); + m_AuthMgr.reset(); + m_Http = {}; + + ShutdownWorkerPools(); + + m_JobQueue.reset(); + } + catch (const std::exception& Ex) + { + ZEN_ERROR("exception thrown during Cleanup() in {}: '{}'", ZEN_APP_NAME, Ex.what()); + } +} + +void +ZenStorageServer::EnqueueStateMarkerTimer() +{ + ZEN_MEMSCOPE(GetZenserverTag()); + m_StateMarkerTimer.expires_after(std::chrono::seconds(5)); + m_StateMarkerTimer.async_wait([this](const asio::error_code&) { CheckStateMarker(); }); + EnsureIoRunner(); +} + +void +ZenStorageServer::CheckStateMarker() +{ + ZEN_MEMSCOPE(GetZenserverTag()); + std::filesystem::path StateMarkerPath = m_DataRoot / "state_marker"; + try + { + if (!IsFile(StateMarkerPath)) + { + ZEN_WARN("state marker at {} has been deleted, exiting", StateMarkerPath); + RequestExit(1); + return; + } + } + catch (const std::exception& Ex) + { + ZEN_WARN("state marker at {} could not be checked, reason: '{}'", StateMarkerPath, Ex.what()); + RequestExit(1); + return; + } + EnqueueStateMarkerTimer(); +} + +void +ZenStorageServer::Flush() +{ + ZEN_TRACE_CPU("ZenStorageServer::Flush"); + + if (m_CidStore) + m_CidStore->Flush(); + + if (m_StructuredCacheService) + m_StructuredCacheService->Flush(); + + if (m_ProjectStore) + m_ProjectStore->Flush(); + + if (m_BuildCidStore) + m_BuildCidStore->Flush(); +} + +////////////////////////////////////////////////////////////////////////// + +ZenStorageServerMain::ZenStorageServerMain(ZenStorageServerOptions& ServerOptions) +: ZenServerMain(ServerOptions) +, m_ServerOptions(ServerOptions) +{ +} + +void +ZenStorageServerMain::DoRun(ZenServerState::ZenServerEntry* Entry) +{ + ZenStorageServer Server; + Server.SetDataRoot(m_ServerOptions.DataDir); + Server.SetContentRoot(m_ServerOptions.ContentDir); + Server.SetTestMode(m_ServerOptions.IsTest); + Server.SetDedicatedMode(m_ServerOptions.IsDedicated); + + auto ServerCleanup = MakeGuard([&Server] { Server.Cleanup(); }); + + int EffectiveBasePort = Server.Initialize(m_ServerOptions, Entry); + if (EffectiveBasePort == -1) + { + // Server.Initialize has already logged what the issue is - just exit with failure code here. + std::exit(1); + } + + Entry->EffectiveListenPort = uint16_t(EffectiveBasePort); + if (EffectiveBasePort != m_ServerOptions.BasePort) + { + ZEN_INFO(ZEN_APP_NAME " - relocated to base port {}", EffectiveBasePort); + m_ServerOptions.BasePort = EffectiveBasePort; + } + + std::unique_ptr ShutdownThread; + std::unique_ptr ShutdownEvent; + + ExtendableStringBuilder<64> ShutdownEventName; + ShutdownEventName << "Zen_" << m_ServerOptions.BasePort << "_Shutdown"; + ShutdownEvent.reset(new NamedEvent{ShutdownEventName}); + + // Monitor shutdown signals + + ShutdownThread.reset(new std::thread{[&] { + SetCurrentThreadName("shutdown_monitor"); + + ZEN_INFO("shutdown monitor thread waiting for shutdown signal '{}' for process {}", ShutdownEventName, zen::GetCurrentProcessId()); + + if (ShutdownEvent->Wait()) + { + ZEN_INFO("shutdown signal for pid {} received", zen::GetCurrentProcessId()); + Server.RequestExit(0); + } + else + { + ZEN_INFO("shutdown signal wait() failed"); + } + }}); + + auto CleanupShutdown = MakeGuard([&ShutdownEvent, &ShutdownThread] { + ReportServiceStatus(ServiceStatus::Stopping); + + if (ShutdownEvent) + { + ShutdownEvent->Set(); + } + if (ShutdownThread && ShutdownThread->joinable()) + { + ShutdownThread->join(); + } + }); + + // If we have a parent process, establish the mechanisms we need + // to be able to communicate readiness with the parent + + Server.SetIsReadyFunc([&] { + std::error_code Ec; + m_LockFile.Update(MakeLockData(true), Ec); + ReportServiceStatus(ServiceStatus::Running); + NotifyReady(); + }); + + Server.Run(); +} + +} // namespace zen diff --git a/src/zenserver/storage/zenstorageserver.h b/src/zenserver/storage/zenstorageserver.h new file mode 100644 index 000000000..e4c31399d --- /dev/null +++ b/src/zenserver/storage/zenstorageserver.h @@ -0,0 +1,113 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zenserver.h" + +#include +#include +#include +#include +#include +#include + +#include "admin/admin.h" +#include "buildstore/httpbuildstore.h" +#include "cache/httpstructuredcache.h" +#include "diag/diagsvcs.h" +#include "frontend/frontend.h" +#include "objectstore/objectstore.h" +#include "projectstore/httpprojectstore.h" +#include "stats/statsreporter.h" +#include "upstream/upstream.h" +#include "vfs/vfsservice.h" +#include "workspaces/httpworkspaces.h" + +namespace zen { + +class ZenStorageServer : public ZenServerBase +{ + ZenStorageServer& operator=(ZenStorageServer&&) = delete; + ZenStorageServer(ZenStorageServer&&) = delete; + +public: + ZenStorageServer(); + ~ZenStorageServer(); + + void SetDedicatedMode(bool State) { m_IsDedicatedMode = State; } + void SetTestMode(bool State) { m_TestMode = State; } + void SetDataRoot(std::filesystem::path Root) { m_DataRoot = Root; } + void SetContentRoot(std::filesystem::path Root) { m_ContentRoot = Root; } + + int Initialize(const ZenStorageServerOptions& ServerOptions, ZenServerState::ZenServerEntry* ServerEntry); + void Run(); + void Cleanup(); + +private: + void InitializeState(const ZenStorageServerOptions& ServerOptions); + void InitializeStructuredCache(const ZenStorageServerOptions& ServerOptions); + void Flush(); + + bool m_IsDedicatedMode = false; + bool m_TestMode = false; + bool m_DebugOptionForcedCrash = false; + std::string m_StartupScrubOptions; + CbObject m_RootManifest; + std::filesystem::path m_DataRoot; + std::filesystem::path m_ContentRoot; + asio::steady_timer m_StateMarkerTimer{m_IoContext}; + + void EnqueueStateMarkerTimer(); + void CheckStateMarker(); + + std::unique_ptr m_AuthMgr; + std::unique_ptr m_AuthService; + void InitializeAuthentication(const ZenStorageServerOptions& ServerOptions); + + void InitializeServices(const ZenStorageServerOptions& ServerOptions); + void RegisterServices(); + + HttpStatsService m_StatsService; + std::unique_ptr m_JobQueue; + GcManager m_GcManager; + GcScheduler m_GcScheduler{m_GcManager}; + std::unique_ptr m_CidStore; + Ref m_CacheStore; + std::unique_ptr m_OpenProcessCache; + HttpTestService m_TestService; + std::unique_ptr m_BuildCidStore; + std::unique_ptr m_BuildStore; + +#if ZEN_WITH_TESTS + HttpTestingService m_TestingService; +#endif + + RefPtr m_ProjectStore; + std::unique_ptr m_VfsServiceImpl; + std::unique_ptr m_HttpProjectService; + std::unique_ptr m_Workspaces; + std::unique_ptr m_HttpWorkspacesService; + std::unique_ptr m_UpstreamCache; + std::unique_ptr m_UpstreamService; + std::unique_ptr m_StructuredCacheService; + std::unique_ptr m_FrontendService; + std::unique_ptr m_ObjStoreService; + std::unique_ptr m_BuildStoreService; + std::unique_ptr m_VfsService; + std::unique_ptr m_AdminService; +}; + +class ZenStorageServerMain : public ZenServerMain +{ +public: + ZenStorageServerMain(ZenStorageServerOptions& ServerOptions); + virtual void DoRun(ZenServerState::ZenServerEntry* Entry) override; + + ZenStorageServerMain(const ZenStorageServerMain&) = delete; + ZenStorageServerMain& operator=(const ZenStorageServerMain&) = delete; + +private: + ZenStorageServerOptions& m_ServerOptions; +}; + +} // namespace zen diff --git a/src/zenserver/storageconfig.cpp b/src/zenserver/storageconfig.cpp deleted file mode 100644 index 86bb09c21..000000000 --- a/src/zenserver/storageconfig.cpp +++ /dev/null @@ -1,1055 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "storageconfig.h" - -#include -#include -#include -#include - -#include "config/luaconfig.h" - -ZEN_THIRD_PARTY_INCLUDES_START -#include -#include -#include -ZEN_THIRD_PARTY_INCLUDES_END - -namespace zen { - -void -ValidateOptions(ZenStorageServerOptions& ServerOptions) -{ - if (ServerOptions.EncryptionKey.empty() == false) - { - const auto Key = AesKey256Bit::FromString(ServerOptions.EncryptionKey); - - if (Key.IsValid() == false) - { - throw OptionParseException(fmt::format("'--encryption-aes-key' ('{}') is malformed", ServerOptions.EncryptionKey), {}); - } - } - - if (ServerOptions.EncryptionIV.empty() == false) - { - const auto IV = AesIV128Bit::FromString(ServerOptions.EncryptionIV); - - if (IV.IsValid() == false) - { - throw OptionParseException(fmt::format("'--encryption-aes-iv' ('{}') is malformed", ServerOptions.EncryptionIV), {}); - } - } - if (ServerOptions.HttpServerConfig.ForceLoopback && ServerOptions.IsDedicated) - { - throw OptionParseException("'--dedicated' conflicts with '--http-forceloopback'", {}); - } - if (ServerOptions.GcConfig.AttachmentPassCount > ZenGcConfig::GcMaxAttachmentPassCount) - { - throw OptionParseException(fmt::format("'--gc-attachment-passes' ('{}') is invalid, maximum is {}.", - ServerOptions.GcConfig.AttachmentPassCount, - ZenGcConfig::GcMaxAttachmentPassCount), - {}); - } - if (ServerOptions.GcConfig.UseGCV2 == false) - { - ZEN_WARN("'--gc-v2=false' is deprecated, reverting to '--gc-v2=true'"); - ServerOptions.GcConfig.UseGCV2 = true; - } -} - -class ZenStructuredCacheBucketsConfigOption : public LuaConfig::OptionValue -{ -public: - ZenStructuredCacheBucketsConfigOption(std::vector>& Value) : Value(Value) {} - virtual void Print(std::string_view Indent, StringBuilderBase& StringBuilder) override - { - if (Value.empty()) - { - StringBuilder.Append("{}"); - return; - } - LuaConfig::LuaContainerWriter Writer(StringBuilder, Indent); - for (const std::pair& Bucket : Value) - { - Writer.BeginContainer(""); - { - Writer.WriteValue("name", Bucket.first); - const ZenStructuredCacheBucketConfig& BucketConfig = Bucket.second; - - Writer.WriteValue("maxblocksize", fmt::format("{}", BucketConfig.MaxBlockSize)); - Writer.BeginContainer("memlayer"); - { - Writer.WriteValue("sizethreshold", fmt::format("{}", BucketConfig.MemCacheSizeThreshold)); - } - Writer.EndContainer(); - - Writer.WriteValue("payloadalignment", fmt::format("{}", BucketConfig.PayloadAlignment)); - Writer.WriteValue("largeobjectthreshold", fmt::format("{}", BucketConfig.PayloadAlignment)); - Writer.WriteValue("limitoverwrites", fmt::format("{}", BucketConfig.LimitOverwrites)); - } - Writer.EndContainer(); - } - } - virtual void Parse(sol::object Object) override - { - if (sol::optional Buckets = Object.as()) - { - for (const auto& Kv : Buckets.value()) - { - if (sol::optional Bucket = Kv.second.as()) - { - ZenStructuredCacheBucketConfig BucketConfig; - std::string Name = Kv.first.as(); - if (Name.empty()) - { - throw OptionParseException("Cache bucket option must have a name.", {}); - } - - const uint64_t MaxBlockSize = Bucket.value().get_or("maxblocksize", BucketConfig.MaxBlockSize); - if (MaxBlockSize == 0) - { - throw OptionParseException( - fmt::format("'maxblocksize' option for cache bucket '{}' is invalid. It must be non-zero.", Name), - {}); - } - BucketConfig.MaxBlockSize = MaxBlockSize; - - if (sol::optional Memlayer = Bucket.value().get_or("memlayer", sol::table())) - { - const uint64_t MemCacheSizeThreshold = Bucket.value().get_or("sizethreshold", BucketConfig.MemCacheSizeThreshold); - if (MemCacheSizeThreshold == 0) - { - throw OptionParseException( - fmt::format("'memlayer.sizethreshold' option for cache bucket '{}' is invalid. It must be non-zero.", Name), - {}); - } - BucketConfig.MemCacheSizeThreshold = Bucket.value().get_or("sizethreshold", BucketConfig.MemCacheSizeThreshold); - } - - const uint32_t PayloadAlignment = Bucket.value().get_or("payloadalignment", BucketConfig.PayloadAlignment); - if (PayloadAlignment == 0 || !IsPow2(PayloadAlignment)) - { - throw OptionParseException( - fmt::format( - "'payloadalignment' option for cache bucket '{}' is invalid. It needs to be non-zero and a power of two.", - Name), - {}); - } - BucketConfig.PayloadAlignment = PayloadAlignment; - - const uint64_t LargeObjectThreshold = Bucket.value().get_or("largeobjectthreshold", BucketConfig.LargeObjectThreshold); - if (LargeObjectThreshold == 0) - { - throw OptionParseException( - fmt::format("'largeobjectthreshold' option for cache bucket '{}' is invalid. It must be non-zero.", Name), - {}); - } - BucketConfig.LargeObjectThreshold = LargeObjectThreshold; - - BucketConfig.LimitOverwrites = Bucket.value().get_or("limitoverwrites", BucketConfig.LimitOverwrites); - - Value.push_back(std::make_pair(std::move(Name), BucketConfig)); - } - } - } - } - std::vector>& Value; -}; - -UpstreamCachePolicy -ParseUpstreamCachePolicy(std::string_view Options) -{ - if (Options == "readonly") - { - return UpstreamCachePolicy::Read; - } - else if (Options == "writeonly") - { - return UpstreamCachePolicy::Write; - } - else if (Options == "disabled") - { - return UpstreamCachePolicy::Disabled; - } - else - { - return UpstreamCachePolicy::ReadWrite; - } -} - -ZenObjectStoreConfig -ParseBucketConfigs(std::span Buckets) -{ - using namespace std::literals; - - ZenObjectStoreConfig Cfg; - - // split bucket args in the form of "{BucketName};{LocalPath}" - for (std::string_view Bucket : Buckets) - { - ZenObjectStoreConfig::BucketConfig NewBucket; - - if (auto Idx = Bucket.find_first_of(";"); Idx != std::string_view::npos) - { - NewBucket.Name = Bucket.substr(0, Idx); - NewBucket.Directory = Bucket.substr(Idx + 1); - } - else - { - NewBucket.Name = Bucket; - } - - Cfg.Buckets.push_back(std::move(NewBucket)); - } - - return Cfg; -} - -class CachePolicyOption : public LuaConfig::OptionValue -{ -public: - CachePolicyOption(UpstreamCachePolicy& Value) : Value(Value) {} - virtual void Print(std::string_view, StringBuilderBase& StringBuilder) override - { - switch (Value) - { - case UpstreamCachePolicy::Read: - StringBuilder.Append("readonly"); - break; - case UpstreamCachePolicy::Write: - StringBuilder.Append("writeonly"); - break; - case UpstreamCachePolicy::Disabled: - StringBuilder.Append("disabled"); - break; - case UpstreamCachePolicy::ReadWrite: - StringBuilder.Append("readwrite"); - break; - default: - ZEN_ASSERT(false); - } - } - virtual void Parse(sol::object Object) override - { - std::string PolicyString = Object.as(); - if (PolicyString == "readonly") - { - Value = UpstreamCachePolicy::Read; - } - else if (PolicyString == "writeonly") - { - Value = UpstreamCachePolicy::Write; - } - else if (PolicyString == "disabled") - { - Value = UpstreamCachePolicy::Disabled; - } - else if (PolicyString == "readwrite") - { - Value = UpstreamCachePolicy::ReadWrite; - } - } - UpstreamCachePolicy& Value; -}; - -class ZenAuthConfigOption : public LuaConfig::OptionValue -{ -public: - ZenAuthConfigOption(ZenAuthConfig& Value) : Value(Value) {} - virtual void Print(std::string_view Indent, StringBuilderBase& StringBuilder) override - { - if (Value.OpenIdProviders.empty()) - { - StringBuilder.Append("{}"); - return; - } - LuaConfig::LuaContainerWriter Writer(StringBuilder, Indent); - for (const ZenOpenIdProviderConfig& Config : Value.OpenIdProviders) - { - Writer.BeginContainer(""); - { - Writer.WriteValue("name", Config.Name); - Writer.WriteValue("url", Config.Url); - Writer.WriteValue("clientid", Config.ClientId); - } - Writer.EndContainer(); - } - } - virtual void Parse(sol::object Object) override - { - if (sol::optional OpenIdProviders = Object.as()) - { - for (const auto& Kv : OpenIdProviders.value()) - { - if (sol::optional OpenIdProvider = Kv.second.as()) - { - std::string Name = OpenIdProvider.value().get_or("name", std::string("Default")); - std::string Url = OpenIdProvider.value().get_or("url", std::string()); - std::string ClientId = OpenIdProvider.value().get_or("clientid", std::string()); - - Value.OpenIdProviders.push_back({.Name = std::move(Name), .Url = std::move(Url), .ClientId = std::move(ClientId)}); - } - } - } - } - ZenAuthConfig& Value; -}; - -class ZenObjectStoreConfigOption : public LuaConfig::OptionValue -{ -public: - ZenObjectStoreConfigOption(ZenObjectStoreConfig& Value) : Value(Value) {} - virtual void Print(std::string_view Indent, StringBuilderBase& StringBuilder) override - { - if (Value.Buckets.empty()) - { - StringBuilder.Append("{}"); - return; - } - LuaConfig::LuaContainerWriter Writer(StringBuilder, Indent); - for (const ZenObjectStoreConfig::BucketConfig& Config : Value.Buckets) - { - Writer.BeginContainer(""); - { - Writer.WriteValue("name", Config.Name); - std::string Directory = Config.Directory.string(); - LuaConfig::EscapeBackslash(Directory); - Writer.WriteValue("directory", Directory); - } - Writer.EndContainer(); - } - } - virtual void Parse(sol::object Object) override - { - if (sol::optional Buckets = Object.as()) - { - for (const auto& Kv : Buckets.value()) - { - if (sol::optional Bucket = Kv.second.as()) - { - std::string Name = Bucket.value().get_or("name", std::string("Default")); - std::string Directory = Bucket.value().get_or("directory", std::string()); - - Value.Buckets.push_back({.Name = std::move(Name), .Directory = MakeSafeAbsolutePath(Directory)}); - } - } - } - } - ZenObjectStoreConfig& Value; -}; - -std::shared_ptr -MakeOption(UpstreamCachePolicy& Value) -{ - return std::make_shared(Value); -}; - -std::shared_ptr -MakeOption(ZenAuthConfig& Value) -{ - return std::make_shared(Value); -}; - -std::shared_ptr -MakeOption(ZenObjectStoreConfig& Value) -{ - return std::make_shared(Value); -}; - -std::shared_ptr -MakeOption(std::vector>& Value) -{ - return std::make_shared(Value); -}; - -void -ParseConfigFile(const std::filesystem::path& Path, - ZenStorageServerOptions& ServerOptions, - const cxxopts::ParseResult& CmdLineResult, - std::string_view OutputConfigFile) -{ - ZEN_TRACE_CPU("ParseConfigFile"); - - using namespace std::literals; - - LuaConfig::Options LuaOptions; - - AddServerConfigOptions(LuaOptions, ServerOptions); - - ////// server - LuaOptions.AddOption("server.pluginsconfigfile"sv, ServerOptions.PluginsConfigFile, "plugins-config"sv); - - ////// objectstore - LuaOptions.AddOption("server.objectstore.enabled"sv, ServerOptions.ObjectStoreEnabled, "objectstore-enabled"sv); - LuaOptions.AddOption("server.objectstore.buckets"sv, ServerOptions.ObjectStoreConfig); - - ////// buildsstore - LuaOptions.AddOption("server.buildstore.enabled"sv, ServerOptions.BuildStoreConfig.Enabled, "buildstore-enabled"sv); - LuaOptions.AddOption("server.buildstore.disksizelimit"sv, ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit, "buildstore-disksizelimit"); - - ////// cache - LuaOptions.AddOption("cache.enable"sv, ServerOptions.StructuredCacheConfig.Enabled); - LuaOptions.AddOption("cache.writelog"sv, ServerOptions.StructuredCacheConfig.WriteLogEnabled, "cache-write-log"sv); - LuaOptions.AddOption("cache.accesslog"sv, ServerOptions.StructuredCacheConfig.AccessLogEnabled, "cache-access-log"sv); - - LuaOptions.AddOption("cache.buckets"sv, ServerOptions.StructuredCacheConfig.PerBucketConfigs, "cache.buckets"sv); - - LuaOptions.AddOption("cache.memlayer.sizethreshold"sv, - ServerOptions.StructuredCacheConfig.BucketConfig.MemCacheSizeThreshold, - "cache-memlayer-sizethreshold"sv); - LuaOptions.AddOption("cache.memlayer.targetfootprint"sv, - ServerOptions.StructuredCacheConfig.MemTargetFootprintBytes, - "cache-memlayer-targetfootprint"sv); - LuaOptions.AddOption("cache.memlayer.triminterval"sv, - ServerOptions.StructuredCacheConfig.MemTrimIntervalSeconds, - "cache-memlayer-triminterval"sv); - LuaOptions.AddOption("cache.memlayer.maxage"sv, ServerOptions.StructuredCacheConfig.MemMaxAgeSeconds, "cache-memlayer-maxage"sv); - - LuaOptions.AddOption("cache.bucket.maxblocksize"sv, - ServerOptions.StructuredCacheConfig.BucketConfig.MaxBlockSize, - "cache-bucket-maxblocksize"sv); - LuaOptions.AddOption("cache.bucket.memlayer.sizethreshold"sv, - ServerOptions.StructuredCacheConfig.BucketConfig.MemCacheSizeThreshold, - "cache-bucket-memlayer-sizethreshold"sv); - LuaOptions.AddOption("cache.bucket.payloadalignment"sv, - ServerOptions.StructuredCacheConfig.BucketConfig.PayloadAlignment, - "cache-bucket-payloadalignment"sv); - LuaOptions.AddOption("cache.bucket.largeobjectthreshold"sv, - ServerOptions.StructuredCacheConfig.BucketConfig.LargeObjectThreshold, - "cache-bucket-largeobjectthreshold"sv); - LuaOptions.AddOption("cache.bucket.limitoverwrites"sv, - ServerOptions.StructuredCacheConfig.BucketConfig.LimitOverwrites, - "cache-bucket-limit-overwrites"sv); - - ////// cache.upstream - LuaOptions.AddOption("cache.upstream.policy"sv, ServerOptions.UpstreamCacheConfig.CachePolicy, "upstream-cache-policy"sv); - LuaOptions.AddOption("cache.upstream.upstreamthreadcount"sv, - ServerOptions.UpstreamCacheConfig.UpstreamThreadCount, - "upstream-thread-count"sv); - LuaOptions.AddOption("cache.upstream.connecttimeoutms"sv, - ServerOptions.UpstreamCacheConfig.ConnectTimeoutMilliseconds, - "upstream-connect-timeout-ms"sv); - LuaOptions.AddOption("cache.upstream.timeoutms"sv, ServerOptions.UpstreamCacheConfig.TimeoutMilliseconds, "upstream-timeout-ms"sv); - - ////// cache.upstream.jupiter - LuaOptions.AddOption("cache.upstream.jupiter.name"sv, ServerOptions.UpstreamCacheConfig.JupiterConfig.Name); - LuaOptions.AddOption("cache.upstream.jupiter.url"sv, ServerOptions.UpstreamCacheConfig.JupiterConfig.Url, "upstream-jupiter-url"sv); - LuaOptions.AddOption("cache.upstream.jupiter.oauthprovider"sv, - ServerOptions.UpstreamCacheConfig.JupiterConfig.OAuthUrl, - "upstream-jupiter-oauth-url"sv); - LuaOptions.AddOption("cache.upstream.jupiter.oauthclientid"sv, - ServerOptions.UpstreamCacheConfig.JupiterConfig.OAuthClientId, - "upstream-jupiter-oauth-clientid"); - LuaOptions.AddOption("cache.upstream.jupiter.oauthclientsecret"sv, - ServerOptions.UpstreamCacheConfig.JupiterConfig.OAuthClientSecret, - "upstream-jupiter-oauth-clientsecret"sv); - LuaOptions.AddOption("cache.upstream.jupiter.openidprovider"sv, - ServerOptions.UpstreamCacheConfig.JupiterConfig.OpenIdProvider, - "upstream-jupiter-openid-provider"sv); - LuaOptions.AddOption("cache.upstream.jupiter.token"sv, - ServerOptions.UpstreamCacheConfig.JupiterConfig.AccessToken, - "upstream-jupiter-token"sv); - LuaOptions.AddOption("cache.upstream.jupiter.namespace"sv, - ServerOptions.UpstreamCacheConfig.JupiterConfig.Namespace, - "upstream-jupiter-namespace"sv); - LuaOptions.AddOption("cache.upstream.jupiter.ddcnamespace"sv, - ServerOptions.UpstreamCacheConfig.JupiterConfig.DdcNamespace, - "upstream-jupiter-namespace-ddc"sv); - - ////// cache.upstream.zen - // LuaOptions.AddOption("cache.upstream.zen"sv, ServerOptions.UpstreamCacheConfig.ZenConfig); - LuaOptions.AddOption("cache.upstream.zen.name"sv, ServerOptions.UpstreamCacheConfig.ZenConfig.Name); - LuaOptions.AddOption("cache.upstream.zen.dns"sv, ServerOptions.UpstreamCacheConfig.ZenConfig.Dns); - LuaOptions.AddOption("cache.upstream.zen.url"sv, ServerOptions.UpstreamCacheConfig.ZenConfig.Urls); - - ////// gc - LuaOptions.AddOption("gc.enabled"sv, ServerOptions.GcConfig.Enabled, "gc-enabled"sv); - LuaOptions.AddOption("gc.v2"sv, ServerOptions.GcConfig.UseGCV2, "gc-v2"sv); - - LuaOptions.AddOption("gc.monitorintervalseconds"sv, ServerOptions.GcConfig.MonitorIntervalSeconds, "gc-monitor-interval-seconds"sv); - LuaOptions.AddOption("gc.intervalseconds"sv, ServerOptions.GcConfig.IntervalSeconds, "gc-interval-seconds"sv); - LuaOptions.AddOption("gc.collectsmallobjects"sv, ServerOptions.GcConfig.CollectSmallObjects, "gc-small-objects"sv); - LuaOptions.AddOption("gc.diskreservesize"sv, ServerOptions.GcConfig.DiskReserveSize, "disk-reserve-size"sv); - LuaOptions.AddOption("gc.disksizesoftlimit"sv, ServerOptions.GcConfig.DiskSizeSoftLimit, "gc-disksize-softlimit"sv); - LuaOptions.AddOption("gc.lowdiskspacethreshold"sv, - ServerOptions.GcConfig.MinimumFreeDiskSpaceToAllowWrites, - "gc-low-diskspace-threshold"sv); - LuaOptions.AddOption("gc.lightweightintervalseconds"sv, - ServerOptions.GcConfig.LightweightIntervalSeconds, - "gc-lightweight-interval-seconds"sv); - LuaOptions.AddOption("gc.compactblockthreshold"sv, - ServerOptions.GcConfig.CompactBlockUsageThresholdPercent, - "gc-compactblock-threshold"sv); - LuaOptions.AddOption("gc.verbose"sv, ServerOptions.GcConfig.Verbose, "gc-verbose"sv); - LuaOptions.AddOption("gc.single-threaded"sv, ServerOptions.GcConfig.SingleThreaded, "gc-single-threaded"sv); - LuaOptions.AddOption("gc.cache.attachment.store"sv, ServerOptions.GcConfig.StoreCacheAttachmentMetaData, "gc-cache-attachment-store"); - LuaOptions.AddOption("gc.projectstore.attachment.store"sv, - ServerOptions.GcConfig.StoreProjectAttachmentMetaData, - "gc-projectstore-attachment-store"); - LuaOptions.AddOption("gc.attachment.passes"sv, ServerOptions.GcConfig.AttachmentPassCount, "gc-attachment-passes"sv); - LuaOptions.AddOption("gc.validation"sv, ServerOptions.GcConfig.EnableValidation, "gc-validation"); - - LuaOptions.AddOption("gc.cache.maxdurationseconds"sv, ServerOptions.GcConfig.Cache.MaxDurationSeconds, "gc-cache-duration-seconds"sv); - LuaOptions.AddOption("gc.projectstore.duration.seconds"sv, - ServerOptions.GcConfig.ProjectStore.MaxDurationSeconds, - "gc-projectstore-duration-seconds"); - LuaOptions.AddOption("gc.buildstore.duration.seconds"sv, - ServerOptions.GcConfig.BuildStore.MaxDurationSeconds, - "gc-buildstore-duration-seconds"); - - ////// security - LuaOptions.AddOption("security.encryptionaeskey"sv, ServerOptions.EncryptionKey, "encryption-aes-key"sv); - LuaOptions.AddOption("security.encryptionaesiv"sv, ServerOptions.EncryptionIV, "encryption-aes-iv"sv); - LuaOptions.AddOption("security.openidproviders"sv, ServerOptions.AuthConfig); - - ////// workspaces - LuaOptions.AddOption("workspaces.enabled"sv, ServerOptions.WorksSpacesConfig.Enabled, "workspaces-enabled"sv); - LuaOptions.AddOption("workspaces.allowconfigchanges"sv, - ServerOptions.WorksSpacesConfig.AllowConfigurationChanges, - "workspaces-allow-changes"sv); - - LuaOptions.Parse(Path, CmdLineResult); - - // These have special command line processing so we make sure we export them if they were configured on command line - if (!ServerOptions.AuthConfig.OpenIdProviders.empty()) - { - LuaOptions.Touch("security.openidproviders"sv); - } - if (!ServerOptions.ObjectStoreConfig.Buckets.empty()) - { - LuaOptions.Touch("server.objectstore.buckets"sv); - } - if (!ServerOptions.StructuredCacheConfig.PerBucketConfigs.empty()) - { - LuaOptions.Touch("cache.buckets"sv); - } - - if (!OutputConfigFile.empty()) - { - std::filesystem::path WritePath(MakeSafeAbsolutePath(OutputConfigFile)); - ExtendableStringBuilder<512> ConfigStringBuilder; - LuaOptions.Print(ConfigStringBuilder, CmdLineResult); - BasicFile Output; - Output.Open(WritePath, BasicFile::Mode::kTruncate); - Output.Write(ConfigStringBuilder.Data(), ConfigStringBuilder.Size(), 0); - } -} - -void -ParsePluginsConfigFile(const std::filesystem::path& Path, ZenStorageServerOptions& ServerOptions, int BasePort) -{ - using namespace std::literals; - - IoBuffer Body = IoBufferBuilder::MakeFromFile(Path); - std::string JsonText(reinterpret_cast(Body.GetData()), Body.GetSize()); - std::string JsonError; - json11::Json PluginsInfo = json11::Json::parse(JsonText, JsonError); - if (!JsonError.empty()) - { - ZEN_WARN("failed parsing plugins config file '{}'. Reason: '{}'", Path, JsonError); - return; - } - for (const json11::Json& PluginInfo : PluginsInfo.array_items()) - { - if (!PluginInfo.is_object()) - { - ZEN_WARN("the json file '{}' does not contain a valid plugin definition, object expected, got '{}'", Path, PluginInfo.dump()); - continue; - } - - HttpServerPluginConfig Config = {}; - - bool bNeedsPort = true; - - for (const std::pair& Items : PluginInfo.object_items()) - { - if (!Items.second.is_string()) - { - ZEN_WARN("the json file '{}' does not contain a valid plugins definition, string expected, got '{}'", - Path, - Items.second.dump()); - continue; - } - - const std::string& Name = Items.first; - const std::string& Value = Items.second.string_value(); - - if (Name == "name"sv) - Config.PluginName = Value; - else - { - Config.PluginOptions.push_back({Name, Value}); - - if (Name == "port"sv) - { - bNeedsPort = false; - } - } - } - - // add a default base port in case if json config didn't provide one - if (bNeedsPort) - { - Config.PluginOptions.push_back({"port", std::to_string(BasePort)}); - } - - ServerOptions.HttpServerConfig.PluginConfigs.push_back(Config); - } -} - -void -ZenStorageServerCmdLineOptions::AddCliOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions) -{ - options.add_options()("snapshot-dir", - "Specify a snapshot of server state to mirror into the persistence root at startup", - cxxopts::value(BaseSnapshotDir)); - options.add_options()("plugins-config", "Path to plugins config file", cxxopts::value(PluginsConfigFile)); - options.add_options()("scrub", - "Validate state at startup", - cxxopts::value(ServerOptions.ScrubOptions)->implicit_value("yes"), - "(nocas,nogc,nodelete,yes,no)*"); - - AddSecurityOptions(options, ServerOptions); - AddCacheOptions(options, ServerOptions); - AddGcOptions(options, ServerOptions); - AddObjectStoreOptions(options, ServerOptions); - AddBuildStoreOptions(options, ServerOptions); - AddWorkspacesOptions(options, ServerOptions); -} - -void -ZenStorageServerCmdLineOptions::AddSecurityOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions) -{ - options.add_option("security", - "", - "encryption-aes-key", - "256 bit AES encryption key", - cxxopts::value(ServerOptions.EncryptionKey), - ""); - - options.add_option("security", - "", - "encryption-aes-iv", - "128 bit AES encryption initialization vector", - cxxopts::value(ServerOptions.EncryptionIV), - ""); - - options.add_option("security", - "", - "openid-provider-name", - "Open ID provider name", - cxxopts::value(OpenIdProviderName), - "Default"); - - options.add_option("security", "", "openid-provider-url", "Open ID provider URL", cxxopts::value(OpenIdProviderUrl), ""); - options.add_option("security", "", "openid-client-id", "Open ID client ID", cxxopts::value(OpenIdClientId), ""); -} - -void -ZenStorageServerCmdLineOptions::AddCacheOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions) -{ - options.add_option("cache", - "", - "upstream-cache-policy", - "", - cxxopts::value(UpstreamCachePolicyOptions)->default_value(""), - "Upstream cache policy (readwrite|readonly|writeonly|disabled)"); - - options.add_option("cache", - "", - "upstream-jupiter-url", - "URL to a Jupiter instance", - cxxopts::value(ServerOptions.UpstreamCacheConfig.JupiterConfig.Url)->default_value(""), - ""); - - options.add_option("cache", - "", - "upstream-jupiter-oauth-url", - "URL to the OAuth provier", - cxxopts::value(ServerOptions.UpstreamCacheConfig.JupiterConfig.OAuthUrl)->default_value(""), - ""); - - options.add_option("cache", - "", - "upstream-jupiter-oauth-clientid", - "The OAuth client ID", - cxxopts::value(ServerOptions.UpstreamCacheConfig.JupiterConfig.OAuthClientId)->default_value(""), - ""); - - options.add_option("cache", - "", - "upstream-jupiter-oauth-clientsecret", - "The OAuth client secret", - cxxopts::value(ServerOptions.UpstreamCacheConfig.JupiterConfig.OAuthClientSecret)->default_value(""), - ""); - - options.add_option("cache", - "", - "upstream-jupiter-openid-provider", - "Name of a registered Open ID provider", - cxxopts::value(ServerOptions.UpstreamCacheConfig.JupiterConfig.OpenIdProvider)->default_value(""), - ""); - - options.add_option("cache", - "", - "upstream-jupiter-token", - "A static authentication token", - cxxopts::value(ServerOptions.UpstreamCacheConfig.JupiterConfig.AccessToken)->default_value(""), - ""); - - options.add_option("cache", - "", - "upstream-jupiter-namespace", - "The Common Blob Store API namespace", - cxxopts::value(ServerOptions.UpstreamCacheConfig.JupiterConfig.Namespace)->default_value(""), - ""); - - options.add_option("cache", - "", - "upstream-jupiter-namespace-ddc", - "The lecacy DDC namespace", - cxxopts::value(ServerOptions.UpstreamCacheConfig.JupiterConfig.DdcNamespace)->default_value(""), - ""); - - options.add_option("cache", - "", - "upstream-zen-url", - "URL to remote Zen server. Use a comma separated list to choose the one with the best latency.", - cxxopts::value>(ServerOptions.UpstreamCacheConfig.ZenConfig.Urls), - ""); - - options.add_option("cache", - "", - "upstream-zen-dns", - "DNS that resolves to one or more Zen server instance(s)", - cxxopts::value>(ServerOptions.UpstreamCacheConfig.ZenConfig.Dns), - ""); - - options.add_option("cache", - "", - "upstream-thread-count", - "Number of threads used for upstream procsssing", - cxxopts::value(ServerOptions.UpstreamCacheConfig.UpstreamThreadCount)->default_value("4"), - ""); - - options.add_option("cache", - "", - "upstream-connect-timeout-ms", - "Connect timeout in millisecond(s). Default 5000 ms.", - cxxopts::value(ServerOptions.UpstreamCacheConfig.ConnectTimeoutMilliseconds)->default_value("5000"), - ""); - - options.add_option("cache", - "", - "upstream-timeout-ms", - "Timeout in millisecond(s). Default 0 ms", - cxxopts::value(ServerOptions.UpstreamCacheConfig.TimeoutMilliseconds)->default_value("0"), - ""); - - options.add_option("cache", - "", - "cache-write-log", - "Whether cache write log is enabled", - cxxopts::value(ServerOptions.StructuredCacheConfig.WriteLogEnabled)->default_value("false"), - ""); - - options.add_option("cache", - "", - "cache-access-log", - "Whether cache access log is enabled", - cxxopts::value(ServerOptions.StructuredCacheConfig.AccessLogEnabled)->default_value("false"), - ""); - - options.add_option( - "cache", - "", - "cache-memlayer-sizethreshold", - "The largest size of a cache entry that may be cached in memory. Default set to 1024 (1 Kb). Set to 0 to disable memory " - "caching. " - "Obsolete, replaced by `--cache-bucket-memlayer-sizethreshold`", - cxxopts::value(ServerOptions.StructuredCacheConfig.BucketConfig.MemCacheSizeThreshold)->default_value("1024"), - ""); - - options.add_option("cache", - "", - "cache-memlayer-targetfootprint", - "Max allowed memory used by cache memory layer per namespace in bytes. Default set to 536870912 (512 Mb).", - cxxopts::value(ServerOptions.StructuredCacheConfig.MemTargetFootprintBytes)->default_value("536870912"), - ""); - - options.add_option("cache", - "", - "cache-memlayer-triminterval", - "Minimum time between each attempt to trim cache memory layers in seconds. Default set to 60 (1 min).", - cxxopts::value(ServerOptions.StructuredCacheConfig.MemTrimIntervalSeconds)->default_value("60"), - ""); - - options.add_option("cache", - "", - "cache-memlayer-maxage", - "Maximum age of payloads when trimming cache memory layers in seconds. Default set to 86400 (1 day).", - cxxopts::value(ServerOptions.StructuredCacheConfig.MemMaxAgeSeconds)->default_value("86400"), - ""); - - options.add_option("cache", - "", - "cache-bucket-maxblocksize", - "Max size of cache bucket blocks. Default set to 1073741824 (1GB).", - cxxopts::value(ServerOptions.StructuredCacheConfig.BucketConfig.MaxBlockSize)->default_value("1073741824"), - ""); - - options.add_option("cache", - "", - "cache-bucket-payloadalignment", - "Payload alignement for cache bucket blocks. Default set to 16.", - cxxopts::value(ServerOptions.StructuredCacheConfig.BucketConfig.PayloadAlignment)->default_value("16"), - ""); - - options.add_option( - "cache", - "", - "cache-bucket-largeobjectthreshold", - "Threshold for storing cache bucket values as loose files. Default set to 131072 (128 KB).", - cxxopts::value(ServerOptions.StructuredCacheConfig.BucketConfig.LargeObjectThreshold)->default_value("131072"), - ""); - - options.add_option( - "cache", - "", - "cache-bucket-memlayer-sizethreshold", - "The largest size of a cache entry that may be cached in memory. Default set to 1024 (1 Kb). Set to 0 to disable memory " - "caching.", - cxxopts::value(ServerOptions.StructuredCacheConfig.BucketConfig.MemCacheSizeThreshold)->default_value("1024"), - ""); - - options.add_option("cache", - "", - "cache-bucket-limit-overwrites", - "Whether to require policy flag pattern before allowing overwrites in cache bucket", - cxxopts::value(ServerOptions.StructuredCacheConfig.BucketConfig.LimitOverwrites)->default_value("false"), - ""); -} - -void -ZenStorageServerCmdLineOptions::AddGcOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions) -{ - options.add_option("gc", - "", - "gc-cache-attachment-store", - "Enable storing attachments referenced by a cache record in block store meta data.", - cxxopts::value(ServerOptions.GcConfig.StoreCacheAttachmentMetaData)->default_value("false"), - ""); - - options.add_option("gc", - "", - "gc-projectstore-attachment-store", - "Enable storing attachments referenced by project oplogs in meta data.", - cxxopts::value(ServerOptions.GcConfig.StoreProjectAttachmentMetaData)->default_value("false"), - ""); - - options.add_option("gc", - "", - "gc-validation", - "Enable validation of references after full GC.", - cxxopts::value(ServerOptions.GcConfig.EnableValidation)->default_value("true"), - ""); - - options.add_option("gc", - "", - "gc-enabled", - "Whether garbage collection is enabled or not.", - cxxopts::value(ServerOptions.GcConfig.Enabled)->default_value("true"), - ""); - - options.add_option("gc", - "", - "gc-v2", - "Use V2 of GC implementation or not.", - cxxopts::value(ServerOptions.GcConfig.UseGCV2)->default_value("true"), - ""); - - options.add_option("gc", - "", - "gc-small-objects", - "Whether garbage collection of small objects is enabled or not.", - cxxopts::value(ServerOptions.GcConfig.CollectSmallObjects)->default_value("true"), - ""); - - options.add_option("gc", - "", - "gc-interval-seconds", - "Garbage collection interval in seconds. Default set to 3600 (1 hour).", - cxxopts::value(ServerOptions.GcConfig.IntervalSeconds)->default_value("3600"), - ""); - - options.add_option("gc", - "", - "gc-lightweight-interval-seconds", - "Lightweight garbage collection interval in seconds. Default set to 900 (30 min).", - cxxopts::value(ServerOptions.GcConfig.LightweightIntervalSeconds)->default_value("900"), - ""); - - options.add_option("gc", - "", - "gc-cache-duration-seconds", - "Max duration in seconds before Z$ entries get evicted. Default set to 1209600 (2 weeks)", - cxxopts::value(ServerOptions.GcConfig.Cache.MaxDurationSeconds)->default_value("1209600"), - ""); - - options.add_option("gc", - "", - "gc-projectstore-duration-seconds", - "Max duration in seconds before project store entries get evicted. Default set to 1209600 (2 weeks)", - cxxopts::value(ServerOptions.GcConfig.ProjectStore.MaxDurationSeconds)->default_value("1209600"), - ""); - - options.add_option("gc", - "", - "gc-buildstore-duration-seconds", - "Max duration in seconds before build store entries get evicted. Default set to 604800 (1 week)", - cxxopts::value(ServerOptions.GcConfig.BuildStore.MaxDurationSeconds)->default_value("604800"), - ""); - - options.add_option("gc", - "", - "disk-reserve-size", - "Size of gc disk reserve in bytes. Default set to 268435456 (256 Mb). Set to zero to disable.", - cxxopts::value(ServerOptions.GcConfig.DiskReserveSize)->default_value("268435456"), - ""); - - options.add_option("gc", - "", - "gc-monitor-interval-seconds", - "Garbage collection monitoring interval in seconds. Default set to 30 (30 seconds)", - cxxopts::value(ServerOptions.GcConfig.MonitorIntervalSeconds)->default_value("30"), - ""); - - options.add_option("gc", - "", - "gc-low-diskspace-threshold", - "Minimum free space on disk to allow writes to disk. Default set to 268435456 (256 Mb). Set to zero to disable.", - cxxopts::value(ServerOptions.GcConfig.MinimumFreeDiskSpaceToAllowWrites)->default_value("268435456"), - ""); - - options.add_option("gc", - "", - "gc-disksize-softlimit", - "Garbage collection disk usage soft limit. Default set to 0 (Off).", - cxxopts::value(ServerOptions.GcConfig.DiskSizeSoftLimit)->default_value("0"), - ""); - - options.add_option("gc", - "", - "gc-compactblock-threshold", - "Garbage collection - how much of a compact block should be used to skip compacting the block. 0 - compact only " - "empty eligible blocks, 100 - compact all non-full eligible blocks.", - cxxopts::value(ServerOptions.GcConfig.CompactBlockUsageThresholdPercent)->default_value("60"), - ""); - - options.add_option("gc", - "", - "gc-verbose", - "Enable verbose logging for GC.", - cxxopts::value(ServerOptions.GcConfig.Verbose)->default_value("false"), - ""); - - options.add_option("gc", - "", - "gc-single-threaded", - "Force GC to run single threaded.", - cxxopts::value(ServerOptions.GcConfig.SingleThreaded)->default_value("false"), - ""); - - options.add_option("gc", - "", - "gc-attachment-passes", - "Limit the range of unreferenced attachments included in GC check by breaking it into passes. Default is one pass " - "which includes all the attachments.", - cxxopts::value(ServerOptions.GcConfig.AttachmentPassCount)->default_value("1"), - ""); -} - -void -ZenStorageServerCmdLineOptions::AddObjectStoreOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions) -{ - options.add_option("objectstore", - "", - "objectstore-enabled", - "Whether the object store is enabled or not.", - cxxopts::value(ServerOptions.ObjectStoreEnabled)->default_value("false"), - ""); - - options.add_option("objectstore", - "", - "objectstore-bucket", - "Object store bucket mappings.", - cxxopts::value>(BucketConfigs), - ""); -} - -void -ZenStorageServerCmdLineOptions::AddBuildStoreOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions) -{ - options.add_option("buildstore", - "", - "buildstore-enabled", - "Whether the builds store is enabled or not.", - cxxopts::value(ServerOptions.BuildStoreConfig.Enabled)->default_value("false"), - ""); - options.add_option("buildstore", - "", - "buildstore-disksizelimit", - "Max number of bytes before build store entries get evicted. Default set to 1099511627776 (1TB week)", - cxxopts::value(ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit)->default_value("1099511627776"), - ""); -} - -void -ZenStorageServerCmdLineOptions::AddWorkspacesOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions) -{ - options.add_option("workspaces", - "", - "workspaces-enabled", - "", - cxxopts::value(ServerOptions.WorksSpacesConfig.Enabled)->default_value("true"), - "Enable workspaces support with folder sharing"); - - options.add_option("workspaces", - "", - "workspaces-allow-changes", - "", - cxxopts::value(ServerOptions.WorksSpacesConfig.AllowConfigurationChanges)->default_value("false"), - "Allow adding/modifying/deleting of workspace and shares via http endpoint"); -} - -void -ZenStorageServerCmdLineOptions::ApplyOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions) -{ - ServerOptions.BaseSnapshotDir = MakeSafeAbsolutePath(BaseSnapshotDir); - ServerOptions.PluginsConfigFile = MakeSafeAbsolutePath(PluginsConfigFile); - ServerOptions.UpstreamCacheConfig.CachePolicy = ParseUpstreamCachePolicy(UpstreamCachePolicyOptions); - - if (!BaseSnapshotDir.empty()) - { - if (ServerOptions.DataDir.empty()) - throw OptionParseException("'--snapshot-dir' requires '--data-dir'", options.help()); - - if (!IsDir(ServerOptions.BaseSnapshotDir)) - throw std::runtime_error(fmt::format("'--snapshot-dir' ('{}') must be a directory", ServerOptions.BaseSnapshotDir)); - } - - if (OpenIdProviderUrl.empty() == false) - { - if (OpenIdClientId.empty()) - { - throw OptionParseException("'--openid-provider-url' requires '--openid-client-id'", options.help()); - } - - ServerOptions.AuthConfig.OpenIdProviders.push_back( - {.Name = OpenIdProviderName, .Url = OpenIdProviderUrl, .ClientId = OpenIdClientId}); - } - - ServerOptions.ObjectStoreConfig = ParseBucketConfigs(BucketConfigs); -} - -} // namespace zen diff --git a/src/zenserver/storageconfig.h b/src/zenserver/storageconfig.h deleted file mode 100644 index 2c3197226..000000000 --- a/src/zenserver/storageconfig.h +++ /dev/null @@ -1,203 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "config.h" - -namespace zen { - -struct ZenUpstreamJupiterConfig -{ - std::string Name; - std::string Url; - std::string OAuthUrl; - std::string OAuthClientId; - std::string OAuthClientSecret; - std::string OpenIdProvider; - std::string AccessToken; - std::string Namespace; - std::string DdcNamespace; -}; - -struct ZenUpstreamZenConfig -{ - std::string Name; - std::vector Urls; - std::vector Dns; -}; - -enum class UpstreamCachePolicy : uint8_t -{ - Disabled = 0, - Read = 1 << 0, - Write = 1 << 1, - ReadWrite = Read | Write -}; - -struct ZenUpstreamCacheConfig -{ - ZenUpstreamJupiterConfig JupiterConfig; - ZenUpstreamZenConfig ZenConfig; - int32_t UpstreamThreadCount = 4; - int32_t ConnectTimeoutMilliseconds = 5000; - int32_t TimeoutMilliseconds = 0; - UpstreamCachePolicy CachePolicy = UpstreamCachePolicy::ReadWrite; -}; - -struct ZenCacheEvictionPolicy -{ - int32_t MaxDurationSeconds = 24 * 60 * 60; -}; - -struct ZenProjectStoreEvictionPolicy -{ - int32_t MaxDurationSeconds = 7 * 24 * 60 * 60; -}; - -struct ZenBuildStoreEvictionPolicy -{ - int32_t MaxDurationSeconds = 3 * 24 * 60 * 60; -}; - -struct ZenGcConfig -{ - // ZenCasEvictionPolicy Cas; - ZenCacheEvictionPolicy Cache; - ZenProjectStoreEvictionPolicy ProjectStore; - ZenBuildStoreEvictionPolicy BuildStore; - int32_t MonitorIntervalSeconds = 30; - int32_t IntervalSeconds = 0; - bool CollectSmallObjects = true; - bool Enabled = true; - uint64_t DiskReserveSize = 1ul << 28; - uint64_t DiskSizeSoftLimit = 0; - int32_t LightweightIntervalSeconds = 0; - uint64_t MinimumFreeDiskSpaceToAllowWrites = 1ul << 28; - bool UseGCV2 = false; - uint32_t CompactBlockUsageThresholdPercent = 90; - bool Verbose = false; - bool SingleThreaded = false; - static constexpr uint16_t GcMaxAttachmentPassCount = 256; - uint16_t AttachmentPassCount = 1; - bool StoreCacheAttachmentMetaData = false; - bool StoreProjectAttachmentMetaData = false; - bool EnableValidation = true; -}; - -struct ZenOpenIdProviderConfig -{ - std::string Name; - std::string Url; - std::string ClientId; -}; - -struct ZenAuthConfig -{ - std::vector OpenIdProviders; -}; - -struct ZenObjectStoreConfig -{ - struct BucketConfig - { - std::string Name; - std::filesystem::path Directory; - }; - - std::vector Buckets; -}; - -struct ZenStructuredCacheBucketConfig -{ - uint64_t MaxBlockSize = 1ull << 30; - uint32_t PayloadAlignment = 1u << 4; - uint64_t MemCacheSizeThreshold = 1 * 1024; - uint64_t LargeObjectThreshold = 128 * 1024; - bool LimitOverwrites = false; -}; - -struct ZenStructuredCacheConfig -{ - bool Enabled = true; - bool WriteLogEnabled = false; - bool AccessLogEnabled = false; - std::vector> PerBucketConfigs; - ZenStructuredCacheBucketConfig BucketConfig; - uint64_t MemTargetFootprintBytes = 512 * 1024 * 1024; - uint64_t MemTrimIntervalSeconds = 60; - uint64_t MemMaxAgeSeconds = gsl::narrow(std::chrono::seconds(std::chrono::days(1)).count()); -}; - -struct ZenProjectStoreConfig -{ - bool StoreCacheAttachmentMetaData = false; - bool StoreProjectAttachmentMetaData = false; -}; - -struct ZenBuildStoreConfig -{ - bool Enabled = false; - uint64_t MaxDiskSpaceLimit = 1u * 1024u * 1024u * 1024u * 1024u; // 1TB -}; - -struct ZenWorkspacesConfig -{ - bool Enabled = false; - bool AllowConfigurationChanges = false; -}; - -struct ZenStorageServerOptions : public ZenServerOptions -{ - ZenUpstreamCacheConfig UpstreamCacheConfig; - ZenGcConfig GcConfig; - ZenAuthConfig AuthConfig; - ZenObjectStoreConfig ObjectStoreConfig; - ZenStructuredCacheConfig StructuredCacheConfig; - ZenProjectStoreConfig ProjectStoreConfig; - ZenBuildStoreConfig BuildStoreConfig; - ZenWorkspacesConfig WorksSpacesConfig; - std::filesystem::path PluginsConfigFile; // Path to plugins config file - std::filesystem::path BaseSnapshotDir; // Path to server state snapshot (will be copied into data dir on start) - bool ObjectStoreEnabled = false; - std::string ScrubOptions; -}; - -void ParseConfigFile(const std::filesystem::path& Path, - ZenStorageServerOptions& ServerOptions, - const cxxopts::ParseResult& CmdLineResult, - std::string_view OutputConfigFile); - -void ParsePluginsConfigFile(const std::filesystem::path& Path, ZenStorageServerOptions& ServerOptions, int BasePort); -void ValidateOptions(ZenStorageServerOptions& ServerOptions); - -struct ZenStorageServerCmdLineOptions -{ - // Note to those adding future options; std::filesystem::path-type options - // must be read into a std::string first. As of cxxopts-3.0.0 it uses a >> - // stream operator to convert argv value into the options type. std::fs::path - // expects paths in streams to be quoted but argv paths are unquoted. By - // going into a std::string first, paths with whitespace parse correctly. - std::string PluginsConfigFile; - std::string BaseSnapshotDir; - - void AddCliOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions); - void ApplyOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions); - - std::string OpenIdProviderName; - std::string OpenIdProviderUrl; - std::string OpenIdClientId; - - void AddSecurityOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions); - - std::string UpstreamCachePolicyOptions; - - void AddCacheOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions); - - void AddGcOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions); - - std::vector BucketConfigs; - - void AddObjectStoreOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions); - void AddBuildStoreOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions); - void AddWorkspacesOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions); -}; - -} // namespace zen diff --git a/src/zenserver/upstream/upstream.h b/src/zenserver/upstream/upstream.h deleted file mode 100644 index 4d45687fc..000000000 --- a/src/zenserver/upstream/upstream.h +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include -#include -#include diff --git a/src/zenserver/upstream/upstreamcache.cpp b/src/zenserver/upstream/upstreamcache.cpp deleted file mode 100644 index 8558e2a10..000000000 --- a/src/zenserver/upstream/upstreamcache.cpp +++ /dev/null @@ -1,2134 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "upstreamcache.h" -#include "zen.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include -#include - -#include -#include - -#include "cache/httpstructuredcache.h" -#include "diag/logging.h" - -#include - -#include -#include -#include -#include - -namespace zen { - -using namespace std::literals; - -namespace detail { - - class UpstreamStatus - { - public: - UpstreamEndpointState EndpointState() const { return static_cast(m_State.load(std::memory_order_relaxed)); } - - UpstreamEndpointStatus EndpointStatus() const - { - const UpstreamEndpointState State = EndpointState(); - { - std::unique_lock _(m_Mutex); - return {.Reason = m_ErrorText, .State = State}; - } - } - - void Set(UpstreamEndpointState NewState) - { - m_State.store(static_cast(NewState), std::memory_order_relaxed); - { - std::unique_lock _(m_Mutex); - m_ErrorText.clear(); - } - } - - void Set(UpstreamEndpointState NewState, std::string ErrorText) - { - m_State.store(static_cast(NewState), std::memory_order_relaxed); - { - std::unique_lock _(m_Mutex); - m_ErrorText = std::move(ErrorText); - } - } - - void SetFromErrorCode(int32_t ErrorCode, std::string_view ErrorText) - { - if (ErrorCode != 0) - { - Set(ErrorCode == 401 ? UpstreamEndpointState::kUnauthorized : UpstreamEndpointState::kError, std::string(ErrorText)); - } - } - - private: - mutable std::mutex m_Mutex; - std::string m_ErrorText; - std::atomic_uint32_t m_State; - }; - - class JupiterUpstreamEndpoint final : public UpstreamEndpoint - { - public: - JupiterUpstreamEndpoint(const JupiterClientOptions& Options, const UpstreamAuthConfig& AuthConfig, AuthMgr& Mgr) - : m_AuthMgr(Mgr) - , m_Log(zen::logging::Get("upstream")) - { - ZEN_ASSERT(!Options.Name.empty()); - m_Info.Name = Options.Name; - m_Info.Url = Options.ServiceUrl; - - std::function TokenProvider; - - if (AuthConfig.OAuthUrl.empty() == false) - { - TokenProvider = httpclientauth::CreateFromOAuthClientCredentials( - {.Url = AuthConfig.OAuthUrl, .ClientId = AuthConfig.OAuthClientId, .ClientSecret = AuthConfig.OAuthClientSecret}); - } - else if (!AuthConfig.OpenIdProvider.empty()) - { - TokenProvider = httpclientauth::CreateFromOpenIdProvider(m_AuthMgr, AuthConfig.OpenIdProvider); - } - else if (!AuthConfig.AccessToken.empty()) - { - TokenProvider = httpclientauth::CreateFromStaticToken(AuthConfig.AccessToken); - } - else - { - TokenProvider = httpclientauth::CreateFromDefaultOpenIdProvider(m_AuthMgr); - } - - m_Client = new JupiterClient(Options, std::move(TokenProvider)); - } - - virtual ~JupiterUpstreamEndpoint() {} - - virtual const UpstreamEndpointInfo& GetEndpointInfo() const override { return m_Info; } - - virtual UpstreamEndpointStatus Initialize() override - { - ZEN_TRACE_CPU("Upstream::Jupiter::Initialize"); - - try - { - if (m_Status.EndpointState() == UpstreamEndpointState::kOk) - { - return {.State = UpstreamEndpointState::kOk}; - } - - JupiterSession Session(m_Client->Logger(), m_Client->Client(), m_AllowRedirect); - const JupiterResult Result = Session.Authenticate(); - - if (Result.Success) - { - m_Status.Set(UpstreamEndpointState::kOk); - } - else if (Result.ErrorCode != 0) - { - m_Status.SetFromErrorCode(Result.ErrorCode, Result.Reason); - } - else - { - m_Status.Set(UpstreamEndpointState::kUnauthorized); - } - - return m_Status.EndpointStatus(); - } - catch (const std::exception& Err) - { - m_Status.Set(UpstreamEndpointState::kError, Err.what()); - - return {.Reason = Err.what(), .State = GetState()}; - } - } - - std::string_view GetActualBlobStoreNamespace(std::string_view Namespace) - { - if (Namespace == ZenCacheStore::DefaultNamespace) - { - return m_Client->DefaultBlobStoreNamespace(); - } - return Namespace; - } - - virtual UpstreamEndpointState GetState() override { return m_Status.EndpointState(); } - - virtual UpstreamEndpointStatus GetStatus() override { return m_Status.EndpointStatus(); } - - virtual GetUpstreamCacheSingleResult GetCacheRecord(std::string_view Namespace, - const CacheKey& CacheKey, - ZenContentType Type) override - { - ZEN_TRACE_CPU("Upstream::Jupiter::GetSingleCacheRecord"); - - try - { - JupiterSession Session(m_Client->Logger(), m_Client->Client(), m_AllowRedirect); - JupiterResult Result; - - std::string_view BlobStoreNamespace = GetActualBlobStoreNamespace(Namespace); - - if (Type == ZenContentType::kCompressedBinary) - { - Result = Session.GetRef(BlobStoreNamespace, CacheKey.Bucket, CacheKey.Hash, ZenContentType::kCbObject); - - if (Result.Success) - { - const CbValidateError ValidationResult = ValidateCompactBinary(Result.Response, CbValidateMode::All); - if (Result.Success = ValidationResult == CbValidateError::None; Result.Success) - { - CbObject CacheRecord = LoadCompactBinaryObject(Result.Response); - IoBuffer ContentBuffer; - int NumAttachments = 0; - - CacheRecord.IterateAttachments([&](CbFieldView AttachmentHash) { - JupiterResult AttachmentResult = Session.GetCompressedBlob(BlobStoreNamespace, AttachmentHash.AsHash()); - Result.ReceivedBytes += AttachmentResult.ReceivedBytes; - Result.SentBytes += AttachmentResult.SentBytes; - Result.ElapsedSeconds += AttachmentResult.ElapsedSeconds; - Result.ErrorCode = AttachmentResult.ErrorCode; - - IoHash RawHash; - uint64_t RawSize; - if (CompressedBuffer::ValidateCompressedHeader(AttachmentResult.Response, RawHash, RawSize)) - { - Result.Response = AttachmentResult.Response; - ++NumAttachments; - } - else - { - Result.Success = false; - } - }); - if (NumAttachments != 1) - { - Result.Success = false; - } - } - } - } - else - { - const ZenContentType AcceptType = Type == ZenContentType::kCbPackage ? ZenContentType::kCbObject : Type; - Result = Session.GetRef(BlobStoreNamespace, CacheKey.Bucket, CacheKey.Hash, AcceptType); - - if (Result.Success && Type == ZenContentType::kCbPackage) - { - CbPackage Package; - - const CbValidateError ValidationResult = ValidateCompactBinary(Result.Response, CbValidateMode::All); - if (Result.Success = ValidationResult == CbValidateError::None; Result.Success) - { - CbObject CacheRecord = LoadCompactBinaryObject(Result.Response); - - CacheRecord.IterateAttachments([&](CbFieldView AttachmentHash) { - JupiterResult AttachmentResult = Session.GetCompressedBlob(BlobStoreNamespace, AttachmentHash.AsHash()); - Result.ReceivedBytes += AttachmentResult.ReceivedBytes; - Result.SentBytes += AttachmentResult.SentBytes; - Result.ElapsedSeconds += AttachmentResult.ElapsedSeconds; - Result.ErrorCode = AttachmentResult.ErrorCode; - - IoHash RawHash; - uint64_t RawSize; - if (CompressedBuffer Chunk = - CompressedBuffer::FromCompressed(SharedBuffer(AttachmentResult.Response), RawHash, RawSize)) - { - Package.AddAttachment(CbAttachment(Chunk, AttachmentHash.AsHash())); - } - else - { - Result.Success = false; - } - }); - - Package.SetObject(CacheRecord); - } - - if (Result.Success) - { - BinaryWriter MemStream; - Package.Save(MemStream); - - Result.Response = IoBuffer(IoBuffer::Clone, MemStream.Data(), MemStream.Size()); - } - } - } - - m_Status.SetFromErrorCode(Result.ErrorCode, Result.Reason); - - if (Result.ErrorCode == 0) - { - return {.Status = {.Bytes = gsl::narrow(Result.ReceivedBytes), - .ElapsedSeconds = Result.ElapsedSeconds, - .Success = Result.Success}, - .Value = Result.Response, - .Source = &m_Info}; - } - else - { - return {.Status = {.Error{.ErrorCode = Result.ErrorCode, .Reason = std::move(Result.Reason)}}}; - } - } - catch (const std::exception& Err) - { - m_Status.Set(UpstreamEndpointState::kError, Err.what()); - - return {.Status = {.Error{.ErrorCode = -1, .Reason = Err.what()}}}; - } - } - - virtual GetUpstreamCacheResult GetCacheRecords(std::string_view Namespace, - std::span Requests, - OnCacheRecordGetComplete&& OnComplete) override - { - ZEN_TRACE_CPU("Upstream::Jupiter::GetCacheRecords"); - - JupiterSession Session(m_Client->Logger(), m_Client->Client(), m_AllowRedirect); - GetUpstreamCacheResult Result; - - for (CacheKeyRequest* Request : Requests) - { - const CacheKey& CacheKey = Request->Key; - CbPackage Package; - CbObject Record; - - double ElapsedSeconds = 0.0; - if (!Result.Error) - { - std::string_view BlobStoreNamespace = GetActualBlobStoreNamespace(Namespace); - JupiterResult RefResult = Session.GetRef(BlobStoreNamespace, CacheKey.Bucket, CacheKey.Hash, ZenContentType::kCbObject); - AppendResult(RefResult, Result); - ElapsedSeconds = RefResult.ElapsedSeconds; - - m_Status.SetFromErrorCode(RefResult.ErrorCode, RefResult.Reason); - - if (RefResult.ErrorCode == 0) - { - const CbValidateError ValidationResult = ValidateCompactBinary(RefResult.Response, CbValidateMode::All); - if (ValidationResult == CbValidateError::None) - { - Record = LoadCompactBinaryObject(RefResult.Response); - Record.IterateAttachments([&](CbFieldView AttachmentHash) { - JupiterResult BlobResult = Session.GetCompressedBlob(BlobStoreNamespace, AttachmentHash.AsHash()); - AppendResult(BlobResult, Result); - - m_Status.SetFromErrorCode(BlobResult.ErrorCode, BlobResult.Reason); - - if (BlobResult.ErrorCode == 0) - { - IoHash RawHash; - uint64_t RawSize; - if (CompressedBuffer Chunk = - CompressedBuffer::FromCompressed(SharedBuffer(BlobResult.Response), RawHash, RawSize)) - { - if (RawHash == AttachmentHash.AsHash()) - { - Package.AddAttachment(CbAttachment(Chunk, RawHash)); - } - } - } - }); - } - } - } - - OnComplete( - {.Request = *Request, .Record = Record, .Package = Package, .ElapsedSeconds = ElapsedSeconds, .Source = &m_Info}); - } - - return Result; - } - - virtual GetUpstreamCacheSingleResult GetCacheChunk(std::string_view Namespace, - const CacheKey&, - const IoHash& ValueContentId) override - { - ZEN_TRACE_CPU("Upstream::Jupiter::GetSingleCacheChunk"); - - try - { - JupiterSession Session(m_Client->Logger(), m_Client->Client(), m_AllowRedirect); - std::string_view BlobStoreNamespace = GetActualBlobStoreNamespace(Namespace); - const JupiterResult Result = Session.GetCompressedBlob(BlobStoreNamespace, ValueContentId); - - m_Status.SetFromErrorCode(Result.ErrorCode, Result.Reason); - - if (Result.ErrorCode == 0) - { - return {.Status = {.Bytes = gsl::narrow(Result.ReceivedBytes), - .ElapsedSeconds = Result.ElapsedSeconds, - .Success = Result.Success}, - .Value = Result.Response, - .Source = &m_Info}; - } - else - { - return {.Status = {.Error{.ErrorCode = Result.ErrorCode, .Reason = std::move(Result.Reason)}}}; - } - } - catch (const std::exception& Err) - { - m_Status.Set(UpstreamEndpointState::kError, Err.what()); - - return {.Status = {.Error{.ErrorCode = -1, .Reason = Err.what()}}}; - } - } - - virtual GetUpstreamCacheResult GetCacheChunks(std::string_view Namespace, - std::span CacheChunkRequests, - OnCacheChunksGetComplete&& OnComplete) override final - { - ZEN_TRACE_CPU("Upstream::Jupiter::GetCacheChunks"); - - JupiterSession Session(m_Client->Logger(), m_Client->Client(), m_AllowRedirect); - GetUpstreamCacheResult Result; - - for (CacheChunkRequest* RequestPtr : CacheChunkRequests) - { - CacheChunkRequest& Request = *RequestPtr; - IoBuffer Payload; - IoHash RawHash = IoHash::Zero; - uint64_t RawSize = 0; - - double ElapsedSeconds = 0.0; - bool IsCompressed = false; - if (!Result.Error) - { - std::string_view BlobStoreNamespace = GetActualBlobStoreNamespace(Namespace); - const JupiterResult BlobResult = - Request.ChunkId == IoHash::Zero - ? Session.GetInlineBlob(BlobStoreNamespace, Request.Key.Bucket, Request.Key.Hash, Request.ChunkId) - : Session.GetCompressedBlob(BlobStoreNamespace, Request.ChunkId); - ElapsedSeconds = BlobResult.ElapsedSeconds; - Payload = BlobResult.Response; - - AppendResult(BlobResult, Result); - - m_Status.SetFromErrorCode(BlobResult.ErrorCode, BlobResult.Reason); - if (Payload && IsCompressedBinary(Payload.GetContentType())) - { - IsCompressed = CompressedBuffer::ValidateCompressedHeader(Payload, RawHash, RawSize); - } - } - - if (IsCompressed) - { - OnComplete({.Request = Request, - .RawHash = RawHash, - .RawSize = RawSize, - .Value = Payload, - .ElapsedSeconds = ElapsedSeconds, - .Source = &m_Info}); - } - else - { - OnComplete({.Request = Request, .RawHash = IoHash::Zero, .RawSize = 0, .Value = IoBuffer()}); - } - } - - return Result; - } - - virtual GetUpstreamCacheResult GetCacheValues(std::string_view Namespace, - std::span CacheValueRequests, - OnCacheValueGetComplete&& OnComplete) override final - { - ZEN_TRACE_CPU("Upstream::Jupiter::GetCacheValues"); - - JupiterSession Session(m_Client->Logger(), m_Client->Client(), m_AllowRedirect); - GetUpstreamCacheResult Result; - - for (CacheValueRequest* RequestPtr : CacheValueRequests) - { - CacheValueRequest& Request = *RequestPtr; - IoBuffer Payload; - IoHash RawHash = IoHash::Zero; - uint64_t RawSize = 0; - - double ElapsedSeconds = 0.0; - bool IsCompressed = false; - if (!Result.Error) - { - std::string_view BlobStoreNamespace = GetActualBlobStoreNamespace(Namespace); - IoHash PayloadHash; - const JupiterResult BlobResult = - Session.GetInlineBlob(BlobStoreNamespace, Request.Key.Bucket, Request.Key.Hash, PayloadHash); - ElapsedSeconds = BlobResult.ElapsedSeconds; - Payload = BlobResult.Response; - - AppendResult(BlobResult, Result); - - m_Status.SetFromErrorCode(BlobResult.ErrorCode, BlobResult.Reason); - if (Payload) - { - if (IsCompressedBinary(Payload.GetContentType())) - { - IsCompressed = CompressedBuffer::ValidateCompressedHeader(Payload, RawHash, RawSize) && RawHash != PayloadHash; - } - else - { - CompressedBuffer Compressed = CompressedBuffer::Compress(SharedBuffer(Payload)); - RawHash = Compressed.DecodeRawHash(); - if (RawHash == PayloadHash) - { - IsCompressed = true; - } - else - { - ZEN_WARN("Jupiter request for inline payload of {}/{}/{} has hash {}, expected hash {} from header", - Namespace, - Request.Key.Bucket, - Request.Key.Hash.ToHexString(), - RawHash.ToHexString(), - PayloadHash.ToHexString()); - } - } - } - } - - if (IsCompressed) - { - OnComplete({.Request = Request, - .RawHash = RawHash, - .RawSize = RawSize, - .Value = Payload, - .ElapsedSeconds = ElapsedSeconds, - .Source = &m_Info}); - } - else - { - OnComplete({.Request = Request, .RawHash = IoHash::Zero, .RawSize = 0, .Value = IoBuffer()}); - } - } - - return Result; - } - - virtual PutUpstreamCacheResult PutCacheRecord(const UpstreamCacheRecord& CacheRecord, - IoBuffer RecordValue, - std::span Values) override - { - ZEN_TRACE_CPU("Upstream::Jupiter::PutCacheRecord"); - - ZEN_ASSERT(CacheRecord.ValueContentIds.size() == Values.size()); - const int32_t MaxAttempts = 3; - - try - { - JupiterSession Session(m_Client->Logger(), m_Client->Client(), m_AllowRedirect); - - if (CacheRecord.Type == ZenContentType::kBinary) - { - JupiterResult Result; - for (uint32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) - { - std::string_view BlobStoreNamespace = GetActualBlobStoreNamespace(CacheRecord.Namespace); - Result = Session.PutRef(BlobStoreNamespace, - CacheRecord.Key.Bucket, - CacheRecord.Key.Hash, - RecordValue, - ZenContentType::kBinary); - } - - m_Status.SetFromErrorCode(Result.ErrorCode, Result.Reason); - - return {.Reason = std::move(Result.Reason), - .Bytes = gsl::narrow(Result.ReceivedBytes), - .ElapsedSeconds = Result.ElapsedSeconds, - .Success = Result.Success}; - } - else if (CacheRecord.Type == ZenContentType::kCompressedBinary) - { - IoHash RawHash; - uint64_t RawSize; - if (!CompressedBuffer::ValidateCompressedHeader(RecordValue, RawHash, RawSize)) - { - return {.Reason = std::string("Invalid compressed value buffer"), .Success = false}; - } - - CbObjectWriter ReferencingObject; - ReferencingObject.AddBinaryAttachment("RawHash", RawHash); - ReferencingObject.AddInteger("RawSize", RawSize); - - return PerformStructuredPut( - Session, - CacheRecord.Namespace, - CacheRecord.Key, - ReferencingObject.Save().GetBuffer().AsIoBuffer(), - MaxAttempts, - [&](const IoHash& ValueContentId, IoBuffer& OutBuffer, std::string& OutReason) { - if (ValueContentId != RawHash) - { - OutReason = - fmt::format("Value '{}' MISMATCHED from compressed buffer raw hash {}", ValueContentId, RawHash); - return false; - } - - OutBuffer = RecordValue; - return true; - }); - } - else - { - return PerformStructuredPut( - Session, - CacheRecord.Namespace, - CacheRecord.Key, - RecordValue, - MaxAttempts, - [&](const IoHash& ValueContentId, IoBuffer& OutBuffer, std::string& OutReason) { - const auto It = - std::find(std::begin(CacheRecord.ValueContentIds), std::end(CacheRecord.ValueContentIds), ValueContentId); - - if (It == std::end(CacheRecord.ValueContentIds)) - { - OutReason = fmt::format("value '{}' MISSING from local cache", ValueContentId); - return false; - } - - const size_t Idx = std::distance(std::begin(CacheRecord.ValueContentIds), It); - - OutBuffer = Values[Idx]; - return true; - }); - } - } - catch (const std::exception& Err) - { - m_Status.Set(UpstreamEndpointState::kError, Err.what()); - - return {.Reason = std::string(Err.what()), .Success = false}; - } - } - - virtual UpstreamEndpointStats& Stats() override { return m_Stats; } - - private: - static void AppendResult(const JupiterResult& Result, GetUpstreamCacheResult& Out) - { - Out.Success &= Result.Success; - Out.Bytes += gsl::narrow(Result.ReceivedBytes); - Out.ElapsedSeconds += Result.ElapsedSeconds; - - if (Result.ErrorCode) - { - Out.Error = {.ErrorCode = Result.ErrorCode, .Reason = std::move(Result.Reason)}; - } - }; - - PutUpstreamCacheResult PerformStructuredPut( - JupiterSession& Session, - std::string_view Namespace, - const CacheKey& Key, - IoBuffer ObjectBuffer, - const int32_t MaxAttempts, - std::function&& BlobFetchFn) - { - int64_t TotalBytes = 0ull; - double TotalElapsedSeconds = 0.0; - - std::string_view BlobStoreNamespace = GetActualBlobStoreNamespace(Namespace); - const auto PutBlobs = [&](std::span ValueContentIds, std::string& OutReason) -> bool { - for (const IoHash& ValueContentId : ValueContentIds) - { - IoBuffer BlobBuffer; - if (!BlobFetchFn(ValueContentId, BlobBuffer, OutReason)) - { - return false; - } - - JupiterResult BlobResult; - for (int32_t Attempt = 0; Attempt < MaxAttempts && !BlobResult.Success; Attempt++) - { - BlobResult = Session.PutCompressedBlob(BlobStoreNamespace, ValueContentId, BlobBuffer); - } - - m_Status.SetFromErrorCode(BlobResult.ErrorCode, BlobResult.Reason); - - if (!BlobResult.Success) - { - OutReason = fmt::format("upload value '{}' FAILED, reason '{}'", ValueContentId, BlobResult.Reason); - return false; - } - - TotalBytes += gsl::narrow(BlobResult.ReceivedBytes); - TotalElapsedSeconds += BlobResult.ElapsedSeconds; - } - - return true; - }; - - PutRefResult RefResult; - for (int32_t Attempt = 0; Attempt < MaxAttempts && !RefResult.Success; Attempt++) - { - RefResult = Session.PutRef(BlobStoreNamespace, Key.Bucket, Key.Hash, ObjectBuffer, ZenContentType::kCbObject); - } - - m_Status.SetFromErrorCode(RefResult.ErrorCode, RefResult.Reason); - - if (!RefResult.Success) - { - return {.Reason = fmt::format("upload cache record '{}/{}' FAILED, reason '{}'", Key.Bucket, Key.Hash, RefResult.Reason), - .Success = false}; - } - - TotalBytes += gsl::narrow(RefResult.ReceivedBytes); - TotalElapsedSeconds += RefResult.ElapsedSeconds; - - std::string Reason; - if (!PutBlobs(RefResult.Needs, Reason)) - { - return {.Reason = std::move(Reason), .Success = false}; - } - - const IoHash RefHash = IoHash::HashBuffer(ObjectBuffer); - FinalizeRefResult FinalizeResult = Session.FinalizeRef(BlobStoreNamespace, Key.Bucket, Key.Hash, RefHash); - - m_Status.SetFromErrorCode(FinalizeResult.ErrorCode, FinalizeResult.Reason); - - if (!FinalizeResult.Success) - { - return { - .Reason = fmt::format("finalize cache record '{}/{}' FAILED, reason '{}'", Key.Bucket, Key.Hash, FinalizeResult.Reason), - .Success = false}; - } - - if (!FinalizeResult.Needs.empty()) - { - if (!PutBlobs(FinalizeResult.Needs, Reason)) - { - return {.Reason = std::move(Reason), .Success = false}; - } - - FinalizeResult = Session.FinalizeRef(BlobStoreNamespace, Key.Bucket, Key.Hash, RefHash); - - m_Status.SetFromErrorCode(FinalizeResult.ErrorCode, FinalizeResult.Reason); - - if (!FinalizeResult.Success) - { - return {.Reason = fmt::format("finalize '{}/{}' FAILED, reason '{}'", Key.Bucket, Key.Hash, FinalizeResult.Reason), - .Success = false}; - } - - if (!FinalizeResult.Needs.empty()) - { - ExtendableStringBuilder<256> Sb; - for (const IoHash& MissingHash : FinalizeResult.Needs) - { - Sb << MissingHash.ToHexString() << ","; - } - - return { - .Reason = fmt::format("finalize '{}/{}' FAILED, still needs value(s) '{}'", Key.Bucket, Key.Hash, Sb.ToString()), - .Success = false}; - } - } - - TotalBytes += gsl::narrow(FinalizeResult.ReceivedBytes); - TotalElapsedSeconds += FinalizeResult.ElapsedSeconds; - - return {.Bytes = TotalBytes, .ElapsedSeconds = TotalElapsedSeconds, .Success = true}; - } - - LoggerRef Log() { return m_Log; } - - AuthMgr& m_AuthMgr; - LoggerRef m_Log; - UpstreamEndpointInfo m_Info; - UpstreamStatus m_Status; - UpstreamEndpointStats m_Stats; - RefPtr m_Client; - const bool m_AllowRedirect = false; - }; - - class ZenUpstreamEndpoint final : public UpstreamEndpoint - { - struct ZenEndpoint - { - std::string Url; - std::string Reason; - double Latency{}; - bool Ok = false; - - bool operator<(const ZenEndpoint& RHS) const { return Ok && RHS.Ok ? Latency < RHS.Latency : Ok; } - }; - - public: - ZenUpstreamEndpoint(const ZenStructuredCacheClientOptions& Options) - : m_Log(zen::logging::Get("upstream")) - , m_ConnectTimeout(Options.ConnectTimeout) - , m_Timeout(Options.Timeout) - { - ZEN_ASSERT(!Options.Name.empty()); - m_Info.Name = Options.Name; - - for (const auto& Url : Options.Urls) - { - m_Endpoints.push_back({.Url = Url}); - } - } - - ~ZenUpstreamEndpoint() {} - - virtual const UpstreamEndpointInfo& GetEndpointInfo() const override { return m_Info; } - - virtual UpstreamEndpointStatus Initialize() override - { - ZEN_TRACE_CPU("Upstream::Zen::Initialize"); - - try - { - if (m_Status.EndpointState() == UpstreamEndpointState::kOk) - { - return {.State = UpstreamEndpointState::kOk}; - } - - const ZenEndpoint& Ep = GetEndpoint(); - - if (m_Info.Url != Ep.Url) - { - ZEN_INFO("Setting Zen upstream URL to '{}'", Ep.Url); - m_Info.Url = Ep.Url; - } - - if (Ep.Ok) - { - RwLock::ExclusiveLockScope _(m_ClientLock); - m_Client = new ZenStructuredCacheClient({.Url = m_Info.Url, .ConnectTimeout = m_ConnectTimeout, .Timeout = m_Timeout}); - m_Status.Set(UpstreamEndpointState::kOk); - } - else - { - m_Status.Set(UpstreamEndpointState::kError, Ep.Reason); - } - - return m_Status.EndpointStatus(); - } - catch (const std::exception& Err) - { - m_Status.Set(UpstreamEndpointState::kError, Err.what()); - - return {.Reason = Err.what(), .State = GetState()}; - } - } - - virtual UpstreamEndpointState GetState() override { return m_Status.EndpointState(); } - - virtual UpstreamEndpointStatus GetStatus() override { return m_Status.EndpointStatus(); } - - virtual GetUpstreamCacheSingleResult GetCacheRecord(std::string_view Namespace, - const CacheKey& CacheKey, - ZenContentType Type) override - { - ZEN_TRACE_CPU("Upstream::Zen::GetSingleCacheRecord"); - - try - { - ZenStructuredCacheSession Session(GetClientRef()); - const ZenCacheResult Result = Session.GetCacheRecord(Namespace, CacheKey.Bucket, CacheKey.Hash, Type); - - m_Status.SetFromErrorCode(Result.ErrorCode, Result.Reason); - - if (Result.ErrorCode == 0) - { - return {.Status = {.Bytes = Result.Bytes, .ElapsedSeconds = Result.ElapsedSeconds, .Success = Result.Success}, - .Value = Result.Response, - .Source = &m_Info}; - } - else - { - return {.Status = {.Error{.ErrorCode = Result.ErrorCode, .Reason = std::move(Result.Reason)}}}; - } - } - catch (const std::exception& Err) - { - m_Status.Set(UpstreamEndpointState::kError, Err.what()); - - return {.Status = {.Error{.ErrorCode = -1, .Reason = Err.what()}}}; - } - } - - virtual GetUpstreamCacheResult GetCacheRecords(std::string_view Namespace, - std::span Requests, - OnCacheRecordGetComplete&& OnComplete) override - { - ZEN_TRACE_CPU("Upstream::Zen::GetCacheRecords"); - ZEN_ASSERT(Requests.size() > 0); - - CbObjectWriter BatchRequest; - BatchRequest << "Method"sv - << "GetCacheRecords"sv; - BatchRequest << "Accept"sv << kCbPkgMagic; - - BatchRequest.BeginObject("Params"sv); - { - CachePolicy DefaultPolicy = Requests[0]->Policy.GetRecordPolicy(); - BatchRequest << "DefaultPolicy"sv << WriteToString<128>(DefaultPolicy); - - BatchRequest << "Namespace"sv << Namespace; - - BatchRequest.BeginArray("Requests"sv); - for (CacheKeyRequest* Request : Requests) - { - BatchRequest.BeginObject(); - { - const CacheKey& Key = Request->Key; - BatchRequest.BeginObject("Key"sv); - { - BatchRequest << "Bucket"sv << Key.Bucket; - BatchRequest << "Hash"sv << Key.Hash; - } - BatchRequest.EndObject(); - if (!Request->Policy.IsUniform() || Request->Policy.GetRecordPolicy() != DefaultPolicy) - { - BatchRequest.SetName("Policy"sv); - Request->Policy.Save(BatchRequest); - } - } - BatchRequest.EndObject(); - } - BatchRequest.EndArray(); - } - BatchRequest.EndObject(); - - ZenCacheResult Result; - - { - ZenStructuredCacheSession Session(GetClientRef()); - Result = Session.InvokeRpc(BatchRequest.Save()); - } - - m_Status.SetFromErrorCode(Result.ErrorCode, Result.Reason); - - if (Result.Success) - { - CbPackage BatchResponse; - if (ParsePackageMessageWithLegacyFallback(Result.Response, BatchResponse)) - { - CbArrayView Results = BatchResponse.GetObject()["Result"sv].AsArrayView(); - if (Results.Num() != Requests.size()) - { - ZEN_WARN("Upstream::Zen::GetCacheRecords invalid number of Response results from Upstream."); - } - else - { - for (size_t Index = 0; CbFieldView Record : Results) - { - CacheKeyRequest* Request = Requests[Index++]; - OnComplete({.Request = *Request, - .Record = Record.AsObjectView(), - .Package = BatchResponse, - .ElapsedSeconds = Result.ElapsedSeconds, - .Source = &m_Info}); - } - - return {.Bytes = Result.Bytes, .ElapsedSeconds = Result.ElapsedSeconds, .Success = true}; - } - } - else - { - ZEN_WARN("Upstream::Zen::GetCacheRecords invalid Response from Upstream."); - } - } - - for (CacheKeyRequest* Request : Requests) - { - OnComplete({.Request = *Request, .Record = CbObjectView(), .Package = CbPackage()}); - } - - return {.Error{.ErrorCode = Result.ErrorCode, .Reason = std::move(Result.Reason)}}; - } - - virtual GetUpstreamCacheSingleResult GetCacheChunk(std::string_view Namespace, - const CacheKey& CacheKey, - const IoHash& ValueContentId) override - { - ZEN_TRACE_CPU("Upstream::Zen::GetCacheChunk"); - - try - { - ZenStructuredCacheSession Session(GetClientRef()); - const ZenCacheResult Result = Session.GetCacheChunk(Namespace, CacheKey.Bucket, CacheKey.Hash, ValueContentId); - - m_Status.SetFromErrorCode(Result.ErrorCode, Result.Reason); - - if (Result.ErrorCode == 0) - { - return {.Status = {.Bytes = Result.Bytes, .ElapsedSeconds = Result.ElapsedSeconds, .Success = Result.Success}, - .Value = Result.Response, - .Source = &m_Info}; - } - else - { - return {.Status = {.Error{.ErrorCode = Result.ErrorCode, .Reason = std::move(Result.Reason)}}}; - } - } - catch (const std::exception& Err) - { - m_Status.Set(UpstreamEndpointState::kError, Err.what()); - - return {.Status = {.Error{.ErrorCode = -1, .Reason = Err.what()}}}; - } - } - - virtual GetUpstreamCacheResult GetCacheValues(std::string_view Namespace, - std::span CacheValueRequests, - OnCacheValueGetComplete&& OnComplete) override final - { - ZEN_TRACE_CPU("Upstream::Zen::GetCacheValues"); - ZEN_ASSERT(!CacheValueRequests.empty()); - - CbObjectWriter BatchRequest; - BatchRequest << "Method"sv - << "GetCacheValues"sv; - BatchRequest << "Accept"sv << kCbPkgMagic; - - BatchRequest.BeginObject("Params"sv); - { - CachePolicy DefaultPolicy = CacheValueRequests[0]->Policy; - BatchRequest << "DefaultPolicy"sv << WriteToString<128>(DefaultPolicy).ToView(); - BatchRequest << "Namespace"sv << Namespace; - - BatchRequest.BeginArray("Requests"sv); - { - for (CacheValueRequest* RequestPtr : CacheValueRequests) - { - const CacheValueRequest& Request = *RequestPtr; - - BatchRequest.BeginObject(); - { - BatchRequest.BeginObject("Key"sv); - BatchRequest << "Bucket"sv << Request.Key.Bucket; - BatchRequest << "Hash"sv << Request.Key.Hash; - BatchRequest.EndObject(); - if (Request.Policy != DefaultPolicy) - { - BatchRequest << "Policy"sv << WriteToString<128>(Request.Policy).ToView(); - } - } - BatchRequest.EndObject(); - } - } - BatchRequest.EndArray(); - } - BatchRequest.EndObject(); - - ZenCacheResult Result; - - { - ZenStructuredCacheSession Session(GetClientRef()); - Result = Session.InvokeRpc(BatchRequest.Save()); - } - - m_Status.SetFromErrorCode(Result.ErrorCode, Result.Reason); - - if (Result.Success) - { - CbPackage BatchResponse; - if (ParsePackageMessageWithLegacyFallback(Result.Response, BatchResponse)) - { - CbArrayView Results = BatchResponse.GetObject()["Result"sv].AsArrayView(); - if (CacheValueRequests.size() != Results.Num()) - { - ZEN_WARN("Upstream::Zen::GetCacheValues invalid number of Response results from Upstream."); - } - else - { - for (size_t RequestIndex = 0; CbFieldView ChunkField : Results) - { - CacheValueRequest& Request = *CacheValueRequests[RequestIndex++]; - CbObjectView ChunkObject = ChunkField.AsObjectView(); - IoHash RawHash = ChunkObject["RawHash"sv].AsHash(); - IoBuffer Payload; - uint64_t RawSize = 0; - if (RawHash != IoHash::Zero) - { - bool Success = false; - const CbAttachment* Attachment = BatchResponse.FindAttachment(RawHash); - if (Attachment) - { - if (const CompressedBuffer& Compressed = Attachment->AsCompressedBinary()) - { - Payload = Compressed.GetCompressed().Flatten().AsIoBuffer(); - Payload.SetContentType(ZenContentType::kCompressedBinary); - RawSize = Compressed.DecodeRawSize(); - Success = true; - } - } - if (!Success) - { - CbFieldView RawSizeField = ChunkObject["RawSize"sv]; - RawSize = RawSizeField.AsUInt64(); - Success = !RawSizeField.HasError(); - } - if (!Success) - { - RawHash = IoHash::Zero; - } - } - OnComplete({.Request = Request, - .RawHash = RawHash, - .RawSize = RawSize, - .Value = std::move(Payload), - .ElapsedSeconds = Result.ElapsedSeconds, - .Source = &m_Info}); - } - - return {.Bytes = Result.Bytes, .ElapsedSeconds = Result.ElapsedSeconds, .Success = true}; - } - } - else - { - ZEN_WARN("Upstream::Zen::GetCacheValues invalid Response from Upstream."); - } - } - - for (CacheValueRequest* RequestPtr : CacheValueRequests) - { - OnComplete({.Request = *RequestPtr, .RawHash = IoHash::Zero, .RawSize = 0, .Value = IoBuffer()}); - } - - return {.Error{.ErrorCode = Result.ErrorCode, .Reason = std::move(Result.Reason)}}; - } - - virtual GetUpstreamCacheResult GetCacheChunks(std::string_view Namespace, - std::span CacheChunkRequests, - OnCacheChunksGetComplete&& OnComplete) override final - { - ZEN_TRACE_CPU("Upstream::Zen::GetCacheChunks"); - ZEN_ASSERT(!CacheChunkRequests.empty()); - - CbObjectWriter BatchRequest; - BatchRequest << "Method"sv - << "GetCacheChunks"sv; - BatchRequest << "Accept"sv << kCbPkgMagic; - - BatchRequest.BeginObject("Params"sv); - { - CachePolicy DefaultPolicy = CacheChunkRequests[0]->Policy; - BatchRequest << "DefaultPolicy"sv << WriteToString<128>(DefaultPolicy).ToView(); - BatchRequest << "Namespace"sv << Namespace; - - BatchRequest.BeginArray("ChunkRequests"sv); - { - for (CacheChunkRequest* RequestPtr : CacheChunkRequests) - { - const CacheChunkRequest& Request = *RequestPtr; - - BatchRequest.BeginObject(); - { - BatchRequest.BeginObject("Key"sv); - BatchRequest << "Bucket"sv << Request.Key.Bucket; - BatchRequest << "Hash"sv << Request.Key.Hash; - BatchRequest.EndObject(); - if (Request.ValueId) - { - BatchRequest.AddObjectId("ValueId"sv, Request.ValueId); - } - if (Request.ChunkId != Request.ChunkId.Zero) - { - BatchRequest << "ChunkId"sv << Request.ChunkId; - } - if (Request.RawOffset != 0) - { - BatchRequest << "RawOffset"sv << Request.RawOffset; - } - if (Request.RawSize != UINT64_MAX) - { - BatchRequest << "RawSize"sv << Request.RawSize; - } - if (Request.Policy != DefaultPolicy) - { - BatchRequest << "Policy"sv << WriteToString<128>(Request.Policy).ToView(); - } - } - BatchRequest.EndObject(); - } - } - BatchRequest.EndArray(); - } - BatchRequest.EndObject(); - - ZenCacheResult Result; - - { - ZenStructuredCacheSession Session(GetClientRef()); - Result = Session.InvokeRpc(BatchRequest.Save()); - } - - m_Status.SetFromErrorCode(Result.ErrorCode, Result.Reason); - - if (Result.Success) - { - CbPackage BatchResponse; - if (ParsePackageMessageWithLegacyFallback(Result.Response, BatchResponse)) - { - CbArrayView Results = BatchResponse.GetObject()["Result"sv].AsArrayView(); - if (CacheChunkRequests.size() != Results.Num()) - { - ZEN_WARN("Upstream::Zen::GetCacheChunks invalid number of Response results from Upstream."); - } - else - { - for (size_t RequestIndex = 0; CbFieldView ChunkField : Results) - { - CacheChunkRequest& Request = *CacheChunkRequests[RequestIndex++]; - CbObjectView ChunkObject = ChunkField.AsObjectView(); - IoHash RawHash = ChunkObject["RawHash"sv].AsHash(); - IoBuffer Payload; - uint64_t RawSize = 0; - if (RawHash != IoHash::Zero) - { - bool Success = false; - const CbAttachment* Attachment = BatchResponse.FindAttachment(RawHash); - if (Attachment) - { - if (const CompressedBuffer& Compressed = Attachment->AsCompressedBinary()) - { - Payload = Compressed.GetCompressed().Flatten().AsIoBuffer(); - Payload.SetContentType(ZenContentType::kCompressedBinary); - RawSize = Compressed.DecodeRawSize(); - Success = true; - } - } - if (!Success) - { - CbFieldView RawSizeField = ChunkObject["RawSize"sv]; - RawSize = RawSizeField.AsUInt64(); - Success = !RawSizeField.HasError(); - } - if (!Success) - { - RawHash = IoHash::Zero; - } - } - OnComplete({.Request = Request, - .RawHash = RawHash, - .RawSize = RawSize, - .Value = std::move(Payload), - .ElapsedSeconds = Result.ElapsedSeconds, - .Source = &m_Info}); - } - - return {.Bytes = Result.Bytes, .ElapsedSeconds = Result.ElapsedSeconds, .Success = true}; - } - } - else - { - ZEN_WARN("Upstream::Zen::GetCacheChunks invalid Response from Upstream."); - } - } - - for (CacheChunkRequest* RequestPtr : CacheChunkRequests) - { - OnComplete({.Request = *RequestPtr, .RawHash = IoHash::Zero, .RawSize = 0, .Value = IoBuffer()}); - } - - return {.Error{.ErrorCode = Result.ErrorCode, .Reason = std::move(Result.Reason)}}; - } - - virtual PutUpstreamCacheResult PutCacheRecord(const UpstreamCacheRecord& CacheRecord, - IoBuffer RecordValue, - std::span Values) override - { - ZEN_TRACE_CPU("Upstream::Zen::PutCacheRecord"); - - ZEN_ASSERT(CacheRecord.ValueContentIds.size() == Values.size()); - const int32_t MaxAttempts = 3; - - try - { - ZenStructuredCacheSession Session(GetClientRef()); - ZenCacheResult Result; - int64_t TotalBytes = 0ull; - double TotalElapsedSeconds = 0.0; - - if (CacheRecord.Type == ZenContentType::kCbPackage) - { - CbPackage Package; - Package.SetObject(CbObject(SharedBuffer(RecordValue))); - - for (const IoBuffer& Value : Values) - { - IoHash RawHash; - uint64_t RawSize; - if (CompressedBuffer AttachmentBuffer = CompressedBuffer::FromCompressed(SharedBuffer(Value), RawHash, RawSize)) - { - Package.AddAttachment(CbAttachment(AttachmentBuffer, RawHash)); - } - else - { - return {.Reason = std::string("Invalid value buffer"), .Success = false}; - } - } - - BinaryWriter MemStream; - Package.Save(MemStream); - IoBuffer PackagePayload(IoBuffer::Wrap, MemStream.Data(), MemStream.Size()); - - for (uint32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) - { - Result = Session.PutCacheRecord(CacheRecord.Namespace, - CacheRecord.Key.Bucket, - CacheRecord.Key.Hash, - PackagePayload, - CacheRecord.Type); - } - - m_Status.SetFromErrorCode(Result.ErrorCode, Result.Reason); - - TotalBytes = Result.Bytes; - TotalElapsedSeconds = Result.ElapsedSeconds; - } - else if (CacheRecord.Type == ZenContentType::kCompressedBinary) - { - IoHash RawHash; - uint64_t RawSize; - CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(RecordValue), RawHash, RawSize); - if (!Compressed) - { - return {.Reason = std::string("Invalid value compressed buffer"), .Success = false}; - } - - CbPackage BatchPackage; - CbObjectWriter BatchWriter; - BatchWriter << "Method"sv - << "PutCacheValues"sv; - BatchWriter << "Accept"sv << kCbPkgMagic; - - BatchWriter.BeginObject("Params"sv); - { - // DefaultPolicy unspecified and expected to be Default - - BatchWriter << "Namespace"sv << CacheRecord.Namespace; - - BatchWriter.BeginArray("Requests"sv); - { - BatchWriter.BeginObject(); - { - const CacheKey& Key = CacheRecord.Key; - BatchWriter.BeginObject("Key"sv); - { - BatchWriter << "Bucket"sv << Key.Bucket; - BatchWriter << "Hash"sv << Key.Hash; - } - BatchWriter.EndObject(); - // Policy unspecified and expected to be Default - BatchWriter.AddBinaryAttachment("RawHash"sv, RawHash); - BatchPackage.AddAttachment(CbAttachment(Compressed, RawHash)); - } - BatchWriter.EndObject(); - } - BatchWriter.EndArray(); - } - BatchWriter.EndObject(); - BatchPackage.SetObject(BatchWriter.Save()); - - Result.Success = false; - for (uint32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) - { - Result = Session.InvokeRpc(BatchPackage); - } - - m_Status.SetFromErrorCode(Result.ErrorCode, Result.Reason); - - TotalBytes += Result.Bytes; - TotalElapsedSeconds += Result.ElapsedSeconds; - } - else - { - for (size_t Idx = 0, Count = Values.size(); Idx < Count; Idx++) - { - Result.Success = false; - for (uint32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) - { - Result = Session.PutCacheValue(CacheRecord.Namespace, - CacheRecord.Key.Bucket, - CacheRecord.Key.Hash, - CacheRecord.ValueContentIds[Idx], - Values[Idx]); - } - - m_Status.SetFromErrorCode(Result.ErrorCode, Result.Reason); - - TotalBytes += Result.Bytes; - TotalElapsedSeconds += Result.ElapsedSeconds; - - if (!Result.Success) - { - return {.Reason = "Failed to upload value", - .Bytes = TotalBytes, - .ElapsedSeconds = TotalElapsedSeconds, - .Success = false}; - } - } - - Result.Success = false; - for (uint32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) - { - Result = Session.PutCacheRecord(CacheRecord.Namespace, - CacheRecord.Key.Bucket, - CacheRecord.Key.Hash, - RecordValue, - CacheRecord.Type); - } - - m_Status.SetFromErrorCode(Result.ErrorCode, Result.Reason); - - TotalBytes += Result.Bytes; - TotalElapsedSeconds += Result.ElapsedSeconds; - } - - return {.Reason = std::move(Result.Reason), - .Bytes = TotalBytes, - .ElapsedSeconds = TotalElapsedSeconds, - .Success = Result.Success}; - } - catch (const std::exception& Err) - { - m_Status.Set(UpstreamEndpointState::kError, Err.what()); - - return {.Reason = std::string(Err.what()), .Success = false}; - } - } - - virtual UpstreamEndpointStats& Stats() override { return m_Stats; } - - private: - Ref GetClientRef() - { - // m_Client can be modified at any time by a different thread. - // Make sure we safely bump the refcount inside a scope lock - RwLock::SharedLockScope _(m_ClientLock); - ZEN_ASSERT(m_Client); - Ref ClientRef(m_Client); - _.ReleaseNow(); - return ClientRef; - } - - const ZenEndpoint& GetEndpoint() - { - for (ZenEndpoint& Ep : m_Endpoints) - { - Ref Client( - new ZenStructuredCacheClient({.Url = Ep.Url, .ConnectTimeout = std::chrono::milliseconds(1000)})); - ZenStructuredCacheSession Session(std::move(Client)); - const int32_t SampleCount = 2; - - Ep.Ok = false; - Ep.Latency = {}; - - for (int32_t Sample = 0; Sample < SampleCount; ++Sample) - { - ZenCacheResult Result = Session.CheckHealth(); - Ep.Ok = Result.Success; - Ep.Reason = std::move(Result.Reason); - Ep.Latency += Result.ElapsedSeconds; - } - Ep.Latency /= double(SampleCount); - } - - std::sort(std::begin(m_Endpoints), std::end(m_Endpoints)); - - for (const auto& Ep : m_Endpoints) - { - ZEN_INFO("ping 'Zen' endpoint '{}' latency '{:.3}s' {}", Ep.Url, Ep.Latency, Ep.Ok ? "OK" : Ep.Reason); - } - - return m_Endpoints.front(); - } - - LoggerRef Log() { return m_Log; } - - LoggerRef m_Log; - UpstreamEndpointInfo m_Info; - UpstreamStatus m_Status; - UpstreamEndpointStats m_Stats; - std::vector m_Endpoints; - std::chrono::milliseconds m_ConnectTimeout; - std::chrono::milliseconds m_Timeout; - RwLock m_ClientLock; - RefPtr m_Client; - }; - -} // namespace detail - -////////////////////////////////////////////////////////////////////////// - -class UpstreamCacheImpl final : public UpstreamCache -{ -public: - UpstreamCacheImpl(const UpstreamCacheOptions& Options, ZenCacheStore& CacheStore, CidStore& CidStore) - : m_Log(logging::Get("upstream")) - , m_Options(Options) - , m_CacheStore(CacheStore) - , m_CidStore(CidStore) - { - } - - virtual ~UpstreamCacheImpl() { Shutdown(); } - - virtual void Initialize() override - { - ZEN_TRACE_CPU("Upstream::Initialize"); - - m_RunState.IsRunning = true; - } - - virtual bool IsActive() override - { - std::shared_lock _(m_EndpointsMutex); - return !m_Endpoints.empty(); - } - - virtual void RegisterEndpoint(std::unique_ptr Endpoint) override - { - ZEN_TRACE_CPU("Upstream::RegisterEndpoint"); - - const UpstreamEndpointStatus Status = Endpoint->Initialize(); - const UpstreamEndpointInfo& Info = Endpoint->GetEndpointInfo(); - - if (Status.State == UpstreamEndpointState::kOk) - { - ZEN_INFO("register endpoint '{} - {}' {}", Info.Name, Info.Url, ToString(Status.State)); - } - else - { - ZEN_WARN("register endpoint '{} - {}' {}", Info.Name, Info.Url, ToString(Status.State)); - } - - // Register endpoint even if it fails, the health monitor thread will probe failing endpoint(s) - std::unique_lock _(m_EndpointsMutex); - if (m_Endpoints.empty()) - { - for (uint32_t Idx = 0; Idx < m_Options.ThreadCount; Idx++) - { - m_UpstreamThreads.emplace_back(&UpstreamCacheImpl::ProcessUpstreamQueue, this, Idx + 1); - } - - m_EndpointMonitorThread = std::thread(&UpstreamCacheImpl::MonitorEndpoints, this); - } - m_Endpoints.emplace_back(std::move(Endpoint)); - } - - virtual void IterateEndpoints(std::function&& Fn) override - { - std::shared_lock _(m_EndpointsMutex); - - for (auto& Ep : m_Endpoints) - { - if (!Fn(*Ep)) - { - break; - } - } - } - - virtual GetUpstreamCacheSingleResult GetCacheRecord(std::string_view Namespace, const CacheKey& CacheKey, ZenContentType Type) override - { - ZEN_TRACE_CPU("Upstream::GetCacheRecord"); - - std::shared_lock _(m_EndpointsMutex); - - if (m_Options.ReadUpstream) - { - for (auto& Endpoint : m_Endpoints) - { - if (Endpoint->GetState() != UpstreamEndpointState::kOk) - { - continue; - } - - UpstreamEndpointStats& Stats = Endpoint->Stats(); - metrics::OperationTiming::Scope Scope(Stats.CacheGetRequestTiming); - GetUpstreamCacheSingleResult Result = Endpoint->GetCacheRecord(Namespace, CacheKey, Type); - Scope.Stop(); - - Stats.CacheGetCount.Increment(1); - Stats.CacheGetTotalBytes.Increment(Result.Status.Bytes); - - if (Result.Status.Success) - { - Stats.CacheHitCount.Increment(1); - - return Result; - } - - if (Result.Status.Error) - { - Stats.CacheErrorCount.Increment(1); - - ZEN_WARN("get cache record FAILED, endpoint '{}', reason '{}', error code '{}'", - Endpoint->GetEndpointInfo().Url, - Result.Status.Error.Reason, - Result.Status.Error.ErrorCode); - } - } - } - - return {}; - } - - virtual void GetCacheRecords(std::string_view Namespace, - std::span Requests, - OnCacheRecordGetComplete&& OnComplete) override final - { - ZEN_TRACE_CPU("Upstream::GetCacheRecords"); - - std::shared_lock _(m_EndpointsMutex); - - std::vector RemainingKeys(Requests.begin(), Requests.end()); - - if (m_Options.ReadUpstream) - { - for (auto& Endpoint : m_Endpoints) - { - if (RemainingKeys.empty()) - { - break; - } - - if (Endpoint->GetState() != UpstreamEndpointState::kOk) - { - continue; - } - - UpstreamEndpointStats& Stats = Endpoint->Stats(); - std::vector Missing; - GetUpstreamCacheResult Result; - { - metrics::OperationTiming::Scope Scope(Stats.CacheGetRequestTiming); - - Result = Endpoint->GetCacheRecords(Namespace, RemainingKeys, [&](CacheRecordGetCompleteParams&& Params) { - if (Params.Record) - { - OnComplete(std::forward(Params)); - - Stats.CacheHitCount.Increment(1); - } - else - { - Missing.push_back(&Params.Request); - } - }); - } - - Stats.CacheGetCount.Increment(int64_t(RemainingKeys.size())); - Stats.CacheGetTotalBytes.Increment(Result.Bytes); - - if (Result.Error) - { - Stats.CacheErrorCount.Increment(1); - - ZEN_WARN("get cache record(s) (rpc) FAILED, endpoint '{}', reason '{}', error code '{}'", - Endpoint->GetEndpointInfo().Url, - Result.Error.Reason, - Result.Error.ErrorCode); - } - - RemainingKeys = std::move(Missing); - } - } - - const UpstreamEndpointInfo Info; - for (CacheKeyRequest* Request : RemainingKeys) - { - OnComplete({.Request = *Request, .Record = CbObjectView(), .Package = CbPackage()}); - } - } - - virtual void GetCacheChunks(std::string_view Namespace, - std::span CacheChunkRequests, - OnCacheChunksGetComplete&& OnComplete) override final - { - ZEN_TRACE_CPU("Upstream::GetCacheChunks"); - - std::shared_lock _(m_EndpointsMutex); - - std::vector RemainingKeys(CacheChunkRequests.begin(), CacheChunkRequests.end()); - - if (m_Options.ReadUpstream) - { - for (auto& Endpoint : m_Endpoints) - { - if (RemainingKeys.empty()) - { - break; - } - - if (Endpoint->GetState() != UpstreamEndpointState::kOk) - { - continue; - } - - UpstreamEndpointStats& Stats = Endpoint->Stats(); - std::vector Missing; - GetUpstreamCacheResult Result; - { - metrics::OperationTiming::Scope Scope(Endpoint->Stats().CacheGetRequestTiming); - - Result = Endpoint->GetCacheChunks(Namespace, RemainingKeys, [&](CacheChunkGetCompleteParams&& Params) { - if (Params.RawHash != Params.RawHash.Zero) - { - OnComplete(std::forward(Params)); - - Stats.CacheHitCount.Increment(1); - } - else - { - Missing.push_back(&Params.Request); - } - }); - } - - Stats.CacheGetCount.Increment(int64_t(RemainingKeys.size())); - Stats.CacheGetTotalBytes.Increment(Result.Bytes); - - if (Result.Error) - { - Stats.CacheErrorCount.Increment(1); - - ZEN_WARN("get cache chunks(s) (rpc) FAILED, endpoint '{}', reason '{}', error code '{}'", - Endpoint->GetEndpointInfo().Url, - Result.Error.Reason, - Result.Error.ErrorCode); - } - - RemainingKeys = std::move(Missing); - } - } - - const UpstreamEndpointInfo Info; - for (CacheChunkRequest* RequestPtr : RemainingKeys) - { - OnComplete({.Request = *RequestPtr, .RawHash = IoHash::Zero, .RawSize = 0, .Value = IoBuffer()}); - } - } - - virtual GetUpstreamCacheSingleResult GetCacheChunk(std::string_view Namespace, - const CacheKey& CacheKey, - const IoHash& ValueContentId) override - { - ZEN_TRACE_CPU("Upstream::GetCacheChunk"); - - if (m_Options.ReadUpstream) - { - for (auto& Endpoint : m_Endpoints) - { - if (Endpoint->GetState() != UpstreamEndpointState::kOk) - { - continue; - } - - UpstreamEndpointStats& Stats = Endpoint->Stats(); - metrics::OperationTiming::Scope Scope(Stats.CacheGetRequestTiming); - GetUpstreamCacheSingleResult Result = Endpoint->GetCacheChunk(Namespace, CacheKey, ValueContentId); - Scope.Stop(); - - Stats.CacheGetCount.Increment(1); - Stats.CacheGetTotalBytes.Increment(Result.Status.Bytes); - - if (Result.Status.Success) - { - Stats.CacheHitCount.Increment(1); - - return Result; - } - - if (Result.Status.Error) - { - Stats.CacheErrorCount.Increment(1); - - ZEN_WARN("get cache chunk FAILED, endpoint '{}', reason '{}', error code '{}'", - Endpoint->GetEndpointInfo().Url, - Result.Status.Error.Reason, - Result.Status.Error.ErrorCode); - } - } - } - - return {}; - } - - virtual void GetCacheValues(std::string_view Namespace, - std::span CacheValueRequests, - OnCacheValueGetComplete&& OnComplete) override final - { - ZEN_TRACE_CPU("Upstream::GetCacheValues"); - - std::shared_lock _(m_EndpointsMutex); - - std::vector RemainingKeys(CacheValueRequests.begin(), CacheValueRequests.end()); - - if (m_Options.ReadUpstream) - { - for (auto& Endpoint : m_Endpoints) - { - if (RemainingKeys.empty()) - { - break; - } - - if (Endpoint->GetState() != UpstreamEndpointState::kOk) - { - continue; - } - - UpstreamEndpointStats& Stats = Endpoint->Stats(); - std::vector Missing; - GetUpstreamCacheResult Result; - { - metrics::OperationTiming::Scope Scope(Endpoint->Stats().CacheGetRequestTiming); - - Result = Endpoint->GetCacheValues(Namespace, RemainingKeys, [&](CacheValueGetCompleteParams&& Params) { - if (Params.RawHash != Params.RawHash.Zero) - { - OnComplete(std::forward(Params)); - - Stats.CacheHitCount.Increment(1); - } - else - { - Missing.push_back(&Params.Request); - } - }); - } - - Stats.CacheGetCount.Increment(int64_t(RemainingKeys.size())); - Stats.CacheGetTotalBytes.Increment(Result.Bytes); - - if (Result.Error) - { - Stats.CacheErrorCount.Increment(1); - - ZEN_WARN("get cache values(s) (rpc) FAILED, endpoint '{}', reason '{}', error code '{}'", - Endpoint->GetEndpointInfo().Url, - Result.Error.Reason, - Result.Error.ErrorCode); - } - - RemainingKeys = std::move(Missing); - } - } - - const UpstreamEndpointInfo Info; - for (CacheValueRequest* RequestPtr : RemainingKeys) - { - OnComplete({.Request = *RequestPtr, .RawHash = IoHash::Zero, .RawSize = 0, .Value = IoBuffer()}); - } - } - - virtual void EnqueueUpstream(UpstreamCacheRecord CacheRecord) override - { - if (m_RunState.IsRunning && m_Options.WriteUpstream && m_Endpoints.size() > 0) - { - if (!m_UpstreamThreads.empty()) - { - m_UpstreamQueue.Enqueue(std::move(CacheRecord)); - } - else - { - ProcessCacheRecord(std::move(CacheRecord)); - } - } - } - - virtual void GetStatus(CbObjectWriter& Status) override - { - ZEN_TRACE_CPU("Upstream::GetStatus"); - - Status << "active" << IsActive(); - Status << "reading" << m_Options.ReadUpstream; - Status << "writing" << m_Options.WriteUpstream; - Status << "worker_threads" << m_UpstreamThreads.size(); - Status << "queue_count" << m_UpstreamQueue.Size(); - - Status.BeginArray("endpoints"); - for (const auto& Ep : m_Endpoints) - { - const UpstreamEndpointInfo& EpInfo = Ep->GetEndpointInfo(); - const UpstreamEndpointStatus EpStatus = Ep->GetStatus(); - UpstreamEndpointStats& EpStats = Ep->Stats(); - - Status.BeginObject(); - Status << "name" << EpInfo.Name; - Status << "url" << EpInfo.Url; - Status << "state" << ToString(EpStatus.State); - Status << "reason" << EpStatus.Reason; - - Status.BeginObject("cache"sv); - { - const int64_t GetCount = EpStats.CacheGetCount.Value(); - const int64_t HitCount = EpStats.CacheHitCount.Value(); - const int64_t ErrorCount = EpStats.CacheErrorCount.Value(); - const double HitRatio = GetCount > 0 ? double(HitCount) / double(GetCount) : 0.0; - const double ErrorRatio = GetCount > 0 ? double(ErrorCount) / double(GetCount) : 0.0; - - metrics::EmitSnapshot("get_requests"sv, EpStats.CacheGetRequestTiming, Status); - Status << "get_bytes" << EpStats.CacheGetTotalBytes.Value(); - Status << "get_count" << GetCount; - Status << "hit_count" << HitCount; - Status << "hit_ratio" << HitRatio; - Status << "error_count" << ErrorCount; - Status << "error_ratio" << ErrorRatio; - metrics::EmitSnapshot("put_requests"sv, EpStats.CachePutRequestTiming, Status); - Status << "put_bytes" << EpStats.CachePutTotalBytes.Value(); - } - Status.EndObject(); - - Status.EndObject(); - } - Status.EndArray(); - } - -private: - void ProcessCacheRecord(UpstreamCacheRecord CacheRecord) - { - ZEN_TRACE_CPU("Upstream::ProcessCacheRecord"); - - ZenCacheValue CacheValue; - std::vector Payloads; - - if (!m_CacheStore.Get(CacheRecord.Context, CacheRecord.Namespace, CacheRecord.Key.Bucket, CacheRecord.Key.Hash, CacheValue)) - { - ZEN_WARN("process upstream FAILED, '{}/{}/{}', cache record doesn't exist", - CacheRecord.Namespace, - CacheRecord.Key.Bucket, - CacheRecord.Key.Hash); - return; - } - - for (const IoHash& ValueContentId : CacheRecord.ValueContentIds) - { - if (IoBuffer Payload = m_CidStore.FindChunkByCid(ValueContentId)) - { - Payloads.push_back(Payload); - } - else - { - ZEN_WARN("process upstream FAILED, '{}/{}/{}/{}', ValueContentId doesn't exist in CAS", - CacheRecord.Namespace, - CacheRecord.Key.Bucket, - CacheRecord.Key.Hash, - ValueContentId); - return; - } - } - - std::shared_lock _(m_EndpointsMutex); - - for (auto& Endpoint : m_Endpoints) - { - if (Endpoint->GetState() != UpstreamEndpointState::kOk) - { - continue; - } - - UpstreamEndpointStats& Stats = Endpoint->Stats(); - PutUpstreamCacheResult Result; - { - metrics::OperationTiming::Scope Scope(Stats.CachePutRequestTiming); - Result = Endpoint->PutCacheRecord(CacheRecord, CacheValue.Value, std::span(Payloads)); - } - - Stats.CachePutTotalBytes.Increment(Result.Bytes); - - if (!Result.Success) - { - ZEN_WARN("upload cache record '{}/{}/{}' FAILED, endpoint '{}', reason '{}'", - CacheRecord.Namespace, - CacheRecord.Key.Bucket, - CacheRecord.Key.Hash, - Endpoint->GetEndpointInfo().Url, - Result.Reason); - } - } - } - - void ProcessUpstreamQueue(int ThreadIndex) - { - std::string ThreadName = fmt::format("upstream_{}", ThreadIndex); - SetCurrentThreadName(ThreadName); - - for (;;) - { - UpstreamCacheRecord CacheRecord; - if (m_UpstreamQueue.WaitAndDequeue(CacheRecord)) - { - try - { - ProcessCacheRecord(std::move(CacheRecord)); - } - catch (const std::exception& Err) - { - ZEN_ERROR("upload cache record '{}/{}/{}' FAILED, reason '{}'", - CacheRecord.Namespace, - CacheRecord.Key.Bucket, - CacheRecord.Key.Hash, - Err.what()); - } - } - - if (!m_RunState.IsRunning) - { - break; - } - } - } - - void MonitorEndpoints() - { - SetCurrentThreadName("upstream_monitor"); - - for (;;) - { - { - std::unique_lock lk(m_RunState.Mutex); - if (m_RunState.ExitSignal.wait_for(lk, m_Options.HealthCheckInterval, [this]() { return !m_RunState.IsRunning.load(); })) - { - break; - } - } - - try - { - std::vector Endpoints; - - { - std::shared_lock _(m_EndpointsMutex); - - for (auto& Endpoint : m_Endpoints) - { - UpstreamEndpointState State = Endpoint->GetState(); - if (State == UpstreamEndpointState::kError) - { - Endpoints.push_back(Endpoint.get()); - ZEN_WARN("HEALTH - endpoint '{} - {}' is in error state '{}'", - Endpoint->GetEndpointInfo().Name, - Endpoint->GetEndpointInfo().Url, - Endpoint->GetStatus().Reason); - } - if (State == UpstreamEndpointState::kUnauthorized) - { - Endpoints.push_back(Endpoint.get()); - } - } - } - - for (auto& Endpoint : Endpoints) - { - const UpstreamEndpointInfo& Info = Endpoint->GetEndpointInfo(); - const UpstreamEndpointStatus Status = Endpoint->Initialize(); - - if (Status.State == UpstreamEndpointState::kOk) - { - ZEN_INFO("HEALTH - endpoint '{} - {}' Ok", Info.Name, Info.Url); - } - else - { - const std::string Reason = Status.Reason.empty() ? "" : fmt::format(", reason '{}'", Status.Reason); - ZEN_WARN("HEALTH - endpoint '{} - {}' {} {}", Info.Name, Info.Url, ToString(Status.State), Reason); - } - } - } - catch (const std::exception& Err) - { - ZEN_ERROR("check endpoint(s) health FAILED, reason '{}'", Err.what()); - } - } - } - - void Shutdown() - { - if (m_RunState.Stop()) - { - m_UpstreamQueue.CompleteAdding(); - for (std::thread& Thread : m_UpstreamThreads) - { - Thread.join(); - } - if (m_EndpointMonitorThread.joinable()) - { - m_EndpointMonitorThread.join(); - } - m_UpstreamThreads.clear(); - m_Endpoints.clear(); - } - } - - LoggerRef Log() { return m_Log; } - - using UpstreamQueue = BlockingQueue; - - struct RunState - { - std::mutex Mutex; - std::condition_variable ExitSignal; - std::atomic_bool IsRunning{false}; - - bool Stop() - { - bool Stopped = false; - { - std::lock_guard _(Mutex); - Stopped = IsRunning.exchange(false); - } - if (Stopped) - { - ExitSignal.notify_all(); - } - return Stopped; - } - }; - - LoggerRef m_Log; - UpstreamCacheOptions m_Options; - ZenCacheStore& m_CacheStore; - CidStore& m_CidStore; - UpstreamQueue m_UpstreamQueue; - std::shared_mutex m_EndpointsMutex; - std::vector> m_Endpoints; - std::vector m_UpstreamThreads; - std::thread m_EndpointMonitorThread; - RunState m_RunState; -}; - -////////////////////////////////////////////////////////////////////////// - -std::unique_ptr -UpstreamEndpoint::CreateZenEndpoint(const ZenStructuredCacheClientOptions& Options) -{ - return std::make_unique(Options); -} - -std::unique_ptr -UpstreamEndpoint::CreateJupiterEndpoint(const JupiterClientOptions& Options, const UpstreamAuthConfig& AuthConfig, AuthMgr& Mgr) -{ - return std::make_unique(Options, AuthConfig, Mgr); -} - -std::unique_ptr -CreateUpstreamCache(const UpstreamCacheOptions& Options, ZenCacheStore& CacheStore, CidStore& CidStore) -{ - return std::make_unique(Options, CacheStore, CidStore); -} - -} // namespace zen diff --git a/src/zenserver/upstream/upstreamcache.h b/src/zenserver/upstream/upstreamcache.h deleted file mode 100644 index d5d61c8d9..000000000 --- a/src/zenserver/upstream/upstreamcache.h +++ /dev/null @@ -1,167 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -namespace zen { - -class CbObjectView; -class AuthMgr; -class CbObjectView; -class CbPackage; -class CbObjectWriter; -class CidStore; -class ZenCacheStore; -struct JupiterClientOptions; -class JupiterAccessTokenProvider; -struct ZenStructuredCacheClientOptions; - -struct UpstreamEndpointStats -{ - metrics::OperationTiming CacheGetRequestTiming; - metrics::OperationTiming CachePutRequestTiming; - metrics::Counter CacheGetTotalBytes; - metrics::Counter CachePutTotalBytes; - metrics::Counter CacheGetCount; - metrics::Counter CacheHitCount; - metrics::Counter CacheErrorCount; -}; - -enum class UpstreamEndpointState : uint32_t -{ - kDisabled, - kUnauthorized, - kError, - kOk -}; - -inline std::string_view -ToString(UpstreamEndpointState State) -{ - using namespace std::literals; - - switch (State) - { - case UpstreamEndpointState::kDisabled: - return "Disabled"sv; - case UpstreamEndpointState::kUnauthorized: - return "Unauthorized"sv; - case UpstreamEndpointState::kError: - return "Error"sv; - case UpstreamEndpointState::kOk: - return "Ok"sv; - default: - return "Unknown"sv; - } -} - -struct UpstreamAuthConfig -{ - std::string_view OAuthUrl; - std::string_view OAuthClientId; - std::string_view OAuthClientSecret; - std::string_view OpenIdProvider; - std::string_view AccessToken; -}; - -struct UpstreamEndpointStatus -{ - std::string Reason; - UpstreamEndpointState State; -}; - -struct GetUpstreamCacheSingleResult -{ - GetUpstreamCacheResult Status; - IoBuffer Value; - const UpstreamEndpointInfo* Source = nullptr; -}; - -/** - * The upstream endpoint is responsible for handling upload/downloading of cache records. - */ -class UpstreamEndpoint -{ -public: - virtual ~UpstreamEndpoint() = default; - - virtual UpstreamEndpointStatus Initialize() = 0; - - virtual const UpstreamEndpointInfo& GetEndpointInfo() const = 0; - - virtual UpstreamEndpointState GetState() = 0; - virtual UpstreamEndpointStatus GetStatus() = 0; - - virtual GetUpstreamCacheSingleResult GetCacheRecord(std::string_view Namespace, const CacheKey& CacheKey, ZenContentType Type) = 0; - virtual GetUpstreamCacheResult GetCacheRecords(std::string_view Namespace, - std::span Requests, - OnCacheRecordGetComplete&& OnComplete) = 0; - - virtual GetUpstreamCacheResult GetCacheValues(std::string_view Namespace, - std::span CacheValueRequests, - OnCacheValueGetComplete&& OnComplete) = 0; - - virtual GetUpstreamCacheSingleResult GetCacheChunk(std::string_view Namespace, const CacheKey& CacheKey, const IoHash& PayloadId) = 0; - virtual GetUpstreamCacheResult GetCacheChunks(std::string_view Namespace, - std::span CacheChunkRequests, - OnCacheChunksGetComplete&& OnComplete) = 0; - - virtual PutUpstreamCacheResult PutCacheRecord(const UpstreamCacheRecord& CacheRecord, - IoBuffer RecordValue, - std::span Payloads) = 0; - - virtual UpstreamEndpointStats& Stats() = 0; - - static std::unique_ptr CreateZenEndpoint(const ZenStructuredCacheClientOptions& Options); - - static std::unique_ptr CreateJupiterEndpoint(const JupiterClientOptions& Options, - const UpstreamAuthConfig& AuthConfig, - AuthMgr& Mgr); -}; - -/** - * Manages one or more upstream cache endpoints. - */ - -class UpstreamCache : public UpstreamCacheClient -{ -public: - virtual void Initialize() = 0; - - virtual void RegisterEndpoint(std::unique_ptr Endpoint) = 0; - virtual void IterateEndpoints(std::function&& Fn) = 0; - - virtual GetUpstreamCacheSingleResult GetCacheRecord(std::string_view Namespace, const CacheKey& CacheKey, ZenContentType Type) = 0; - - virtual GetUpstreamCacheSingleResult GetCacheChunk(std::string_view Namespace, - const CacheKey& CacheKey, - const IoHash& ValueContentId) = 0; - - virtual void GetStatus(CbObjectWriter& CbO) = 0; -}; - -struct UpstreamCacheOptions -{ - std::chrono::seconds HealthCheckInterval{5}; - uint32_t ThreadCount = 4; - bool ReadUpstream = true; - bool WriteUpstream = true; -}; - -std::unique_ptr CreateUpstreamCache(const UpstreamCacheOptions& Options, ZenCacheStore& CacheStore, CidStore& CidStore); - -} // namespace zen diff --git a/src/zenserver/upstream/upstreamservice.cpp b/src/zenserver/upstream/upstreamservice.cpp deleted file mode 100644 index 1dcbdb604..000000000 --- a/src/zenserver/upstream/upstreamservice.cpp +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. -#include - -#include - -#include -#include - -namespace zen { - -using namespace std::literals; - -HttpUpstreamService::HttpUpstreamService(UpstreamCache& Upstream, AuthMgr& Mgr) : m_Upstream(Upstream), m_AuthMgr(Mgr) -{ - m_Router.RegisterRoute( - "endpoints", - [this](HttpRouterRequest& Req) { - CbObjectWriter Writer; - Writer.BeginArray("Endpoints"sv); - m_Upstream.IterateEndpoints([&Writer](UpstreamEndpoint& Ep) { - UpstreamEndpointInfo Info = Ep.GetEndpointInfo(); - UpstreamEndpointStatus Status = Ep.GetStatus(); - - Writer.BeginObject(); - Writer << "Name"sv << Info.Name; - Writer << "Url"sv << Info.Url; - Writer << "State"sv << ToString(Status.State); - Writer << "Reason"sv << Status.Reason; - Writer.EndObject(); - - return true; - }); - Writer.EndArray(); - Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Writer.Save()); - }, - HttpVerb::kGet); -} - -HttpUpstreamService::~HttpUpstreamService() -{ -} - -const char* -HttpUpstreamService::BaseUri() const -{ - return "/upstream/"; -} - -void -HttpUpstreamService::HandleRequest(zen::HttpServerRequest& Request) -{ - m_Router.HandleRequest(Request); -} - -} // namespace zen diff --git a/src/zenserver/upstream/upstreamservice.h b/src/zenserver/upstream/upstreamservice.h deleted file mode 100644 index f1da03c8c..000000000 --- a/src/zenserver/upstream/upstreamservice.h +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include - -namespace zen { - -class AuthMgr; -class UpstreamCache; - -class HttpUpstreamService final : public zen::HttpService -{ -public: - HttpUpstreamService(UpstreamCache& Upstream, AuthMgr& Mgr); - virtual ~HttpUpstreamService(); - - virtual const char* BaseUri() const override; - virtual void HandleRequest(zen::HttpServerRequest& Request) override; - -private: - UpstreamCache& m_Upstream; - AuthMgr& m_AuthMgr; - HttpRequestRouter m_Router; -}; - -} // namespace zen diff --git a/src/zenserver/upstream/zen.cpp b/src/zenserver/upstream/zen.cpp deleted file mode 100644 index 25fd3a3bb..000000000 --- a/src/zenserver/upstream/zen.cpp +++ /dev/null @@ -1,251 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "zen.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include "diag/logging.h" - -#include -#include - -namespace zen { - -////////////////////////////////////////////////////////////////////////// - -ZenStructuredCacheClient::ZenStructuredCacheClient(const ZenStructuredCacheClientOptions& Options) -: m_Log(logging::Get(std::string_view("zenclient"))) -, m_ServiceUrl(Options.Url) -, m_ConnectTimeout(Options.ConnectTimeout) -, m_Timeout(Options.Timeout) -{ -} - -ZenStructuredCacheClient::~ZenStructuredCacheClient() -{ -} - -////////////////////////////////////////////////////////////////////////// - -using namespace std::literals; - -ZenStructuredCacheSession::ZenStructuredCacheSession(Ref&& OuterClient) -: m_Log(OuterClient->Log()) -, m_Client(std::move(OuterClient)) -{ -} - -ZenStructuredCacheSession::~ZenStructuredCacheSession() -{ -} - -ZenCacheResult -ZenStructuredCacheSession::CheckHealth() -{ - HttpClient Http{m_Client->ServiceUrl()}; - - HttpClient::Response Response = Http.Get("/health/check"sv); - - if (auto& Error = Response.Error; Error) - { - return {.ErrorCode = static_cast(Error->ErrorCode), .Reason = std::move(Error->ErrorMessage)}; - } - - return {.Bytes = Response.DownloadedBytes, - .ElapsedSeconds = Response.ElapsedSeconds, - .Success = Response.StatusCode == HttpResponseCode::OK}; -} - -ZenCacheResult -ZenStructuredCacheSession::GetCacheRecord(std::string_view Namespace, std::string_view BucketId, const IoHash& Key, ZenContentType Type) -{ - HttpClient Http{m_Client->ServiceUrl()}; - - ExtendableStringBuilder<256> Uri; - Uri << "/z$/"; - if (Namespace != ZenCacheStore::DefaultNamespace) - { - Uri << Namespace << "/"; - } - Uri << BucketId << "/" << Key.ToHexString(); - - HttpClient::Response Response = Http.Get(Uri, {{"Accept", std::string{MapContentTypeToString(Type)}}}); - ZEN_DEBUG("GET {}", Response); - - if (auto& Error = Response.Error; Error) - { - return {.ErrorCode = static_cast(Error->ErrorCode), .Reason = std::move(Error->ErrorMessage)}; - } - - const bool Success = Response.StatusCode == HttpResponseCode::OK; - const IoBuffer Buffer = Success ? Response.ResponsePayload : IoBuffer{}; - - return {.Response = Buffer, .Bytes = Response.DownloadedBytes, .ElapsedSeconds = Response.ElapsedSeconds, .Success = Success}; -} - -ZenCacheResult -ZenStructuredCacheSession::GetCacheChunk(std::string_view Namespace, - std::string_view BucketId, - const IoHash& Key, - const IoHash& ValueContentId) -{ - HttpClient Http{m_Client->ServiceUrl()}; - - ExtendableStringBuilder<256> Uri; - Uri << "/z$/"; - if (Namespace != ZenCacheStore::DefaultNamespace) - { - Uri << Namespace << "/"; - } - Uri << BucketId << "/" << Key.ToHexString() << "/" << ValueContentId.ToHexString(); - - HttpClient::Response Response = Http.Get(Uri, {{"Accept", "application/x-ue-comp"}}); - ZEN_DEBUG("GET {}", Response); - - if (auto& Error = Response.Error; Error) - { - return {.ErrorCode = static_cast(Error->ErrorCode), .Reason = std::move(Error->ErrorMessage)}; - } - - const bool Success = Response.StatusCode == HttpResponseCode::OK; - const IoBuffer Buffer = Success ? Response.ResponsePayload : IoBuffer{}; - - return {.Response = Buffer, .Bytes = Response.DownloadedBytes, .ElapsedSeconds = Response.ElapsedSeconds, .Success = Success}; -} - -ZenCacheResult -ZenStructuredCacheSession::PutCacheRecord(std::string_view Namespace, - std::string_view BucketId, - const IoHash& Key, - IoBuffer Value, - ZenContentType Type) -{ - HttpClient Http{m_Client->ServiceUrl()}; - - ExtendableStringBuilder<256> Uri; - Uri << "/z$/"; - if (Namespace != ZenCacheStore::DefaultNamespace) - { - Uri << Namespace << "/"; - } - Uri << BucketId << "/" << Key.ToHexString(); - - Value.SetContentType(Type); - - HttpClient::Response Response = Http.Put(Uri, Value); - ZEN_DEBUG("PUT {}", Response); - - if (auto& Error = Response.Error; Error) - { - return {.ErrorCode = static_cast(Error->ErrorCode), .Reason = std::move(Error->ErrorMessage)}; - } - - const bool Success = Response.StatusCode == HttpResponseCode::OK || Response.StatusCode == HttpResponseCode::Created; - - return {.Bytes = Response.DownloadedBytes, .ElapsedSeconds = Response.ElapsedSeconds, .Success = Success}; -} - -ZenCacheResult -ZenStructuredCacheSession::PutCacheValue(std::string_view Namespace, - std::string_view BucketId, - const IoHash& Key, - const IoHash& ValueContentId, - IoBuffer Payload) -{ - HttpClient Http{m_Client->ServiceUrl()}; - - ExtendableStringBuilder<256> Uri; - Uri << "/z$/"; - if (Namespace != ZenCacheStore::DefaultNamespace) - { - Uri << Namespace << "/"; - } - Uri << BucketId << "/" << Key.ToHexString() << "/" << ValueContentId.ToHexString(); - - Payload.SetContentType(HttpContentType::kCompressedBinary); - - HttpClient::Response Response = Http.Put(Uri, Payload); - ZEN_DEBUG("PUT {}", Response); - - if (auto& Error = Response.Error; Error) - { - return {.ErrorCode = static_cast(Error->ErrorCode), .Reason = std::move(Error->ErrorMessage)}; - } - - const bool Success = Response.StatusCode == HttpResponseCode::OK || Response.StatusCode == HttpResponseCode::Created; - - return {.Bytes = Response.DownloadedBytes, .ElapsedSeconds = Response.ElapsedSeconds, .Success = Success}; -} - -ZenCacheResult -ZenStructuredCacheSession::InvokeRpc(const CbObjectView& Request) -{ - HttpClient Http{m_Client->ServiceUrl()}; - - ExtendableStringBuilder<256> Uri; - Uri << "/z$/$rpc"; - - // TODO: this seems redundant, we should be able to send the data more directly, without the BinaryWriter - - BinaryWriter BodyWriter; - Request.CopyTo(BodyWriter); - - IoBuffer Body{IoBuffer::Wrap, BodyWriter.GetData(), BodyWriter.GetSize()}; - Body.SetContentType(HttpContentType::kCbObject); - - HttpClient::Response Response = Http.Post(Uri, Body, {{"Accept", "application/x-ue-cbpkg"}}); - ZEN_DEBUG("POST {}", Response); - - if (auto& Error = Response.Error; Error) - { - return {.ErrorCode = static_cast(Error->ErrorCode), .Reason = std::move(Error->ErrorMessage)}; - } - - const bool Success = Response.StatusCode == HttpResponseCode::OK; - const IoBuffer Buffer = Success ? Response.ResponsePayload : IoBuffer{}; - - return {.Response = std::move(Buffer), - .Bytes = Response.DownloadedBytes, - .ElapsedSeconds = Response.ElapsedSeconds, - .Success = Success}; -} - -ZenCacheResult -ZenStructuredCacheSession::InvokeRpc(const CbPackage& Request) -{ - HttpClient Http{m_Client->ServiceUrl()}; - - ExtendableStringBuilder<256> Uri; - Uri << "/z$/$rpc"; - - IoBuffer Message = FormatPackageMessageBuffer(Request).Flatten().AsIoBuffer(); - Message.SetContentType(HttpContentType::kCbPackage); - - HttpClient::Response Response = Http.Post(Uri, Message, {{"Accept", "application/x-ue-cbpkg"}}); - ZEN_DEBUG("POST {}", Response); - - if (auto& Error = Response.Error; Error) - { - return {.ErrorCode = static_cast(Error->ErrorCode), .Reason = std::move(Error->ErrorMessage)}; - } - - const bool Success = Response.StatusCode == HttpResponseCode::OK; - const IoBuffer Buffer = Success ? Response.ResponsePayload : IoBuffer{}; - - return {.Response = std::move(Buffer), - .Bytes = Response.DownloadedBytes, - .ElapsedSeconds = Response.ElapsedSeconds, - .Success = Success}; -} - -} // namespace zen diff --git a/src/zenserver/upstream/zen.h b/src/zenserver/upstream/zen.h deleted file mode 100644 index 6321b46b1..000000000 --- a/src/zenserver/upstream/zen.h +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include -#include -#include -#include -#include -#include - -#include - -struct ZenCacheValue; - -namespace zen { - -class CbObjectWriter; -class CbObjectView; -class CbPackage; -class ZenStructuredCacheClient; - -////////////////////////////////////////////////////////////////////////// - -struct ZenCacheResult -{ - IoBuffer Response; - int64_t Bytes = {}; - double ElapsedSeconds = {}; - int32_t ErrorCode = {}; - std::string Reason; - bool Success = false; -}; - -struct ZenStructuredCacheClientOptions -{ - std::string_view Name; - std::string_view Url; - std::span Urls; - std::chrono::milliseconds ConnectTimeout{}; - std::chrono::milliseconds Timeout{}; -}; - -/** Zen Structured Cache session - * - * This provides a context in which cache queries can be performed - * - * These are currently all synchronous. Will need to be made asynchronous - */ -class ZenStructuredCacheSession -{ -public: - ZenStructuredCacheSession(Ref&& OuterClient); - ~ZenStructuredCacheSession(); - - ZenCacheResult CheckHealth(); - ZenCacheResult GetCacheRecord(std::string_view Namespace, std::string_view BucketId, const IoHash& Key, ZenContentType Type); - ZenCacheResult GetCacheChunk(std::string_view Namespace, std::string_view BucketId, const IoHash& Key, const IoHash& ValueContentId); - ZenCacheResult PutCacheRecord(std::string_view Namespace, - std::string_view BucketId, - const IoHash& Key, - IoBuffer Value, - ZenContentType Type); - ZenCacheResult PutCacheValue(std::string_view Namespace, - std::string_view BucketId, - const IoHash& Key, - const IoHash& ValueContentId, - IoBuffer Payload); - ZenCacheResult InvokeRpc(const CbObjectView& Request); - ZenCacheResult InvokeRpc(const CbPackage& Package); - -private: - inline LoggerRef Log() { return m_Log; } - - LoggerRef m_Log; - Ref m_Client; -}; - -/** Zen Structured Cache client - * - * This represents an endpoint to query -- actual queries should be done via - * ZenStructuredCacheSession - */ -class ZenStructuredCacheClient : public RefCounted -{ -public: - ZenStructuredCacheClient(const ZenStructuredCacheClientOptions& Options); - ~ZenStructuredCacheClient(); - - std::string_view ServiceUrl() const { return m_ServiceUrl; } - - inline LoggerRef Log() { return m_Log; } - -private: - LoggerRef m_Log; - std::string m_ServiceUrl; - std::chrono::milliseconds m_ConnectTimeout; - std::chrono::milliseconds m_Timeout; - - friend class ZenStructuredCacheSession; -}; - -} // namespace zen diff --git a/src/zenserver/vfs/vfsservice.cpp b/src/zenserver/vfs/vfsservice.cpp deleted file mode 100644 index 863ec348a..000000000 --- a/src/zenserver/vfs/vfsservice.cpp +++ /dev/null @@ -1,179 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "vfsservice.h" - -#include - -#include - -namespace zen { - -using namespace std::literals; - -#if ZEN_WITH_VFS - -////////////////////////////////////////////////////////////////////////// - -bool -GetContentAsCbObject(HttpServerRequest& HttpReq, CbObject& Cb) -{ - IoBuffer Payload = HttpReq.ReadPayload(); - HttpContentType PayloadContentType = HttpReq.RequestContentType(); - - switch (PayloadContentType) - { - case HttpContentType::kJSON: - case HttpContentType::kUnknownContentType: - case HttpContentType::kText: - { - std::string JsonText(reinterpret_cast(Payload.GetData()), Payload.GetSize()); - Cb = LoadCompactBinaryFromJson(JsonText).AsObject(); - if (!Cb) - { - HttpReq.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - "Content format not supported, expected JSON format"); - return false; - } - } - break; - case HttpContentType::kCbObject: - Cb = LoadCompactBinaryObject(Payload); - if (!Cb) - { - HttpReq.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - "Content format not supported, expected compact binary format"); - return false; - } - break; - default: - HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid request content type"); - return false; - } - - return true; -} - -////////////////////////////////////////////////////////////////////////// -// -// to test: -// -// echo {"method": "mount", "params": {"path": "d:\\VFS_ROOT"}} | curl.exe http://localhost:8558/vfs --data-binary @- -// echo {"method": "unmount"} | curl.exe http://localhost:8558/vfs --data-binary @- - -VfsService::VfsService(HttpStatusService& StatusService, VfsServiceImpl* ServiceImpl) : m_StatusService(StatusService), m_Impl(ServiceImpl) -{ - m_Router.RegisterRoute( - "info", - [&](HttpRouterRequest& Request) { - CbObjectWriter Cbo; - Cbo << "running" << m_Impl->IsVfsRunning(); - Cbo << "rootpath" << m_Impl->GetMountpointPath(); - - Request.ServerRequest().WriteResponse(HttpResponseCode::OK, Cbo.Save()); - }, - HttpVerb::kGet | HttpVerb::kHead); - - m_Router.RegisterRoute( - "", - [&](HttpRouterRequest& Req) { - CbObject Payload; - - if (!GetContentAsCbObject(Req.ServerRequest(), Payload)) - return; - - std::string_view RpcName = Payload["method"sv].AsString(); - - if (RpcName == "mount"sv) - { - CbObjectView Params = Payload["params"sv].AsObjectView(); - std::string_view Mountpath = Params["path"sv].AsString(); - - if (Mountpath.empty()) - { - return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "no path specified"); - } - - if (m_Impl->IsVfsRunning()) - { - return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "VFS already mounted"); - } - - try - { - m_Impl->Mount(Mountpath); - } - catch (const std::exception& Ex) - { - return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, Ex.what()); - } - - Req.ServerRequest().WriteResponse(HttpResponseCode::OK); - } - else if (RpcName == "unmount"sv) - { - if (!m_Impl->IsVfsRunning()) - { - return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "VFS not active"); - } - - try - { - m_Impl->Unmount(); - } - catch (const std::exception& Ex) - { - return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, Ex.what()); - } - - Req.ServerRequest().WriteResponse(HttpResponseCode::OK); - } - else - { - Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "unknown RPC"sv); - } - }, - HttpVerb::kPost); - m_StatusService.RegisterHandler("vfs", *this); -} - -VfsService::~VfsService() -{ - m_StatusService.UnregisterHandler("vfs", *this); -} - -#else - -VfsService::VfsService(HttpStatusService& StatusService, VfsServiceImpl* ServiceImpl) : m_StatusService(StatusService) -{ - ZEN_UNUSED(ServiceImpl); -} - -VfsService::~VfsService() -{ -} - -#endif - -const char* -VfsService::BaseUri() const -{ - return "/vfs/"; -} - -void -VfsService::HandleStatusRequest(HttpServerRequest& Request) -{ - CbObjectWriter Cbo; - Cbo << "ok" << true; - Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); -} - -void -VfsService::HandleRequest(HttpServerRequest& HttpServiceRequest) -{ - m_Router.HandleRequest(HttpServiceRequest); -} - -} // namespace zen diff --git a/src/zenserver/vfs/vfsservice.h b/src/zenserver/vfs/vfsservice.h deleted file mode 100644 index 4e06da878..000000000 --- a/src/zenserver/vfs/vfsservice.h +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include -#include -#include - -#include - -namespace zen { - -class ProjectStore; -class ZenCacheStore; -struct VfsServiceImpl; - -/** Virtual File System service - - Implements support for exposing data via a virtual file system interface. Currently - this is primarily used to surface various data stored in the local storage service - to users for debugging and exploration purposes. - - Currently, it surfaces information from the structured cache service and from the - project store. - - */ - -class VfsService : public HttpService, public IHttpStatusProvider -{ -public: - explicit VfsService(HttpStatusService& StatusService, VfsServiceImpl* ServiceImpl); - ~VfsService(); - -protected: - virtual const char* BaseUri() const override; - virtual void HandleRequest(HttpServerRequest& HttpServiceRequest) override; - virtual void HandleStatusRequest(HttpServerRequest& Request) override; - -private: - VfsServiceImpl* m_Impl = nullptr; - - HttpStatusService& m_StatusService; - HttpRequestRouter m_Router; - - friend struct VfsServiceDataSource; -}; - -} // namespace zen diff --git a/src/zenserver/workspaces/httpworkspaces.cpp b/src/zenserver/workspaces/httpworkspaces.cpp deleted file mode 100644 index 7ef84743e..000000000 --- a/src/zenserver/workspaces/httpworkspaces.cpp +++ /dev/null @@ -1,1211 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -namespace zen { -using namespace std::literals; - -ZEN_DEFINE_LOG_CATEGORY_STATIC(LogFs, "fs"sv); - -namespace { - - std::filesystem::path GetPathParameter(HttpServerRequest& ServerRequest, std::string_view Name) - { - if (std::string_view Value = ServerRequest.GetQueryParams().GetValue(Name); !Value.empty()) - { - return std::filesystem::path(HttpServerRequest::Decode(Value)); - } - return {}; - } - - void WriteWorkspaceConfig(CbWriter& Writer, const Workspaces::WorkspaceConfiguration& Config) - { - Writer << "id" << Config.Id; - Writer << "root_path" << Config.RootPath.string(); // utf8? - Writer << "allow_share_creation_from_http" << Config.AllowShareCreationFromHttp; - }; - - void WriteWorkspaceShareConfig(CbWriter& Writer, const Workspaces::WorkspaceShareConfiguration& Config) - { - Writer << "id" << Config.Id; - Writer << "share_path" << Config.SharePath.string(); // utf8? - if (!Config.Alias.empty()) - { - Writer << "alias" << Config.Alias; - } - }; - - void WriteWorkspaceAndSharesConfig(CbWriter& Writer, Workspaces& Workspaces, const Workspaces::WorkspaceConfiguration& WorkspaceConfig) - { - WriteWorkspaceConfig(Writer, WorkspaceConfig); - if (std::optional> ShareIds = Workspaces.GetWorkspaceShares(WorkspaceConfig.Id); ShareIds) - { - Writer.BeginArray("shares"); - { - for (const Oid& ShareId : *ShareIds) - { - if (std::optional WorkspaceShareConfig = - Workspaces.GetWorkspaceShareConfiguration(WorkspaceConfig.Id, ShareId); - WorkspaceShareConfig) - { - Writer.BeginObject(); - { - WriteWorkspaceShareConfig(Writer, *WorkspaceShareConfig); - } - Writer.EndObject(); - } - } - } - Writer.EndArray(); - } - } - -} // namespace - -HttpWorkspacesService::HttpWorkspacesService(HttpStatusService& StatusService, - HttpStatsService& StatsService, - const WorkspacesServeConfig& Cfg, - Workspaces& Workspaces) -: m_Log(logging::Get("workspaces")) -, m_StatusService(StatusService) -, m_StatsService(StatsService) -, m_Config(Cfg) -, m_Workspaces(Workspaces) -{ - Initialize(); -} - -HttpWorkspacesService::~HttpWorkspacesService() -{ - m_StatsService.UnregisterHandler("ws", *this); - m_StatusService.UnregisterHandler("ws", *this); -} - -const char* -HttpWorkspacesService::BaseUri() const -{ - return "/ws/"; -} - -void -HttpWorkspacesService::HandleRequest(HttpServerRequest& Request) -{ - metrics::OperationTiming::Scope $(m_HttpRequests); - - if (m_Router.HandleRequest(Request) == false) - { - ZEN_LOG_WARN(LogFs, "No route found for {0}", Request.RelativeUri()); - return Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Not found"sv); - } -} - -void -HttpWorkspacesService::HandleStatsRequest(HttpServerRequest& HttpReq) -{ - ZEN_TRACE_CPU("WorkspacesService::Stats"); - CbObjectWriter Cbo; - - EmitSnapshot("requests", m_HttpRequests, Cbo); - - Cbo.BeginObject("workspaces"); - { - Cbo.BeginObject("workspace"); - { - Cbo << "readcount" << m_WorkspacesStats.WorkspaceReadCount << "writecount" << m_WorkspacesStats.WorkspaceWriteCount - << "deletecount" << m_WorkspacesStats.WorkspaceDeleteCount; - } - Cbo.EndObject(); - - Cbo.BeginObject("workspaceshare"); - { - Cbo << "readcount" << m_WorkspacesStats.WorkspaceShareReadCount << "writecount" << m_WorkspacesStats.WorkspaceShareWriteCount - << "deletecount" << m_WorkspacesStats.WorkspaceShareDeleteCount; - } - Cbo.EndObject(); - - Cbo.BeginObject("chunk"); - { - Cbo << "hitcount" << m_WorkspacesStats.WorkspaceShareChunkHitCount << "misscount" - << m_WorkspacesStats.WorkspaceShareChunkMissCount; - } - Cbo.EndObject(); - - Cbo << "filescount" << m_WorkspacesStats.WorkspaceShareFilesReadCount; - Cbo << "entriescount" << m_WorkspacesStats.WorkspaceShareEntriesReadCount; - Cbo << "batchcount" << m_WorkspacesStats.WorkspaceShareBatchReadCount; - - Cbo << "requestcount" << m_WorkspacesStats.RequestCount; - Cbo << "badrequestcount" << m_WorkspacesStats.BadRequestCount; - } - Cbo.EndObject(); - - return HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); -} - -void -HttpWorkspacesService::HandleStatusRequest(HttpServerRequest& Request) -{ - ZEN_TRACE_CPU("HttpWorkspacesService::Status"); - CbObjectWriter Cbo; - Cbo << "ok" << true; - Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); -} - -void -HttpWorkspacesService::Initialize() -{ - using namespace std::literals; - - ZEN_LOG_INFO(LogFs, "Initializing Workspaces Service"); - - m_Router.AddPattern("workspace_id", "([[:xdigit:]]{24})"); - m_Router.AddPattern("share_id", "([[:xdigit:]]{24})"); - m_Router.AddPattern("chunk", "([[:xdigit:]]{24})"); - m_Router.AddPattern("share_alias", "([[:alnum:]_.\\+\\-\\[\\]]+)"); - - m_Router.RegisterRoute( - "{workspace_id}/{share_id}/files", - [this](HttpRouterRequest& Req) { FilesRequest(Req); }, - HttpVerb::kGet); - - m_Router.RegisterRoute( - "{workspace_id}/{share_id}/{chunk}/info", - [this](HttpRouterRequest& Req) { ChunkInfoRequest(Req); }, - HttpVerb::kGet); - - m_Router.RegisterRoute( - "{workspace_id}/{share_id}/batch", - [this](HttpRouterRequest& Req) { BatchRequest(Req); }, - HttpVerb::kPost); - - m_Router.RegisterRoute( - "{workspace_id}/{share_id}/entries", - [this](HttpRouterRequest& Req) { EntriesRequest(Req); }, - HttpVerb::kGet); - - m_Router.RegisterRoute( - "{workspace_id}/{share_id}/{chunk}", - [this](HttpRouterRequest& Req) { ChunkRequest(Req); }, - HttpVerb::kGet | HttpVerb::kHead); - - m_Router.RegisterRoute( - "share/{share_alias}/files", - [this](HttpRouterRequest& Req) { ShareAliasFilesRequest(Req); }, - HttpVerb::kGet); - - m_Router.RegisterRoute( - "share/{share_alias}/{chunk}/info", - [this](HttpRouterRequest& Req) { ShareAliasChunkInfoRequest(Req); }, - HttpVerb::kGet); - - m_Router.RegisterRoute( - "share/{share_alias}/batch", - [this](HttpRouterRequest& Req) { ShareAliasBatchRequest(Req); }, - HttpVerb::kPost); - - m_Router.RegisterRoute( - "share/{share_alias}/entries", - [this](HttpRouterRequest& Req) { ShareAliasEntriesRequest(Req); }, - HttpVerb::kGet); - - m_Router.RegisterRoute( - "share/{share_alias}/{chunk}", - [this](HttpRouterRequest& Req) { ShareAliasChunkRequest(Req); }, - HttpVerb::kGet | HttpVerb::kHead); - - m_Router.RegisterRoute( - "share/{share_alias}", - [this](HttpRouterRequest& Req) { ShareAliasRequest(Req); }, - HttpVerb::kPut | HttpVerb::kGet | HttpVerb::kDelete); - - m_Router.RegisterRoute( - "{workspace_id}/{share_id}", - [this](HttpRouterRequest& Req) { ShareRequest(Req); }, - HttpVerb::kPut | HttpVerb::kGet | HttpVerb::kDelete); - - m_Router.RegisterRoute( - "{workspace_id}", - [this](HttpRouterRequest& Req) { WorkspaceRequest(Req); }, - HttpVerb::kPut | HttpVerb::kGet | HttpVerb::kDelete); - - m_Router.RegisterRoute( - "refresh", - [this](HttpRouterRequest& Req) { RefreshRequest(Req); }, - HttpVerb::kGet); - - m_Router.RegisterRoute( - "", - [this](HttpRouterRequest& Req) { WorkspacesRequest(Req); }, - HttpVerb::kGet); - - RefreshState(); - - m_StatsService.RegisterHandler("ws", *this); - m_StatusService.RegisterHandler("ws", *this); -} - -std::filesystem::path -HttpWorkspacesService::GetStatePath() const -{ - return m_Config.SystemRootDir / "workspaces"; -} - -void -HttpWorkspacesService::RefreshState() -{ - if (!m_Config.SystemRootDir.empty()) - { - m_Workspaces.RefreshState(GetStatePath()); - } -} - -bool -HttpWorkspacesService::MayChangeConfiguration(const HttpServerRequest& Req) const -{ - ZEN_UNUSED(Req); - return m_Config.AllowConfigurationChanges; -} - -void -HttpWorkspacesService::RefreshRequest(HttpRouterRequest& Req) -{ - HttpServerRequest& ServerRequest = Req.ServerRequest(); - RefreshState(); - return ServerRequest.WriteResponse(HttpResponseCode::OK); -} - -void -HttpWorkspacesService::WorkspacesRequest(HttpRouterRequest& Req) -{ - HttpServerRequest& ServerRequest = Req.ServerRequest(); - - std::vector WorkspaceIds = m_Workspaces.GetWorkspaces(); - CbObjectWriter Response; - Response.BeginArray("workspaces"); - for (const Oid& WorkspaceId : WorkspaceIds) - { - if (std::optional WorkspaceConfig = m_Workspaces.GetWorkspaceConfiguration(WorkspaceId); - WorkspaceConfig) - { - Response.BeginObject(); - { - WriteWorkspaceAndSharesConfig(Response, m_Workspaces, *WorkspaceConfig); - } - Response.EndObject(); - } - } - Response.EndArray(); - - return ServerRequest.WriteResponse(HttpResponseCode::OK, Response.Save()); -} - -void -HttpWorkspacesService::FilesRequest(HttpRouterRequest& Req) -{ - HttpServerRequest& ServerRequest = Req.ServerRequest(); - const Oid WorkspaceId = Oid::TryFromHexString(Req.GetCapture(1)); - if (WorkspaceId == Oid::Zero) - { - m_WorkspacesStats.BadRequestCount++; - return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); - } - const Oid ShareId = Oid::TryFromHexString(Req.GetCapture(2)); - if (ShareId == Oid::Zero) - { - m_WorkspacesStats.BadRequestCount++; - return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Invalid share id '{}'", Req.GetCapture(2))); - } - FilesRequest(Req, WorkspaceId, ShareId); -} - -void -HttpWorkspacesService::ChunkInfoRequest(HttpRouterRequest& Req) -{ - HttpServerRequest& ServerRequest = Req.ServerRequest(); - const Oid WorkspaceId = Oid::TryFromHexString(Req.GetCapture(1)); - if (WorkspaceId == Oid::Zero) - { - m_WorkspacesStats.BadRequestCount++; - return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); - } - const Oid ShareId = Oid::TryFromHexString(Req.GetCapture(2)); - if (ShareId == Oid::Zero) - { - m_WorkspacesStats.BadRequestCount++; - return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Invalid share id '{}'", Req.GetCapture(2))); - } - const Oid ChunkId = Oid::TryFromHexString(Req.GetCapture(3)); - if (ChunkId == Oid::Zero) - { - m_WorkspacesStats.BadRequestCount++; - return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Invalid chunk id '{}'", Req.GetCapture(3))); - } - ChunkInfoRequest(Req, WorkspaceId, ShareId, ChunkId); -} - -void -HttpWorkspacesService::BatchRequest(HttpRouterRequest& Req) -{ - HttpServerRequest& ServerRequest = Req.ServerRequest(); - const Oid WorkspaceId = Oid::TryFromHexString(Req.GetCapture(1)); - if (WorkspaceId == Oid::Zero) - { - m_WorkspacesStats.BadRequestCount++; - return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); - } - const Oid ShareId = Oid::TryFromHexString(Req.GetCapture(2)); - if (ShareId == Oid::Zero) - { - m_WorkspacesStats.BadRequestCount++; - return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Invalid share id '{}'", Req.GetCapture(2))); - } - BatchRequest(Req, WorkspaceId, ShareId); -} - -void -HttpWorkspacesService::EntriesRequest(HttpRouterRequest& Req) -{ - HttpServerRequest& ServerRequest = Req.ServerRequest(); - const Oid WorkspaceId = Oid::TryFromHexString(Req.GetCapture(1)); - if (WorkspaceId == Oid::Zero) - { - m_WorkspacesStats.BadRequestCount++; - return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); - } - const Oid ShareId = Oid::TryFromHexString(Req.GetCapture(2)); - if (ShareId == Oid::Zero) - { - m_WorkspacesStats.BadRequestCount++; - return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Invalid share id '{}'", Req.GetCapture(2))); - } - EntriesRequest(Req, WorkspaceId, ShareId); -} - -void -HttpWorkspacesService::ChunkRequest(HttpRouterRequest& Req) -{ - HttpServerRequest& ServerRequest = Req.ServerRequest(); - const Oid WorkspaceId = Oid::TryFromHexString(Req.GetCapture(1)); - if (WorkspaceId == Oid::Zero) - { - m_WorkspacesStats.BadRequestCount++; - return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); - } - const Oid ShareId = Oid::TryFromHexString(Req.GetCapture(2)); - if (ShareId == Oid::Zero) - { - m_WorkspacesStats.BadRequestCount++; - return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Invalid share id '{}'", Req.GetCapture(2))); - } - const Oid ChunkId = Oid::TryFromHexString(Req.GetCapture(3)); - if (ChunkId == Oid::Zero) - { - m_WorkspacesStats.BadRequestCount++; - return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Invalid chunk id '{}'", Req.GetCapture(3))); - } - ChunkRequest(Req, WorkspaceId, ShareId, ChunkId); -} - -void -HttpWorkspacesService::ShareRequest(HttpRouterRequest& Req) -{ - HttpServerRequest& ServerRequest = Req.ServerRequest(); - const Oid WorkspaceId = Oid::TryFromHexString(Req.GetCapture(1)); - if (WorkspaceId == Oid::Zero) - { - m_WorkspacesStats.BadRequestCount++; - return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); - } - Oid ShareId = Oid::Zero; - if (Req.GetCapture(2) != Oid::Zero.ToString()) - { - ShareId = Oid::TryFromHexString(Req.GetCapture(2)); - if (ShareId == Oid::Zero) - { - m_WorkspacesStats.BadRequestCount++; - return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Invalid share id '{}'", Req.GetCapture(2))); - } - } - ShareRequest(Req, WorkspaceId, ShareId); -} - -void -HttpWorkspacesService::WorkspaceRequest(HttpRouterRequest& Req) -{ - HttpServerRequest& ServerRequest = Req.ServerRequest(); - Oid WorkspaceId = Oid::TryFromHexString(Req.GetCapture(1)); - switch (ServerRequest.RequestVerb()) - { - case HttpVerb::kPut: - { - std::filesystem::path WorkspacePath = GetPathParameter(ServerRequest, "root_path"sv); - if (WorkspacePath.empty()) - { - m_WorkspacesStats.BadRequestCount++; - return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - "Invalid 'root_path' parameter"); - } - - if (Req.GetCapture(1) == Oid::Zero.ToString()) - { - // Synthesize Id - WorkspaceId = Workspaces::PathToId(WorkspacePath); - ZEN_INFO("Generated workspace id from path '{}': {}", WorkspacePath, WorkspaceId); - } - else if (WorkspaceId == Oid::Zero) - { - m_WorkspacesStats.BadRequestCount++; - return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); - } - - if (!MayChangeConfiguration(ServerRequest)) - { - return ServerRequest.WriteResponse(HttpResponseCode::Unauthorized, - HttpContentType::kText, - fmt::format("Adding workspace {} is not allowed", WorkspaceId)); - } - bool AllowShareCreationFromHttp = false; - if (std::string_view Value = ServerRequest.GetQueryParams().GetValue("allow_share_creation_from_http"); Value == "true"sv) - { - AllowShareCreationFromHttp = true; - } - - m_WorkspacesStats.WorkspaceWriteCount++; - Workspaces::WorkspaceConfiguration OldConfig = Workspaces::FindWorkspace(Log(), GetStatePath(), WorkspaceId); - Workspaces::WorkspaceConfiguration NewConfig = {.Id = WorkspaceId, - .RootPath = WorkspacePath, - .AllowShareCreationFromHttp = AllowShareCreationFromHttp}; - if (OldConfig.Id == WorkspaceId && (OldConfig != NewConfig)) - { - return ServerRequest.WriteResponse( - HttpResponseCode::Conflict, - HttpContentType::kText, - fmt::format("Workspace {} already exists with root path '{}'", WorkspaceId, OldConfig.RootPath)); - } - else if (OldConfig.Id == Oid::Zero) - { - if (Workspaces::WorkspaceConfiguration ConfigWithSameRoot = - Workspaces::FindWorkspace(Log(), GetStatePath(), WorkspacePath); - ConfigWithSameRoot.Id != Oid::Zero) - { - return ServerRequest.WriteResponse( - HttpResponseCode::Conflict, - HttpContentType::kText, - fmt::format("Workspace {} already exists with same root path '{}'", ConfigWithSameRoot.Id, WorkspacePath)); - } - } - - bool Created = Workspaces::AddWorkspace(Log(), GetStatePath(), NewConfig); - if (Created) - { - ZEN_ASSERT(OldConfig.Id == Oid::Zero); - RefreshState(); - return ServerRequest.WriteResponse(HttpResponseCode::Created, HttpContentType::kText, fmt::format("{}", WorkspaceId)); - } - else - { - return ServerRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, fmt::format("{}", WorkspaceId)); - } - } - case HttpVerb::kGet: - { - if (WorkspaceId == Oid::Zero) - { - m_WorkspacesStats.BadRequestCount++; - return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); - } - m_WorkspacesStats.WorkspaceReadCount++; - std::optional Workspace = m_Workspaces.GetWorkspaceConfiguration(WorkspaceId); - if (Workspace) - { - CbObjectWriter Response; - WriteWorkspaceAndSharesConfig(Response, m_Workspaces, *Workspace); - return ServerRequest.WriteResponse(HttpResponseCode::OK, Response.Save()); - } - else - { - return ServerRequest.WriteResponse(HttpResponseCode::NotFound); - } - } - case HttpVerb::kDelete: - { - if (WorkspaceId == Oid::Zero) - { - m_WorkspacesStats.BadRequestCount++; - return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); - } - - if (!MayChangeConfiguration(ServerRequest)) - { - return ServerRequest.WriteResponse(HttpResponseCode::Unauthorized, - HttpContentType::kText, - fmt::format("Removing workspace {} is not allowed", WorkspaceId)); - } - - m_WorkspacesStats.WorkspaceDeleteCount++; - bool Deleted = Workspaces::RemoveWorkspace(Log(), GetStatePath(), WorkspaceId); - if (Deleted) - { - RefreshState(); - return ServerRequest.WriteResponse(HttpResponseCode::OK); - } - return ServerRequest.WriteResponse(HttpResponseCode::NotFound); - } - } -} - -void -HttpWorkspacesService::ShareAliasFilesRequest(HttpRouterRequest& Req) -{ - HttpServerRequest& ServerRequest = Req.ServerRequest(); - std::string_view Alias = Req.GetCapture(1); - if (Alias.empty()) - { - return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Invalid alias '{}'", Req.GetCapture(1))); - } - std::optional WorkspaceAndShareId = m_Workspaces.GetShareAlias(Alias); - if (!WorkspaceAndShareId.has_value()) - { - return ServerRequest.WriteResponse(HttpResponseCode::NotFound); - } - FilesRequest(Req, WorkspaceAndShareId.value().WorkspaceId, WorkspaceAndShareId.value().ShareId); -} - -void -HttpWorkspacesService::ShareAliasChunkInfoRequest(HttpRouterRequest& Req) -{ - HttpServerRequest& ServerRequest = Req.ServerRequest(); - std::string_view Alias = Req.GetCapture(1); - if (Alias.empty()) - { - return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Invalid alias '{}'", Req.GetCapture(1))); - } - std::optional WorkspaceAndShareId = m_Workspaces.GetShareAlias(Alias); - if (!WorkspaceAndShareId.has_value()) - { - return ServerRequest.WriteResponse(HttpResponseCode::NotFound); - } - const Oid ChunkId = Oid::TryFromHexString(Req.GetCapture(2)); - if (ChunkId == Oid::Zero) - { - m_WorkspacesStats.BadRequestCount++; - return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Invalid chunk id '{}'", Req.GetCapture(2))); - } - ChunkInfoRequest(Req, WorkspaceAndShareId.value().WorkspaceId, WorkspaceAndShareId.value().ShareId, ChunkId); -} - -void -HttpWorkspacesService::ShareAliasBatchRequest(HttpRouterRequest& Req) -{ - HttpServerRequest& ServerRequest = Req.ServerRequest(); - std::string_view Alias = Req.GetCapture(1); - if (Alias.empty()) - { - return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Invalid alias '{}'", Req.GetCapture(1))); - } - std::optional WorkspaceAndShareId = m_Workspaces.GetShareAlias(Alias); - if (!WorkspaceAndShareId.has_value()) - { - return ServerRequest.WriteResponse(HttpResponseCode::NotFound); - } - BatchRequest(Req, WorkspaceAndShareId.value().WorkspaceId, WorkspaceAndShareId.value().ShareId); -} - -void -HttpWorkspacesService::ShareAliasEntriesRequest(HttpRouterRequest& Req) -{ - HttpServerRequest& ServerRequest = Req.ServerRequest(); - std::string_view Alias = Req.GetCapture(1); - if (Alias.empty()) - { - return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Invalid alias '{}'", Req.GetCapture(1))); - } - std::optional WorkspaceAndShareId = m_Workspaces.GetShareAlias(Alias); - if (!WorkspaceAndShareId.has_value()) - { - return ServerRequest.WriteResponse(HttpResponseCode::NotFound); - } - EntriesRequest(Req, WorkspaceAndShareId.value().WorkspaceId, WorkspaceAndShareId.value().ShareId); -} - -void -HttpWorkspacesService::ShareAliasChunkRequest(HttpRouterRequest& Req) -{ - HttpServerRequest& ServerRequest = Req.ServerRequest(); - std::string_view Alias = Req.GetCapture(1); - if (Alias.empty()) - { - return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Invalid alias '{}'", Req.GetCapture(1))); - } - std::optional WorkspaceAndShareId = m_Workspaces.GetShareAlias(Alias); - if (!WorkspaceAndShareId.has_value()) - { - return ServerRequest.WriteResponse(HttpResponseCode::NotFound); - } - const Oid ChunkId = Oid::TryFromHexString(Req.GetCapture(2)); - if (ChunkId == Oid::Zero) - { - m_WorkspacesStats.BadRequestCount++; - return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Invalid chunk id '{}'", Req.GetCapture(2))); - } - ChunkRequest(Req, WorkspaceAndShareId.value().WorkspaceId, WorkspaceAndShareId.value().ShareId, ChunkId); -} - -void -HttpWorkspacesService::ShareAliasRequest(HttpRouterRequest& Req) -{ - HttpServerRequest& ServerRequest = Req.ServerRequest(); - std::string_view Alias = Req.GetCapture(1); - if (Alias.empty()) - { - return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Invalid alias '{}'", Req.GetCapture(1))); - } - std::optional WorkspaceAndShareId = m_Workspaces.GetShareAlias(Alias); - if (!WorkspaceAndShareId.has_value()) - { - return ServerRequest.WriteResponse(HttpResponseCode::NotFound); - } - ShareRequest(Req, WorkspaceAndShareId.value().WorkspaceId, WorkspaceAndShareId.value().ShareId); -} - -void -HttpWorkspacesService::FilesRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& ShareId) -{ - HttpServerRequest& ServerRequest = Req.ServerRequest(); - - m_WorkspacesStats.WorkspaceShareFilesReadCount++; - - std::unordered_set WantedFieldNames; - if (auto FieldFilter = HttpServerRequest::Decode(ServerRequest.GetQueryParams().GetValue("fieldnames")); !FieldFilter.empty()) - { - if (FieldFilter != "*") // Get all - empty FieldFilter equal getting all fields - { - ForEachStrTok(FieldFilter, ',', [&](std::string_view FieldName) { - WantedFieldNames.insert(std::string(FieldName)); - return true; - }); - } - } - else - { - const bool FilterClient = ServerRequest.GetQueryParams().GetValue("filter"sv) == "client"sv; - WantedFieldNames.insert("id"); - WantedFieldNames.insert("clientpath"); - if (!FilterClient) - { - WantedFieldNames.insert("serverpath"); - } - } - - bool Refresh = false; - if (auto RefreshStr = ServerRequest.GetQueryParams().GetValue("refresh"); !RefreshStr.empty()) - { - Refresh = StrCaseCompare(std::string(RefreshStr).c_str(), "true") == 0; - } - - const bool WantsAllFields = WantedFieldNames.empty(); - - const bool WantsIdField = WantsAllFields || WantedFieldNames.contains("id"); - const bool WantsClientPathField = WantsAllFields || WantedFieldNames.contains("clientpath"); - const bool WantsServerPathField = WantsAllFields || WantedFieldNames.contains("serverpath"); - const bool WantsRawSizeField = WantsAllFields || WantedFieldNames.contains("rawsize"); - const bool WantsSizeField = WantsAllFields || WantedFieldNames.contains("size"); - - std::optional> Files = - m_Workspaces.GetWorkspaceShareFiles(WorkspaceId, ShareId, Refresh, GetSmallWorkerPool(EWorkloadType::Burst)); - if (!Files.has_value()) - { - return ServerRequest.WriteResponse(HttpResponseCode::NotFound); - } - - CbObjectWriter Response; - Response.BeginArray("files"sv); - { - for (const Workspaces::ShareFile& Entry : Files.value()) - { - Response.BeginObject(); - if (WantsIdField) - { - Response << "id"sv << Entry.Id; - } - if (WantsServerPathField) - { - Response << "serverpath"sv << Entry.RelativePath; - } - if (WantsClientPathField) - { - Response << "clientpath"sv << Entry.RelativePath; - } - if (WantsSizeField) - { - Response << "size"sv << Entry.Size; - } - if (WantsRawSizeField) - { - Response << "rawsize"sv << Entry.Size; - } - Response.EndObject(); - } - } - Response.EndArray(); - - return ServerRequest.WriteResponse(HttpResponseCode::OK, Response.Save()); -} - -void -HttpWorkspacesService::ChunkInfoRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& ShareId, const Oid& ChunkId) -{ - HttpServerRequest& ServerRequest = Req.ServerRequest(); - Workspaces::ShareFile File = - m_Workspaces.GetWorkspaceShareChunkInfo(WorkspaceId, ShareId, ChunkId, GetSmallWorkerPool(EWorkloadType::Burst)); - if (File.Id != Oid::Zero) - { - CbObjectWriter Response; - Response << "size"sv << File.Size; - m_WorkspacesStats.WorkspaceShareChunkHitCount++; - return ServerRequest.WriteResponse(HttpResponseCode::OK, Response.Save()); - } - m_WorkspacesStats.WorkspaceShareChunkMissCount++; - return ServerRequest.WriteResponse(HttpResponseCode::NotFound); -} - -void -HttpWorkspacesService::BatchRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& ShareId) -{ - HttpServerRequest& ServerRequest = Req.ServerRequest(); - IoBuffer Payload = ServerRequest.ReadPayload(); - std::optional> ChunkRequests = ParseChunkBatchRequest(Payload); - if (!ChunkRequests.has_value()) - { - m_WorkspacesStats.BadRequestCount++; - return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "batch payload malformed"); - } - m_WorkspacesStats.WorkspaceShareBatchReadCount++; - std::vector Requests; - Requests.reserve(ChunkRequests.value().size()); - std::transform(ChunkRequests.value().begin(), - ChunkRequests.value().end(), - std::back_inserter(Requests), - [](const RequestChunkEntry& Entry) { - return Workspaces::ChunkRequest{.ChunkId = Entry.ChunkId, .Offset = Entry.Offset, .Size = Entry.RequestBytes}; - }); - std::vector Chunks = - m_Workspaces.GetWorkspaceShareChunks(WorkspaceId, ShareId, Requests, GetSmallWorkerPool(EWorkloadType::Burst)); - if (Chunks.empty()) - { - return ServerRequest.WriteResponse(HttpResponseCode::NotFound); - } - for (const IoBuffer& Buffer : Chunks) - { - if (Buffer) - { - m_WorkspacesStats.WorkspaceShareChunkHitCount++; - } - else - { - m_WorkspacesStats.WorkspaceShareChunkMissCount++; - } - } - std::vector Response = BuildChunkBatchResponse(ChunkRequests.value(), Chunks); - if (!Response.empty()) - { - return ServerRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, Response); - } - return ServerRequest.WriteResponse(HttpResponseCode::InternalServerError, - HttpContentType::kText, - fmt::format("failed formatting response for batch of {} chunks", Chunks.size())); -} - -void -HttpWorkspacesService::EntriesRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& ShareId) -{ - HttpServerRequest& ServerRequest = Req.ServerRequest(); - std::string_view OpKey = ServerRequest.GetQueryParams().GetValue("opkey"sv); - if (!OpKey.empty() && OpKey != "file_manifest") - { - m_WorkspacesStats.BadRequestCount++; - return ServerRequest.WriteResponse(HttpResponseCode::NotFound); - } - std::unordered_set WantedFieldNames; - if (auto FieldFilter = HttpServerRequest::Decode(ServerRequest.GetQueryParams().GetValue("fieldfilter")); !FieldFilter.empty()) - { - if (FieldFilter != "*") // Get all - empty FieldFilter equal getting all fields - { - ForEachStrTok(FieldFilter, ',', [&](std::string_view FieldName) { - WantedFieldNames.insert(std::string(FieldName)); - return true; - }); - } - } - - bool Refresh = false; - if (auto RefreshStr = ServerRequest.GetQueryParams().GetValue("refresh"); !RefreshStr.empty()) - { - Refresh = StrCaseCompare(std::string(RefreshStr).c_str(), "true") == 0; - } - - m_WorkspacesStats.WorkspaceShareEntriesReadCount++; - std::optional> Files = - m_Workspaces.GetWorkspaceShareFiles(WorkspaceId, ShareId, Refresh, GetSmallWorkerPool(EWorkloadType::Burst)); - if (!Files.has_value()) - { - return ServerRequest.WriteResponse(HttpResponseCode::NotFound); - } - const bool WantsAllFields = WantedFieldNames.empty(); - - const bool WantsIdField = WantsAllFields || WantedFieldNames.contains("id"); - const bool WantsClientPathField = WantsAllFields || WantedFieldNames.contains("clientpath"); - const bool WantsServerPathField = WantsAllFields || WantedFieldNames.contains("serverpath"); - - CbObjectWriter Response; - - if (OpKey.empty()) - { - Response.BeginArray("entries"sv); - Response.BeginObject(); - } - else - { - Response.BeginObject("entry"sv); - } - { - // Synthesize a fake op - Response << "key" - << "file_manifest"; - - Response.BeginArray("files"); - { - for (const Workspaces::ShareFile& Entry : Files.value()) - { - Response.BeginObject(); - { - if (WantsIdField) - { - Response << "id"sv << Entry.Id; - } - if (WantsServerPathField) - { - Response << "serverpath"sv << Entry.RelativePath; - } - if (WantsClientPathField) - { - Response << "clientpath"sv << Entry.RelativePath; - } - } - Response.EndObject(); - } - } - Response.EndArray(); - } - - if (OpKey.empty()) - { - Response.EndObject(); - Response.EndArray(); - } - else - { - Response.EndObject(); - } - - return ServerRequest.WriteResponse(HttpResponseCode::OK, Response.Save()); -} - -void -HttpWorkspacesService::ChunkRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& ShareId, const Oid& ChunkId) -{ - HttpServerRequest& ServerRequest = Req.ServerRequest(); - - uint64_t Offset = 0; - uint64_t Size = ~(0ull); - if (auto OffsetParm = ServerRequest.GetQueryParams().GetValue("offset"); OffsetParm.empty() == false) - { - if (auto OffsetVal = ParseInt(OffsetParm)) - { - Offset = OffsetVal.value(); - } - else - { - m_WorkspacesStats.BadRequestCount++; - return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Invalid offset parameter '{}'", OffsetParm)); - } - } - - if (auto SizeParm = ServerRequest.GetQueryParams().GetValue("size"); SizeParm.empty() == false) - { - if (auto SizeVal = ParseInt(SizeParm)) - { - Size = SizeVal.value(); - } - else - { - m_WorkspacesStats.BadRequestCount++; - return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Invalid size parameter '{}'", SizeParm)); - } - } - - std::vector Response = m_Workspaces.GetWorkspaceShareChunks( - WorkspaceId, - ShareId, - std::vector{Workspaces::ChunkRequest{.ChunkId = ChunkId, .Offset = Offset, .Size = Size}}, - GetSmallWorkerPool(EWorkloadType::Burst)); - if (!Response.empty() && Response[0]) - { - m_WorkspacesStats.WorkspaceShareChunkHitCount++; - if (Response[0].GetSize() == 0) - { - return ServerRequest.WriteResponse(HttpResponseCode::OK); - } - return ServerRequest.WriteResponse(HttpResponseCode::OK, Response[0].GetContentType(), Response); - } - m_WorkspacesStats.WorkspaceShareChunkMissCount++; - return ServerRequest.WriteResponse(HttpResponseCode::NotFound); -} - -void -HttpWorkspacesService::ShareRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& InShareId) -{ - Oid ShareId = InShareId; - - HttpServerRequest& ServerRequest = Req.ServerRequest(); - switch (ServerRequest.RequestVerb()) - { - case HttpVerb::kPut: - { - std::filesystem::path SharePath = GetPathParameter(ServerRequest, "share_path"sv); - if (SharePath.empty()) - { - m_WorkspacesStats.BadRequestCount++; - return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - "Invalid 'share_path' parameter"); - } - - if (ShareId == Oid::Zero) - { - // Synthesize Id - ShareId = Workspaces::PathToId(SharePath); - ZEN_INFO("Generated workspace id from path '{}': {}", SharePath, ShareId); - } - - std::string Alias = HttpServerRequest::Decode(ServerRequest.GetQueryParams().GetValue("alias"sv)); - if (!AsciiSet::HasOnly(Alias, Workspaces::ValidAliasCharactersSet)) - { - return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid 'alias' parameter"); - } - - Workspaces::WorkspaceConfiguration Workspace = Workspaces::FindWorkspace(Log(), GetStatePath(), WorkspaceId); - if (Workspace.Id == Oid::Zero) - { - return ServerRequest.WriteResponse(HttpResponseCode::NotFound, - HttpContentType::kText, - fmt::format("Workspace '{}' does not exist", WorkspaceId)); - } - - if (!Workspace.AllowShareCreationFromHttp) - { - if (!MayChangeConfiguration(ServerRequest)) - { - return ServerRequest.WriteResponse( - HttpResponseCode::Unauthorized, - HttpContentType::kText, - fmt::format("Adding workspace share {} in workspace {} is not allowed", WorkspaceId, ShareId)); - } - } - - m_WorkspacesStats.WorkspaceShareWriteCount++; - - const Workspaces::WorkspaceShareConfiguration OldConfig = - Workspaces::FindWorkspaceShare(Log(), Workspace.RootPath, ShareId); - const Workspaces::WorkspaceShareConfiguration NewConfig = {.Id = ShareId, - .SharePath = SharePath, - .Alias = std::string(Alias)}; - - if (OldConfig.Id == ShareId && (OldConfig != NewConfig)) - { - return ServerRequest.WriteResponse( - HttpResponseCode::Conflict, - HttpContentType::kText, - fmt::format("Workspace share '{}' already exist in workspace '{}' with share path '{}' and alias '{}'", - ShareId, - WorkspaceId, - OldConfig.SharePath, - OldConfig.Alias)); - } - else if (OldConfig.Id == Oid::Zero) - { - if (Workspaces::WorkspaceShareConfiguration ConfigWithSamePath = - Workspaces::FindWorkspaceShare(Log(), Workspace.RootPath, SharePath); - ConfigWithSamePath.Id != Oid::Zero) - { - return ServerRequest.WriteResponse( - HttpResponseCode::Conflict, - HttpContentType::kText, - fmt::format("Workspace share '{}' already exist in workspace '{}' with same share path '{}' and alias '{}'", - ShareId, - WorkspaceId, - OldConfig.SharePath, - OldConfig.Alias)); - } - } - - if (!IsDir(Workspace.RootPath / NewConfig.SharePath)) - { - return ServerRequest.WriteResponse(HttpResponseCode::NotFound, - HttpContentType::kText, - fmt::format("directory {} does not exist in workspace {} root '{}'", - NewConfig.SharePath, - WorkspaceId, - Workspace.RootPath)); - } - - bool Created = Workspaces::AddWorkspaceShare(Log(), Workspace.RootPath, NewConfig); - if (Created) - { - RefreshState(); - return ServerRequest.WriteResponse(HttpResponseCode::Created, HttpContentType::kText, fmt::format("{}", ShareId)); - } - return ServerRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, fmt::format("{}", ShareId)); - } - case HttpVerb::kGet: - { - if (WorkspaceId == Oid::Zero) - { - m_WorkspacesStats.BadRequestCount++; - return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); - } - if (ShareId == Oid::Zero) - { - m_WorkspacesStats.BadRequestCount++; - return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Invalid share id '{}'", ShareId)); - } - - m_WorkspacesStats.WorkspaceShareReadCount++; - std::optional Config = - m_Workspaces.GetWorkspaceShareConfiguration(WorkspaceId, ShareId); - if (!Config) - { - return ServerRequest.WriteResponse(HttpResponseCode::NotFound); - } - - CbObjectWriter Response; - WriteWorkspaceShareConfig(Response, *Config); - return ServerRequest.WriteResponse(HttpResponseCode::OK, Response.Save()); - } - case HttpVerb::kDelete: - { - if (WorkspaceId == Oid::Zero) - { - m_WorkspacesStats.BadRequestCount++; - return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); - } - if (ShareId == Oid::Zero) - { - m_WorkspacesStats.BadRequestCount++; - return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Invalid share id '{}'", ShareId)); - } - - Workspaces::WorkspaceConfiguration Workspace = Workspaces::FindWorkspace(Log(), GetStatePath(), WorkspaceId); - if (Workspace.Id == Oid::Zero) - { - return ServerRequest.WriteResponse(HttpResponseCode::NotFound); - } - - if (!Workspace.AllowShareCreationFromHttp) - { - if (!MayChangeConfiguration(ServerRequest)) - { - return ServerRequest.WriteResponse( - HttpResponseCode::Unauthorized, - HttpContentType::kText, - fmt::format("Removing workspace share {} in workspace {} is not allowed", WorkspaceId, ShareId)); - } - } - - m_WorkspacesStats.WorkspaceShareDeleteCount++; - bool Deleted = Workspaces::RemoveWorkspaceShare(Log(), Workspace.RootPath, ShareId); - if (Deleted) - { - RefreshState(); - return ServerRequest.WriteResponse(HttpResponseCode::OK); - } - return ServerRequest.WriteResponse(HttpResponseCode::NotFound); - } - } -} - -} // namespace zen diff --git a/src/zenserver/workspaces/httpworkspaces.h b/src/zenserver/workspaces/httpworkspaces.h deleted file mode 100644 index 89a8e8bdc..000000000 --- a/src/zenserver/workspaces/httpworkspaces.h +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include -#include -#include -#include - -namespace zen { - -class Workspaces; - -struct WorkspacesServeConfig -{ - std::filesystem::path SystemRootDir; - bool AllowConfigurationChanges = false; -}; - -class HttpWorkspacesService final : public HttpService, public IHttpStatusProvider, public IHttpStatsProvider -{ -public: - HttpWorkspacesService(HttpStatusService& StatusService, - HttpStatsService& StatsService, - const WorkspacesServeConfig& Cfg, - Workspaces& Workspaces); - virtual ~HttpWorkspacesService(); - - virtual const char* BaseUri() const override; - virtual void HandleRequest(HttpServerRequest& Request) override; - - virtual void HandleStatsRequest(HttpServerRequest& Request) override; - virtual void HandleStatusRequest(HttpServerRequest& Request) override; - -private: - struct WorkspacesStats - { - std::atomic_uint64_t WorkspaceReadCount{}; - std::atomic_uint64_t WorkspaceWriteCount{}; - std::atomic_uint64_t WorkspaceDeleteCount{}; - std::atomic_uint64_t WorkspaceShareReadCount{}; - std::atomic_uint64_t WorkspaceShareWriteCount{}; - std::atomic_uint64_t WorkspaceShareDeleteCount{}; - std::atomic_uint64_t WorkspaceShareFilesReadCount{}; - std::atomic_uint64_t WorkspaceShareEntriesReadCount{}; - std::atomic_uint64_t WorkspaceShareBatchReadCount{}; - std::atomic_uint64_t WorkspaceShareChunkHitCount{}; - std::atomic_uint64_t WorkspaceShareChunkMissCount{}; - std::atomic_uint64_t RequestCount{}; - std::atomic_uint64_t BadRequestCount{}; - }; - - inline LoggerRef Log() { return m_Log; } - - LoggerRef m_Log; - - void Initialize(); - std::filesystem::path GetStatePath() const; - void RefreshState(); - // void WriteState(); - - bool MayChangeConfiguration(const HttpServerRequest& Req) const; - - void WorkspacesRequest(HttpRouterRequest& Req); - void RefreshRequest(HttpRouterRequest& Req); - void FilesRequest(HttpRouterRequest& Req); - void ChunkInfoRequest(HttpRouterRequest& Req); - void BatchRequest(HttpRouterRequest& Req); - void EntriesRequest(HttpRouterRequest& Req); - void ChunkRequest(HttpRouterRequest& Req); - void ShareRequest(HttpRouterRequest& Req); - void WorkspaceRequest(HttpRouterRequest& Req); - - void ShareAliasFilesRequest(HttpRouterRequest& Req); - void ShareAliasChunkInfoRequest(HttpRouterRequest& Req); - void ShareAliasBatchRequest(HttpRouterRequest& Req); - void ShareAliasEntriesRequest(HttpRouterRequest& Req); - void ShareAliasChunkRequest(HttpRouterRequest& Req); - void ShareAliasRequest(HttpRouterRequest& Req); - - void FilesRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& ShareId); - void ChunkInfoRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& ShareId, const Oid& ChunkId); - void BatchRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& ShareId); - void EntriesRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& ShareId); - void ChunkRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& ShareId, const Oid& ChunkId); - void ShareRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& InShareId); - - HttpStatusService& m_StatusService; - HttpStatsService& m_StatsService; - const WorkspacesServeConfig m_Config; - HttpRequestRouter m_Router; - Workspaces& m_Workspaces; - WorkspacesStats m_WorkspacesStats; - metrics::OperationTiming m_HttpRequests; -}; - -} // namespace zen diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index bf59658c8..11afb682c 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -28,13 +28,7 @@ #if ZEN_PLATFORM_WINDOWS # include -#endif - -#if ZEN_PLATFORM_LINUX -# include -#endif - -#if ZEN_PLATFORM_MAC +#else # include #endif @@ -48,7 +42,7 @@ ZEN_THIRD_PARTY_INCLUDES_END ////////////////////////////////////////////////////////////////////////// -#include "config.h" +#include "config/config.h" #include "diag/logging.h" #include diff --git a/src/zenserver/zenstorageserver.cpp b/src/zenserver/zenstorageserver.cpp deleted file mode 100644 index 73896512d..000000000 --- a/src/zenserver/zenstorageserver.cpp +++ /dev/null @@ -1,961 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "zenstorageserver.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#if ZEN_PLATFORM_WINDOWS -# include -#endif - -#if ZEN_PLATFORM_LINUX -# include -#endif - -#if ZEN_PLATFORM_MAC -# include -#endif - -ZEN_THIRD_PARTY_INCLUDES_START -#include -#include -ZEN_THIRD_PARTY_INCLUDES_END - -#include -#include - -////////////////////////////////////////////////////////////////////////// - -#include "diag/logging.h" -#include "storageconfig.h" - -#include - -namespace zen { - -namespace utils { - asio::error_code ResolveHostname(asio::io_context& Ctx, - std::string_view Host, - std::string_view DefaultPort, - std::vector& OutEndpoints) - { - std::string_view Port = DefaultPort; - - if (const size_t Idx = Host.find(":"); Idx != std::string_view::npos) - { - Port = Host.substr(Idx + 1); - Host = Host.substr(0, Idx); - } - - asio::ip::tcp::resolver Resolver(Ctx); - - asio::error_code ErrorCode; - asio::ip::tcp::resolver::results_type Endpoints = Resolver.resolve(Host, Port, ErrorCode); - - if (!ErrorCode) - { - for (const asio::ip::tcp::endpoint Ep : Endpoints) - { - OutEndpoints.push_back(fmt::format("http://{}:{}", Ep.address().to_string(), Ep.port())); - } - } - - return ErrorCode; - } -} // namespace utils - -using namespace std::literals; - -ZenStorageServer::ZenStorageServer() -{ -} - -ZenStorageServer::~ZenStorageServer() -{ -} - -int -ZenStorageServer::Initialize(const ZenStorageServerOptions& ServerOptions, ZenServerState::ZenServerEntry* ServerEntry) -{ - ZEN_TRACE_CPU("ZenStorageServer::Initialize"); - ZEN_MEMSCOPE(GetZenserverTag()); - - const int EffectiveBasePort = ZenServerBase::Initialize(ServerOptions, ServerEntry); - if (EffectiveBasePort < 0) - { - return EffectiveBasePort; - } - - m_DebugOptionForcedCrash = ServerOptions.ShouldCrash; - m_StartupScrubOptions = ServerOptions.ScrubOptions; - - InitializeState(ServerOptions); - InitializeServices(ServerOptions); - RegisterServices(); - - ZenServerBase::Finalize(); - - return EffectiveBasePort; -} - -void -ZenStorageServer::RegisterServices() -{ - m_Http->RegisterService(*m_AuthService); - m_Http->RegisterService(m_StatsService); - m_Http->RegisterService(m_TestService); // NOTE: this is intentionally not limited to test mode as it's useful for diagnostics - -#if ZEN_WITH_TESTS - m_Http->RegisterService(m_TestingService); -#endif - - if (m_StructuredCacheService) - { - m_Http->RegisterService(*m_StructuredCacheService); - } - - if (m_UpstreamService) - { - m_Http->RegisterService(*m_UpstreamService); - } - - if (m_HttpProjectService) - { - m_Http->RegisterService(*m_HttpProjectService); - } - - if (m_HttpWorkspacesService) - { - m_Http->RegisterService(*m_HttpWorkspacesService); - } - - m_FrontendService = std::make_unique(m_ContentRoot, m_StatusService); - - if (m_FrontendService) - { - m_Http->RegisterService(*m_FrontendService); - } - - if (m_ObjStoreService) - { - m_Http->RegisterService(*m_ObjStoreService); - } - - if (m_BuildStoreService) - { - m_Http->RegisterService(*m_BuildStoreService); - } - -#if ZEN_WITH_VFS - m_Http->RegisterService(*m_VfsService); -#endif // ZEN_WITH_VFS - - m_Http->RegisterService(*m_AdminService); -} - -void -ZenStorageServer::InitializeServices(const ZenStorageServerOptions& ServerOptions) -{ - InitializeAuthentication(ServerOptions); - - ZEN_INFO("initializing storage"); - - CidStoreConfiguration Config; - Config.RootDirectory = m_DataRoot / "cas"; - - m_CidStore = std::make_unique(m_GcManager); - m_CidStore->Initialize(Config); - - ZEN_INFO("instantiating project service"); - - m_JobQueue = MakeJobQueue(8, "bgjobs"); - - m_ProjectStore = new ProjectStore(*m_CidStore, m_DataRoot / "projects", m_GcManager, ProjectStore::Configuration{}); - m_HttpProjectService.reset( - new HttpProjectService{*m_CidStore, m_ProjectStore, m_StatusService, m_StatsService, *m_AuthMgr, *m_OpenProcessCache, *m_JobQueue}); - - if (ServerOptions.WorksSpacesConfig.Enabled) - { - m_Workspaces.reset(new Workspaces()); - m_HttpWorkspacesService.reset( - new HttpWorkspacesService(m_StatusService, - m_StatsService, - {.SystemRootDir = ServerOptions.SystemRootDir, - .AllowConfigurationChanges = ServerOptions.WorksSpacesConfig.AllowConfigurationChanges}, - *m_Workspaces)); - } - - if (ServerOptions.BuildStoreConfig.Enabled) - { - CidStoreConfiguration BuildCidConfig; - BuildCidConfig.RootDirectory = m_DataRoot / "builds_cas"; - m_BuildCidStore = std::make_unique(m_GcManager); - m_BuildCidStore->Initialize(BuildCidConfig); - - BuildStoreConfig BuildsCfg; - BuildsCfg.RootDirectory = m_DataRoot / "builds"; - BuildsCfg.MaxDiskSpaceLimit = ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit; - m_BuildStore = std::make_unique(std::move(BuildsCfg), m_GcManager, *m_BuildCidStore); - } - - if (ServerOptions.StructuredCacheConfig.Enabled) - { - InitializeStructuredCache(ServerOptions); - } - else - { - ZEN_INFO("NOT instantiating structured cache service"); - } - - if (ServerOptions.ObjectStoreEnabled) - { - ObjectStoreConfig ObjCfg; - ObjCfg.RootDirectory = m_DataRoot / "obj"; - - for (const auto& Bucket : ServerOptions.ObjectStoreConfig.Buckets) - { - ObjectStoreConfig::BucketConfig NewBucket{.Name = Bucket.Name}; - NewBucket.Directory = Bucket.Directory.empty() ? (ObjCfg.RootDirectory / Bucket.Name) : Bucket.Directory; - ObjCfg.Buckets.push_back(std::move(NewBucket)); - } - - m_ObjStoreService = std::make_unique(m_StatusService, std::move(ObjCfg)); - } - - if (ServerOptions.BuildStoreConfig.Enabled) - { - m_BuildStoreService = std::make_unique(m_StatusService, m_StatsService, *m_BuildStore); - } - -#if ZEN_WITH_VFS - m_VfsServiceImpl = std::make_unique(); - m_VfsServiceImpl->AddService(Ref(m_ProjectStore)); - m_VfsServiceImpl->AddService(Ref(m_CacheStore)); - - m_VfsService = std::make_unique(m_StatusService, m_VfsServiceImpl.get()); -#endif // ZEN_WITH_VFS - - ZEN_INFO("initializing GC, enabled '{}', interval {}, lightweight interval {}", - ServerOptions.GcConfig.Enabled, - NiceTimeSpanMs(ServerOptions.GcConfig.IntervalSeconds * 1000ull), - NiceTimeSpanMs(ServerOptions.GcConfig.LightweightIntervalSeconds * 1000ull)); - - GcSchedulerConfig GcConfig{.RootDirectory = m_DataRoot / "gc", - .MonitorInterval = std::chrono::seconds(ServerOptions.GcConfig.MonitorIntervalSeconds), - .Interval = std::chrono::seconds(ServerOptions.GcConfig.IntervalSeconds), - .MaxCacheDuration = std::chrono::seconds(ServerOptions.GcConfig.Cache.MaxDurationSeconds), - .MaxProjectStoreDuration = std::chrono::seconds(ServerOptions.GcConfig.ProjectStore.MaxDurationSeconds), - .MaxBuildStoreDuration = std::chrono::seconds(ServerOptions.GcConfig.BuildStore.MaxDurationSeconds), - .CollectSmallObjects = ServerOptions.GcConfig.CollectSmallObjects, - .Enabled = ServerOptions.GcConfig.Enabled, - .DiskReserveSize = ServerOptions.GcConfig.DiskReserveSize, - .DiskSizeSoftLimit = ServerOptions.GcConfig.DiskSizeSoftLimit, - .MinimumFreeDiskSpaceToAllowWrites = ServerOptions.GcConfig.MinimumFreeDiskSpaceToAllowWrites, - .LightweightInterval = std::chrono::seconds(ServerOptions.GcConfig.LightweightIntervalSeconds), - .UseGCVersion = ServerOptions.GcConfig.UseGCV2 ? GcVersion::kV2 : GcVersion::kV1_Deprecated, - .CompactBlockUsageThresholdPercent = ServerOptions.GcConfig.CompactBlockUsageThresholdPercent, - .Verbose = ServerOptions.GcConfig.Verbose, - .SingleThreaded = ServerOptions.GcConfig.SingleThreaded, - .AttachmentPassCount = ServerOptions.GcConfig.AttachmentPassCount}; - m_GcScheduler.Initialize(GcConfig); - - // Create and register admin interface last to make sure all is properly initialized - m_AdminService = std::make_unique( - m_GcScheduler, - *m_JobQueue, - m_CacheStore.Get(), - [this]() { Flush(); }, - HttpAdminService::LogPaths{.AbsLogPath = ServerOptions.AbsLogFile, - .HttpLogPath = ServerOptions.DataDir / "logs" / "http.log", - .CacheLogPath = ServerOptions.DataDir / "logs" / "z$.log"}, - ServerOptions); -} - -void -ZenStorageServer::InitializeAuthentication(const ZenStorageServerOptions& ServerOptions) -{ - // Setup authentication manager - { - ZEN_TRACE_CPU("ZenStorageServer::InitAuth"); - std::string EncryptionKey = ServerOptions.EncryptionKey; - - if (EncryptionKey.empty()) - { - EncryptionKey = "abcdefghijklmnopqrstuvxyz0123456"; - - if (ServerOptions.IsDedicated) - { - ZEN_WARN("Using default encryption key for authentication state"); - } - } - - std::string EncryptionIV = ServerOptions.EncryptionIV; - - if (EncryptionIV.empty()) - { - EncryptionIV = "0123456789abcdef"; - - if (ServerOptions.IsDedicated) - { - ZEN_WARN("Using default encryption initialization vector for authentication state"); - } - } - - m_AuthMgr = AuthMgr::Create({.RootDirectory = m_DataRoot / "auth", - .EncryptionKey = AesKey256Bit::FromString(EncryptionKey), - .EncryptionIV = AesIV128Bit::FromString(EncryptionIV)}); - - for (const ZenOpenIdProviderConfig& OpenIdProvider : ServerOptions.AuthConfig.OpenIdProviders) - { - m_AuthMgr->AddOpenIdProvider({.Name = OpenIdProvider.Name, .Url = OpenIdProvider.Url, .ClientId = OpenIdProvider.ClientId}); - } - } - - m_AuthService = std::make_unique(*m_AuthMgr); -} - -void -ZenStorageServer::InitializeState(const ZenStorageServerOptions& ServerOptions) -{ - ZEN_TRACE_CPU("ZenStorageServer::InitializeState"); - - // Check root manifest to deal with schema versioning - - bool WipeState = false; - std::string WipeReason = "Unspecified"; - - if (ServerOptions.IsCleanStart) - { - WipeState = true; - WipeReason = "clean start requested"; - } - - bool UpdateManifest = false; - std::filesystem::path ManifestPath = m_DataRoot / "root_manifest"; - Oid StateId = Oid::Zero; - DateTime CreatedWhen{0}; - - if (!WipeState) - { - FileContents ManifestData = ReadFile(ManifestPath); - - if (ManifestData.ErrorCode) - { - if (ServerOptions.IsFirstRun) - { - ZEN_INFO("Initializing state at '{}'", m_DataRoot); - - UpdateManifest = true; - } - else - { - WipeState = true; - WipeReason = fmt::format("No manifest present at '{}'", ManifestPath); - } - } - else - { - IoBuffer Manifest = ManifestData.Flatten(); - - if (CbValidateError ValidationResult = ValidateCompactBinary(Manifest, CbValidateMode::All); - ValidationResult != CbValidateError::None) - { - ZEN_WARN("Manifest validation failed: {}, state will be wiped", zen::ToString(ValidationResult)); - - WipeState = true; - WipeReason = fmt::format("Validation of manifest at '{}' failed: {}", ManifestPath, zen::ToString(ValidationResult)); - } - else - { - m_RootManifest = LoadCompactBinaryObject(Manifest); - - const int32_t ManifestVersion = m_RootManifest["schema_version"].AsInt32(0); - StateId = m_RootManifest["state_id"].AsObjectId(); - CreatedWhen = m_RootManifest["created"].AsDateTime(); - - if (ManifestVersion != ZEN_CFG_SCHEMA_VERSION) - { - std::filesystem::path ManifestSkipSchemaChangePath = m_DataRoot / "root_manifest.ignore_schema_mismatch"; - if (ManifestVersion != 0 && IsFile(ManifestSkipSchemaChangePath)) - { - ZEN_INFO( - "Schema version {} found in '{}' does not match {}, ignoring mismatch due to existance of '{}' and updating " - "schema version", - ManifestVersion, - ManifestPath, - ZEN_CFG_SCHEMA_VERSION, - ManifestSkipSchemaChangePath); - UpdateManifest = true; - } - else - { - WipeState = true; - WipeReason = - fmt::format("Manifest schema version: {}, differs from required: {}", ManifestVersion, ZEN_CFG_SCHEMA_VERSION); - } - } - } - } - } - - if (StateId == Oid::Zero) - { - StateId = Oid::NewOid(); - UpdateManifest = true; - } - - const DateTime Now = DateTime::Now(); - - if (CreatedWhen.GetTicks() == 0) - { - CreatedWhen = Now; - UpdateManifest = true; - } - - // Handle any state wipe - - if (WipeState) - { - ZEN_WARN("Wiping state at '{}' - reason: '{}'", m_DataRoot, WipeReason); - - std::error_code Ec; - for (const std::filesystem::directory_entry& DirEntry : std::filesystem::directory_iterator{m_DataRoot, Ec}) - { - if (DirEntry.is_directory() && (DirEntry.path().filename() != "logs")) - { - ZEN_INFO("Deleting '{}'", DirEntry.path()); - - DeleteDirectories(DirEntry.path(), Ec); - - if (Ec) - { - ZEN_WARN("Delete of '{}' returned error: '{}'", DirEntry.path(), Ec.message()); - } - } - } - - ZEN_INFO("Wiped all directories in data root"); - - UpdateManifest = true; - } - - // Write manifest - - { - CbObjectWriter Cbo; - Cbo << "schema_version" << ZEN_CFG_SCHEMA_VERSION << "created" << CreatedWhen << "updated" << Now << "state_id" << StateId; - - m_RootManifest = Cbo.Save(); - - if (UpdateManifest) - { - TemporaryFile::SafeWriteFile(ManifestPath, m_RootManifest.GetBuffer().GetView()); - } - - if (!ServerOptions.IsTest) - { - try - { - EmitCentralManifest(ServerOptions.SystemRootDir, StateId, m_RootManifest, ManifestPath); - } - catch (const std::exception& Ex) - { - ZEN_WARN("Unable to emit central manifest: ", Ex.what()); - } - } - } - - // Write state marker - - { - std::filesystem::path StateMarkerPath = m_DataRoot / "state_marker"; - static const std::string_view StateMarkerContent = "deleting this file will cause " ZEN_APP_NAME " to exit"sv; - WriteFile(StateMarkerPath, IoBuffer(IoBuffer::Wrap, StateMarkerContent.data(), StateMarkerContent.size())); - - EnqueueStateMarkerTimer(); - } - - EnqueueStateExitFlagTimer(); -} - -void -ZenStorageServer::InitializeStructuredCache(const ZenStorageServerOptions& ServerOptions) -{ - ZEN_TRACE_CPU("ZenStorageServer::InitializeStructuredCache"); - - using namespace std::literals; - - ZEN_INFO("instantiating structured cache service"); - ZenCacheStore::Configuration Config; - Config.AllowAutomaticCreationOfNamespaces = true; - Config.Logging = {.EnableWriteLog = ServerOptions.StructuredCacheConfig.WriteLogEnabled, - .EnableAccessLog = ServerOptions.StructuredCacheConfig.AccessLogEnabled}; - - for (const auto& It : ServerOptions.StructuredCacheConfig.PerBucketConfigs) - { - const std::string& BucketName = It.first; - const ZenStructuredCacheBucketConfig& ZenBucketConfig = It.second; - ZenCacheDiskLayer::BucketConfiguration BucketConfig = {.MaxBlockSize = ZenBucketConfig.MaxBlockSize, - .PayloadAlignment = ZenBucketConfig.PayloadAlignment, - .MemCacheSizeThreshold = ZenBucketConfig.MemCacheSizeThreshold, - .LargeObjectThreshold = ZenBucketConfig.LargeObjectThreshold, - .LimitOverwrites = ZenBucketConfig.LimitOverwrites}; - Config.NamespaceConfig.DiskLayerConfig.BucketConfigMap.insert_or_assign(BucketName, BucketConfig); - } - Config.NamespaceConfig.DiskLayerConfig.BucketConfig.MaxBlockSize = ServerOptions.StructuredCacheConfig.BucketConfig.MaxBlockSize, - Config.NamespaceConfig.DiskLayerConfig.BucketConfig.PayloadAlignment = - ServerOptions.StructuredCacheConfig.BucketConfig.PayloadAlignment, - Config.NamespaceConfig.DiskLayerConfig.BucketConfig.MemCacheSizeThreshold = - ServerOptions.StructuredCacheConfig.BucketConfig.MemCacheSizeThreshold, - Config.NamespaceConfig.DiskLayerConfig.BucketConfig.LargeObjectThreshold = - ServerOptions.StructuredCacheConfig.BucketConfig.LargeObjectThreshold, - Config.NamespaceConfig.DiskLayerConfig.BucketConfig.LimitOverwrites = ServerOptions.StructuredCacheConfig.BucketConfig.LimitOverwrites; - Config.NamespaceConfig.DiskLayerConfig.MemCacheTargetFootprintBytes = ServerOptions.StructuredCacheConfig.MemTargetFootprintBytes; - Config.NamespaceConfig.DiskLayerConfig.MemCacheTrimIntervalSeconds = ServerOptions.StructuredCacheConfig.MemTrimIntervalSeconds; - Config.NamespaceConfig.DiskLayerConfig.MemCacheMaxAgeSeconds = ServerOptions.StructuredCacheConfig.MemMaxAgeSeconds; - - if (ServerOptions.IsDedicated) - { - Config.NamespaceConfig.DiskLayerConfig.BucketConfig.LargeObjectThreshold = 128 * 1024 * 1024; - } - - m_CacheStore = new ZenCacheStore(m_GcManager, *m_JobQueue, m_DataRoot / "cache", Config, m_GcManager.GetDiskWriteBlocker()); - m_OpenProcessCache = std::make_unique(); - - const ZenUpstreamCacheConfig& UpstreamConfig = ServerOptions.UpstreamCacheConfig; - - UpstreamCacheOptions UpstreamOptions; - UpstreamOptions.ReadUpstream = (uint8_t(ServerOptions.UpstreamCacheConfig.CachePolicy) & uint8_t(UpstreamCachePolicy::Read)) != 0; - UpstreamOptions.WriteUpstream = (uint8_t(ServerOptions.UpstreamCacheConfig.CachePolicy) & uint8_t(UpstreamCachePolicy::Write)) != 0; - - if (UpstreamConfig.UpstreamThreadCount < 32) - { - UpstreamOptions.ThreadCount = static_cast(UpstreamConfig.UpstreamThreadCount); - } - - m_UpstreamCache = CreateUpstreamCache(UpstreamOptions, *m_CacheStore, *m_CidStore); - m_UpstreamService = std::make_unique(*m_UpstreamCache, *m_AuthMgr); - m_UpstreamCache->Initialize(); - - if (ServerOptions.UpstreamCacheConfig.CachePolicy != UpstreamCachePolicy::Disabled) - { - // Zen upstream - { - std::vector ZenUrls = UpstreamConfig.ZenConfig.Urls; - if (!UpstreamConfig.ZenConfig.Dns.empty()) - { - for (const std::string& Dns : UpstreamConfig.ZenConfig.Dns) - { - if (!Dns.empty()) - { - const asio::error_code Err = utils::ResolveHostname(m_IoContext, Dns, "8558"sv, ZenUrls); - if (Err) - { - ZEN_ERROR("resolve of '{}' FAILED, reason '{}'", Dns, Err.message()); - } - } - } - } - - std::erase_if(ZenUrls, [](const auto& Url) { return Url.empty(); }); - - if (!ZenUrls.empty()) - { - const auto ZenEndpointName = UpstreamConfig.ZenConfig.Name.empty() ? "Zen"sv : UpstreamConfig.ZenConfig.Name; - - std::unique_ptr ZenEndpoint = UpstreamEndpoint::CreateZenEndpoint( - {.Name = ZenEndpointName, - .Urls = ZenUrls, - .ConnectTimeout = std::chrono::milliseconds(UpstreamConfig.ConnectTimeoutMilliseconds), - .Timeout = std::chrono::milliseconds(UpstreamConfig.TimeoutMilliseconds)}); - - m_UpstreamCache->RegisterEndpoint(std::move(ZenEndpoint)); - } - } - - // Jupiter upstream - if (UpstreamConfig.JupiterConfig.Url.empty() == false) - { - std::string_view EndpointName = UpstreamConfig.JupiterConfig.Name.empty() ? "Jupiter"sv : UpstreamConfig.JupiterConfig.Name; - - auto Options = JupiterClientOptions{.Name = EndpointName, - .ServiceUrl = UpstreamConfig.JupiterConfig.Url, - .DdcNamespace = UpstreamConfig.JupiterConfig.DdcNamespace, - .BlobStoreNamespace = UpstreamConfig.JupiterConfig.Namespace, - .ConnectTimeout = std::chrono::milliseconds(UpstreamConfig.ConnectTimeoutMilliseconds), - .Timeout = std::chrono::milliseconds(UpstreamConfig.TimeoutMilliseconds)}; - - auto AuthConfig = UpstreamAuthConfig{.OAuthUrl = UpstreamConfig.JupiterConfig.OAuthUrl, - .OAuthClientId = UpstreamConfig.JupiterConfig.OAuthClientId, - .OAuthClientSecret = UpstreamConfig.JupiterConfig.OAuthClientSecret, - .OpenIdProvider = UpstreamConfig.JupiterConfig.OpenIdProvider, - .AccessToken = UpstreamConfig.JupiterConfig.AccessToken}; - - std::unique_ptr JupiterEndpoint = UpstreamEndpoint::CreateJupiterEndpoint(Options, AuthConfig, *m_AuthMgr); - - m_UpstreamCache->RegisterEndpoint(std::move(JupiterEndpoint)); - } - } - - m_StructuredCacheService = std::make_unique(*m_CacheStore, - *m_CidStore, - m_StatsService, - m_StatusService, - *m_UpstreamCache, - m_GcManager.GetDiskWriteBlocker(), - *m_OpenProcessCache); - - m_StatsReporter.AddProvider(m_CacheStore.Get()); - m_StatsReporter.AddProvider(m_CidStore.get()); - m_StatsReporter.AddProvider(m_BuildCidStore.get()); -} - -void -ZenStorageServer::Run() -{ - if (m_ProcessMonitor.IsActive()) - { - CheckOwnerPid(); - } - - if (!m_TestMode) - { - ZEN_INFO( - "__________ _________ __ \n" - "\\____ /____ ____ / _____// |_ ___________ ____ \n" - " / // __ \\ / \\ \\_____ \\\\ __\\/ _ \\_ __ \\_/ __ \\ \n" - " / /\\ ___/| | \\ / \\| | ( <_> ) | \\/\\ ___/ \n" - "/_______ \\___ >___| / /_______ /|__| \\____/|__| \\___ >\n" - " \\/ \\/ \\/ \\/ \\/ \n"); - } - - ZEN_INFO(ZEN_APP_NAME " now running (pid: {})", GetCurrentProcessId()); - -#if ZEN_PLATFORM_WINDOWS - if (zen::windows::IsRunningOnWine()) - { - ZEN_INFO("detected Wine session - " ZEN_APP_NAME " is not formally tested on Wine and may therefore not work or perform well"); - } -#endif - -#if ZEN_USE_SENTRY - ZEN_INFO("sentry crash handler {}", m_UseSentry ? "ENABLED" : "DISABLED"); - if (m_UseSentry) - { - SentryIntegration::ClearCaches(); - } -#endif - - if (m_DebugOptionForcedCrash) - { - ZEN_DEBUG_BREAK(); - } - - const bool IsInteractiveMode = IsInteractiveSession() && !m_TestMode; - - SetNewState(kRunning); - - OnReady(); - - if (!m_StartupScrubOptions.empty()) - { - using namespace std::literals; - - ZEN_INFO("triggering scrub with settings: '{}'", m_StartupScrubOptions); - - bool DoScrub = true; - bool DoWait = false; - GcScheduler::TriggerScrubParams ScrubParams; - - ForEachStrTok(m_StartupScrubOptions, ',', [&](std::string_view Token) { - if (Token == "nocas"sv) - { - ScrubParams.SkipCas = true; - } - else if (Token == "nodelete"sv) - { - ScrubParams.SkipDelete = true; - } - else if (Token == "nogc"sv) - { - ScrubParams.SkipGc = true; - } - else if (Token == "no"sv) - { - DoScrub = false; - } - else if (Token == "wait"sv) - { - DoWait = true; - } - return true; - }); - - if (DoScrub) - { - m_GcScheduler.TriggerScrub(ScrubParams); - - if (DoWait) - { - auto State = m_GcScheduler.Status(); - - while ((State != GcSchedulerStatus::kRunning) && (State != GcSchedulerStatus::kStopped)) - { - Sleep(500); - - State = m_GcScheduler.Status(); - } - - ZEN_INFO("waiting for Scrub/GC to complete..."); - - while (State == GcSchedulerStatus::kRunning) - { - Sleep(500); - - State = m_GcScheduler.Status(); - } - - ZEN_INFO("Scrub/GC completed"); - } - } - } - - if (m_IsPowerCycle) - { - ZEN_INFO("Power cycle mode enabled -- shutting down"); - RequestExit(0); - } - - m_Http->Run(IsInteractiveMode); - - SetNewState(kShuttingDown); - - ZEN_INFO(ZEN_APP_NAME " exiting"); -} - -void -ZenStorageServer::Cleanup() -{ - ZEN_TRACE_CPU("ZenStorageServer::Cleanup"); - ZEN_INFO(ZEN_APP_NAME " cleaning up"); - try - { - m_IoContext.stop(); - if (m_IoRunner.joinable()) - { - m_IoRunner.join(); - } - - if (m_Http) - { - m_Http->Close(); - } - - if (m_JobQueue) - { - m_JobQueue->Stop(); - } - - m_StatsReporter.Shutdown(); - m_GcScheduler.Shutdown(); - - Flush(); - - m_AdminService.reset(); - m_VfsService.reset(); - m_VfsServiceImpl.reset(); - m_ObjStoreService.reset(); - m_FrontendService.reset(); - - m_BuildStoreService.reset(); - m_BuildStore = {}; - m_BuildCidStore.reset(); - - m_StructuredCacheService.reset(); - m_UpstreamService.reset(); - m_UpstreamCache.reset(); - m_CacheStore = {}; - m_OpenProcessCache.reset(); - - m_HttpWorkspacesService.reset(); - m_Workspaces.reset(); - m_HttpProjectService.reset(); - m_ProjectStore = {}; - m_CidStore.reset(); - m_AuthService.reset(); - m_AuthMgr.reset(); - m_Http = {}; - - ShutdownWorkerPools(); - - m_JobQueue.reset(); - } - catch (const std::exception& Ex) - { - ZEN_ERROR("exception thrown during Cleanup() in {}: '{}'", ZEN_APP_NAME, Ex.what()); - } -} - -void -ZenStorageServer::EnqueueStateMarkerTimer() -{ - ZEN_MEMSCOPE(GetZenserverTag()); - m_StateMarkerTimer.expires_after(std::chrono::seconds(5)); - m_StateMarkerTimer.async_wait([this](const asio::error_code&) { CheckStateMarker(); }); - EnsureIoRunner(); -} - -void -ZenStorageServer::CheckStateMarker() -{ - ZEN_MEMSCOPE(GetZenserverTag()); - std::filesystem::path StateMarkerPath = m_DataRoot / "state_marker"; - try - { - if (!IsFile(StateMarkerPath)) - { - ZEN_WARN("state marker at {} has been deleted, exiting", StateMarkerPath); - RequestExit(1); - return; - } - } - catch (const std::exception& Ex) - { - ZEN_WARN("state marker at {} could not be checked, reason: '{}'", StateMarkerPath, Ex.what()); - RequestExit(1); - return; - } - EnqueueStateMarkerTimer(); -} - -void -ZenStorageServer::Flush() -{ - ZEN_TRACE_CPU("ZenStorageServer::Flush"); - - if (m_CidStore) - m_CidStore->Flush(); - - if (m_StructuredCacheService) - m_StructuredCacheService->Flush(); - - if (m_ProjectStore) - m_ProjectStore->Flush(); - - if (m_BuildCidStore) - m_BuildCidStore->Flush(); -} - -////////////////////////////////////////////////////////////////////////// - -ZenStorageServerMain::ZenStorageServerMain(ZenStorageServerOptions& ServerOptions) -: ZenServerMain(ServerOptions) -, m_ServerOptions(ServerOptions) -{ -} - -void -ZenStorageServerMain::DoRun(ZenServerState::ZenServerEntry* Entry) -{ - ZenStorageServer Server; - Server.SetDataRoot(m_ServerOptions.DataDir); - Server.SetContentRoot(m_ServerOptions.ContentDir); - Server.SetTestMode(m_ServerOptions.IsTest); - Server.SetDedicatedMode(m_ServerOptions.IsDedicated); - - auto ServerCleanup = MakeGuard([&Server] { Server.Cleanup(); }); - - int EffectiveBasePort = Server.Initialize(m_ServerOptions, Entry); - if (EffectiveBasePort == -1) - { - // Server.Initialize has already logged what the issue is - just exit with failure code here. - std::exit(1); - } - - Entry->EffectiveListenPort = uint16_t(EffectiveBasePort); - if (EffectiveBasePort != m_ServerOptions.BasePort) - { - ZEN_INFO(ZEN_APP_NAME " - relocated to base port {}", EffectiveBasePort); - m_ServerOptions.BasePort = EffectiveBasePort; - } - - std::unique_ptr ShutdownThread; - std::unique_ptr ShutdownEvent; - - ExtendableStringBuilder<64> ShutdownEventName; - ShutdownEventName << "Zen_" << m_ServerOptions.BasePort << "_Shutdown"; - ShutdownEvent.reset(new NamedEvent{ShutdownEventName}); - - // Monitor shutdown signals - - ShutdownThread.reset(new std::thread{[&] { - SetCurrentThreadName("shutdown_monitor"); - - ZEN_INFO("shutdown monitor thread waiting for shutdown signal '{}' for process {}", ShutdownEventName, zen::GetCurrentProcessId()); - - if (ShutdownEvent->Wait()) - { - ZEN_INFO("shutdown signal for pid {} received", zen::GetCurrentProcessId()); - Server.RequestExit(0); - } - else - { - ZEN_INFO("shutdown signal wait() failed"); - } - }}); - - auto CleanupShutdown = MakeGuard([&ShutdownEvent, &ShutdownThread] { - ReportServiceStatus(ServiceStatus::Stopping); - - if (ShutdownEvent) - { - ShutdownEvent->Set(); - } - if (ShutdownThread && ShutdownThread->joinable()) - { - ShutdownThread->join(); - } - }); - - // If we have a parent process, establish the mechanisms we need - // to be able to communicate readiness with the parent - - Server.SetIsReadyFunc([&] { - std::error_code Ec; - m_LockFile.Update(MakeLockData(true), Ec); - ReportServiceStatus(ServiceStatus::Running); - NotifyReady(); - }); - - Server.Run(); -} - -} // namespace zen diff --git a/src/zenserver/zenstorageserver.h b/src/zenserver/zenstorageserver.h deleted file mode 100644 index e4c31399d..000000000 --- a/src/zenserver/zenstorageserver.h +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include "zenserver.h" - -#include -#include -#include -#include -#include -#include - -#include "admin/admin.h" -#include "buildstore/httpbuildstore.h" -#include "cache/httpstructuredcache.h" -#include "diag/diagsvcs.h" -#include "frontend/frontend.h" -#include "objectstore/objectstore.h" -#include "projectstore/httpprojectstore.h" -#include "stats/statsreporter.h" -#include "upstream/upstream.h" -#include "vfs/vfsservice.h" -#include "workspaces/httpworkspaces.h" - -namespace zen { - -class ZenStorageServer : public ZenServerBase -{ - ZenStorageServer& operator=(ZenStorageServer&&) = delete; - ZenStorageServer(ZenStorageServer&&) = delete; - -public: - ZenStorageServer(); - ~ZenStorageServer(); - - void SetDedicatedMode(bool State) { m_IsDedicatedMode = State; } - void SetTestMode(bool State) { m_TestMode = State; } - void SetDataRoot(std::filesystem::path Root) { m_DataRoot = Root; } - void SetContentRoot(std::filesystem::path Root) { m_ContentRoot = Root; } - - int Initialize(const ZenStorageServerOptions& ServerOptions, ZenServerState::ZenServerEntry* ServerEntry); - void Run(); - void Cleanup(); - -private: - void InitializeState(const ZenStorageServerOptions& ServerOptions); - void InitializeStructuredCache(const ZenStorageServerOptions& ServerOptions); - void Flush(); - - bool m_IsDedicatedMode = false; - bool m_TestMode = false; - bool m_DebugOptionForcedCrash = false; - std::string m_StartupScrubOptions; - CbObject m_RootManifest; - std::filesystem::path m_DataRoot; - std::filesystem::path m_ContentRoot; - asio::steady_timer m_StateMarkerTimer{m_IoContext}; - - void EnqueueStateMarkerTimer(); - void CheckStateMarker(); - - std::unique_ptr m_AuthMgr; - std::unique_ptr m_AuthService; - void InitializeAuthentication(const ZenStorageServerOptions& ServerOptions); - - void InitializeServices(const ZenStorageServerOptions& ServerOptions); - void RegisterServices(); - - HttpStatsService m_StatsService; - std::unique_ptr m_JobQueue; - GcManager m_GcManager; - GcScheduler m_GcScheduler{m_GcManager}; - std::unique_ptr m_CidStore; - Ref m_CacheStore; - std::unique_ptr m_OpenProcessCache; - HttpTestService m_TestService; - std::unique_ptr m_BuildCidStore; - std::unique_ptr m_BuildStore; - -#if ZEN_WITH_TESTS - HttpTestingService m_TestingService; -#endif - - RefPtr m_ProjectStore; - std::unique_ptr m_VfsServiceImpl; - std::unique_ptr m_HttpProjectService; - std::unique_ptr m_Workspaces; - std::unique_ptr m_HttpWorkspacesService; - std::unique_ptr m_UpstreamCache; - std::unique_ptr m_UpstreamService; - std::unique_ptr m_StructuredCacheService; - std::unique_ptr m_FrontendService; - std::unique_ptr m_ObjStoreService; - std::unique_ptr m_BuildStoreService; - std::unique_ptr m_VfsService; - std::unique_ptr m_AdminService; -}; - -class ZenStorageServerMain : public ZenServerMain -{ -public: - ZenStorageServerMain(ZenStorageServerOptions& ServerOptions); - virtual void DoRun(ZenServerState::ZenServerEntry* Entry) override; - - ZenStorageServerMain(const ZenStorageServerMain&) = delete; - ZenStorageServerMain& operator=(const ZenStorageServerMain&) = delete; - -private: - ZenStorageServerOptions& m_ServerOptions; -}; - -} // namespace zen -- cgit v1.2.3